diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 76053a82edc3..024671e20fa6 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -84,8 +84,10 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-main-agent-working-agreement.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-newer-content-host.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-pre-spawn-first-answer.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.test.ts', @@ -107,10 +109,12 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-at-rest.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-live-send.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-send-during-startup-lease-check.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts', @@ -118,7 +122,9 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-writer.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-damaged-chat.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-child-work.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-child.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-children.test.ts', @@ -183,6 +189,8 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/runtime/structured-agent-session-rollback-compatibility.test.ts', 'src/main/runtime/structured-agent-session-spawn-token-descendants.test.ts', 'src/main/runtime/structured-agent-session-startup-reconcile.test.ts', + 'src/main/runtime/structured-agent-session-startup-restoration.test.ts', + 'src/main/runtime/structured-agent-session-tab-listing-startup.test.ts', 'src/main/runtime/structured-claude-pending-rewind.test.ts', 'src/main/ssh-expired-lease-pane-readoption.test.ts', 'src/main/ssh-reattach-pane-cardinality.test.ts', diff --git a/docs/reference/agent-status-store.md b/docs/reference/agent-status-store.md index 912b14d51fe9..77c503837bb3 100644 --- a/docs/reference/agent-status-store.md +++ b/docs/reference/agent-status-store.md @@ -128,11 +128,23 @@ forgot the row would strand a permanently working-looking agent. Two rules the ingest must keep: -- **Never persist a structured row.** The journal is the durable truth for a - structured session and the host republishes on restore. A structured row in - `last-status.json` would hydrate as `restoredUnconfirmed` and then fight the - live republish. The serializer skips rows carrying `structuredHost`, and - hydrate drops any such row found on disk. Applying one therefore also skips +- **Never persist a structured row in the store.** The chat journal database + is the durable truth for a structured session. Each chat's status is stored + there beside its journal (`journal_session_state`), written in the same + transaction as the journal rows it describes, so it is always current. A + chat last written before that table existed, by an older build, or whose row + was derived by other rules, has no current row: it gets one when it is + opened, which also settles it and publishes its status. A listed one is + opened by the background restore after the tab listing answers, as is a + listed chat whose history is still in an older build's per-chat file. A + corrupt chat gets no row until its open rebuilds it. + At host startup, before the tab listing answers, the host republishes the + row of each settled chat that is still listed and not already open, without + opening it; after that it settles every chat a gone process left with work. + A structured row in `last-status.json` would + hydrate as `restoredUnconfirmed` and fight that republish, so the serializer + still skips rows carrying `structuredHost`, and hydrate still drops any found + on disk. Applying one therefore also skips the persist schedule: the walk and stringify could only reproduce the file that is already on disk, once per debounce window for every streaming chat. - **Never let it fight a hook row.** A structured session has no PTY, so no diff --git a/src/main/agent-hooks/first-work-branch-rename.test.ts b/src/main/agent-hooks/first-work-branch-rename.test.ts index 0adc02d7eaa5..ab04fb484157 100644 --- a/src/main/agent-hooks/first-work-branch-rename.test.ts +++ b/src/main/agent-hooks/first-work-branch-rename.test.ts @@ -3,6 +3,7 @@ import type { GlobalSettings } from '../../shared/global-settings-types' import type { Repo } from '../../shared/repo-types' import type { AgentJournalRenderItem } from '../../shared/agent-session-journal-types' import type { AgentSessionJournal } from '../native-chat/agent-session-journal/journal-store' +import { projectStructuredAgentSessionStatusState } from '../../shared/structured-agent-session-projection' import { StructuredAgentSessionStatusFeed } from '../native-chat/agent-session-wire/structured-agent-session-status-feed' import { maybeAutoRenameWorkspaceOnFirstStructuredTurn } from './first-work-structured-session-rename' import { WORKTREE_ID_SEPARATOR } from '../../shared/worktree/id' @@ -104,12 +105,15 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { // A real journal's sequence only ever advances, so the feed's projection // cache must miss on every publish here: this test is about the rename. let sequence = 0 - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks and snapshot() of a journal. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a stand-in journal offering only what the status feed reads (submissions, cursor, Stop marks, its projection, activity). const journal = { - snapshot: () => ({ items, submissions: [] }), + submissions: () => [], stopMarks: { latest: () => null, revision: () => 0 }, lastActivityAt: () => 1, - cursor: () => ({ epoch: 1, sequence: (sequence += 1) }) + cursor: () => ({ epoch: 1, sequence: (sequence += 1) }), + sessionStatus: { + at: (fence?: number) => projectStructuredAgentSessionStatusState(items, [], fence) + } } as unknown as AgentSessionJournal const pending: Promise[] = [] const observe = vi.fn((summary, options) => { @@ -193,24 +197,35 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { const { deps, setDisplayName, setRenameError } = makeDeps({ getRepo: () => ({ id: REPO_ID, kind: 'folder', path: '/workspace/platform' }) as Repo }) - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks and snapshot() of a journal. + const items: AgentJournalRenderItem[] = [ + { + itemId: 'user-1', + sequence: 1, + revision: 1, + observedAt: 1, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Fix auth' }] } + }, + { + itemId: 'turn-1', + sequence: 2, + revision: 1, + observedAt: 1, + body: { + kind: 'status', + text: 'Working', + turnLifecycle: { turnId: 'turn-1', state: 'running' } + } + } + ] + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a stand-in journal offering only what the status feed reads (submissions, cursor, Stop marks, its projection, activity). const journal = { lastActivityAt: () => 1, stopMarks: { latest: () => null, revision: () => 0 }, cursor: () => ({ epoch: 1, sequence: 1 }), - snapshot: () => ({ - items: [ - { body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Fix auth' }] } }, - { - body: { - kind: 'status', - text: 'Working', - turnLifecycle: { turnId: 'turn-1', state: 'running' } - } - } - ], - submissions: [] - }) + sessionStatus: { + at: (fence?: number) => projectStructuredAgentSessionStatusState(items, [], fence) + }, + submissions: () => [] } as unknown as AgentSessionJournal const location = { executionHostId: 'local' as const, diff --git a/src/main/claude/claude-background-wake-retention.test.ts b/src/main/claude/claude-background-wake-retention.test.ts index 427e24e89609..22b355258f32 100644 --- a/src/main/claude/claude-background-wake-retention.test.ts +++ b/src/main/claude/claude-background-wake-retention.test.ts @@ -7,6 +7,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import type { AgentJournalSubmission } from '../../shared/agent-session-journal-types' import { structuredRunningChildWork } from '../../shared/agent-child-work-listing' +import { projectStructuredAgentSessionStatusState } from '../../shared/structured-agent-session-projection' import type { AgentHookServer } from '../agent-hooks/server' import type { StructuredAgentSessionJournalProjections } from '../native-chat/agent-session-wire/structured-agent-session-status-journal-projection' import { StructuredAgentSessionStatusFeed } from '../native-chat/agent-session-wire/structured-agent-session-status-feed' @@ -29,23 +30,29 @@ async function wiredSession() { const run = await producer(host) const submissions: AgentJournalSubmission[] = [] let sequence = 0 + const snapshot = () => ({ + items: [...run.journalItems.values()] + .sort((a, b) => a.sequence - b.sequence) + .map((item) => ({ ...item, revision: 1, observedAt: item.sequence })), + submissions: submissions.map((submission) => ({ ...submission })) + }) const journal = { cursor: () => ({ epoch: 'epoch-1', sequence: ++sequence }), lastActivityAt: () => 1, - submissions: (): ReturnType => - submissions.map((submission) => ({ ...submission })), + submissions: (): ReturnType => snapshot().submissions, item: (itemId: string): ReturnType => { const item = run.journalItems.get(itemId) return item ? { ...item, revision: 1, observedAt: run.now() } : null }, // No Stop was ever pressed here. stopMarks: { latest: () => null, revision: () => 0 }, - snapshot: () => ({ - items: [...run.journalItems.values()] - .sort((a, b) => a.sequence - b.sequence) - .map((item) => ({ ...item, revision: 1, observedAt: run.now() })), - submissions: submissions.map((submission) => ({ ...submission })) - }) + // The journal's own projection, which the feed shares with the status stored beside it. + sessionStatus: { + at: (fence?: number) => { + const { items, submissions: sent } = snapshot() + return projectStructuredAgentSessionStatusState(items, sent, fence) + } + } } const feed = new StructuredAgentSessionStatusFeed({ logger: createStructuredAgentSessionLogger(), @@ -53,7 +60,7 @@ async function wiredSession() { [ parent.sessionId, { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fixture supplies the feed's cursor, clock, Stop marks, snapshot, submissions and item reads. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fixture supplies the feed's cursor, clock, Stop marks, submissions, item reads and projection. journal: journal as unknown as Journal, params: { location: { diff --git a/src/main/claude/claude-subagent-resume-after-restart.test.ts b/src/main/claude/claude-subagent-resume-after-restart.test.ts index d95dd3a48ffc..4275b0bbac30 100644 --- a/src/main/claude/claude-subagent-resume-after-restart.test.ts +++ b/src/main/claude/claude-subagent-resume-after-restart.test.ts @@ -14,6 +14,10 @@ import { import type { AgentSessionJournal } from '../native-chat/agent-session-journal/journal-store' import { createTrackedJournalOpener } from '../native-chat/agent-session-journal/journal-host-database-test-support' import { createDeferredStructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' +import { + appendOpenSettlement, + planOpenSettlement +} from '../native-chat/agent-session-wire/structured-agent-session-open-settlement' import { createClaudeJournalTranslator } from './claude-structured-journal-translation' import { claudeSubagentGroupBody, claudeSubagentGroupIdentity } from './claude-subagent-group-row' import { testEventSinkLogging } from '../native-chat/agent-session-wire/structured-agent-session-logger-test-support' @@ -455,6 +459,15 @@ describe('a Claude subagent resumed after its provider restarted', () => { // contact was lost, not when. await crashed.close() const journal = await openJournal() + // What the conversation's open appends before an acquisition reads the journal. + await appendOpenSettlement( + journal, + planOpenSettlement(journal, null, { acquisition: true }), + 0, + (error) => { + throw error + } + ) const [lost] = rowsListing(journal, 'agent-a')[0]?.agents ?? [] expect(lost).toMatchObject({ id: 'agent-a', state: 'unverifiable' }) expect(lost?.settledAt).toBeUndefined() diff --git a/src/main/native-chat/agent-session-journal/journal-append-status-commit.test.ts b/src/main/native-chat/agent-session-journal/journal-append-status-commit.test.ts new file mode 100644 index 000000000000..38c857f42fbe --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-append-status-commit.test.ts @@ -0,0 +1,206 @@ +// An append's rows, the send ledger answer riding it, and the chat status they give commit in ONE +// transaction, the status written once from the fold holding every row; a failed COMMIT leaves no +// answer, no status write and no fold change. + +import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import { + createTrackedJournalOpener, + loadTestJournal, + openTestJournalHostDatabase, + readTestJournalSessionStatus +} from './journal-host-database-test-support' +import { renderJournalState } from './journal-reducer' +import type { JournalOperationReceipt } from './journal-row-writer' +import { deriveJournalSessionStatus } from './journal-session-state' +import type { AgentSessionJournal } from './journal-store' + +const SESSION = 'session-append' +const FENCE = 3 +const IDENTITY: AgentSessionJournalIdentity = { + sessionId: SESSION, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: codexProviderHandle('thread-append') +} + +const journals = createTrackedJournalOpener() +let root: string +let clock = 1_000 + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-append-status-commit-')) + clock = 1_000 +}) + +afterEach(async () => { + vi.restoreAllMocks() + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +const db = () => openTestJournalHostDatabase(root).db +const stored = () => readTestJournalSessionStatus(root, SESSION) + +async function openChat(): Promise { + const journal = await journals.open({ + identity: IDENTITY, + stateDirectory: root, + now: () => (clock += 1), + mintEpoch: () => 'epoch-1', + currentFence: () => FENCE + }) + // Committed status writes: a rolled-back transaction takes its trigger rows with it. + db().exec(` + CREATE TEMP TABLE IF NOT EXISTS status_writes (session_id TEXT); + CREATE TEMP TABLE IF NOT EXISTS ledger_answers (operation_id TEXT); + CREATE TEMP TRIGGER IF NOT EXISTS count_status_insert AFTER INSERT ON main.journal_session_state + BEGIN INSERT INTO status_writes VALUES (new.session_id); END; + CREATE TEMP TRIGGER IF NOT EXISTS count_status_update AFTER UPDATE ON main.journal_session_state + BEGIN INSERT INTO status_writes VALUES (new.session_id); END;`) + return journal +} + +function statusWrites(): number { + const row = db().prepare('SELECT COUNT(*) AS n FROM status_writes').get() + return typeof row === 'object' && row !== null && 'n' in row ? Number(row.n) : -1 +} + +function ledgerAnswers(): number { + const row = db().prepare('SELECT COUNT(*) AS n FROM ledger_answers').get() + return typeof row === 'object' && row !== null && 'n' in row ? Number(row.n) : -1 +} + +/** A send's ledger answer: one row inside the append's transaction, adopted after its COMMIT. */ +function ledgerReceipt(): JournalOperationReceipt & { adopted: number; inTransaction: boolean[] } { + const inTransaction: boolean[] = [] + const receipt = { + adopted: 0, + inTransaction, + write: (connection: ReturnType) => { + receipt.inTransaction.push(connection.isTransaction) + connection.prepare('INSERT INTO ledger_answers VALUES (?)').run('op-1') + }, + committed: () => { + receipt.adopted += 1 + } + } + return receipt +} + +/** What a fresh open of the chat derives, read back from disk. */ +function freshDerivation() { + const loaded = loadTestJournal(root, SESSION)! + return deriveJournalSessionStatus(loaded.state, { currentFence: FENCE }) +} + +/** Fails the next COMMIT the connection runs, once. */ +function failNextCommit(): void { + const connection = db() + const exec = connection.exec.bind(connection) + let failing = true + vi.spyOn(connection, 'exec').mockImplementation((sql: string) => { + if (failing && sql === 'COMMIT') { + failing = false + throw new Error('COMMIT failed: disk I/O error') + } + return exec(sql) + }) +} + +function send(journal: AgentSessionJournal, id: string, receipt?: JournalOperationReceipt) { + return journal.appendSubmission( + { + clientMessageId: id, + payloadFingerprint: `fp-${id}`, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: id }] }, + fence: FENCE, + handoverRecorded: true + }, + undefined, + receipt + ) +} + +function startFailureBatch() { + return { + settlementId: 'start-failure:gen-1', + fence: FENCE, + recovered: true as const, + mutations: [ + { + kind: 'item' as const, + identity: { provider: 'orca', clientMessageId: 'start-failure:gen-1' } as const, + body: { kind: 'status' as const, tone: 'error' as const, text: 'Codex did not start.' }, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + } + ], + rejectsQueued: agentSessionFailureWords(agentSessionFailureFact('providerStartFailed'), { + surface: 'rejection' + }) + } +} + +describe("a send's ledger answer", () => { + it('commits with its row and a status written once, then is adopted', async () => { + const journal = await openChat() + const writesBefore = statusWrites() + const receipt = ledgerReceipt() + + await send(journal, 'send-1', receipt) + + expect(receipt.inTransaction).toEqual([true]) + expect(receipt.adopted).toBe(1) + expect(ledgerAnswers()).toBe(1) + expect(statusWrites() - writesBefore).toBe(1) + expect(stored()).toEqual(freshDerivation()) + expect(stored()).toMatchObject({ queuedSends: 1 }) + }) + + it('leaves no answer, no status write and no fold change when the COMMIT fails', async () => { + const journal = await openChat() + await send(journal, 'earlier') + const tip = journal.cursor() + const before = stored() + const writesBefore = statusWrites() + const receipt = ledgerReceipt() + failNextCommit() + + await expect(send(journal, 'send-1', receipt)).rejects.toThrow('COMMIT failed') + + expect(receipt.inTransaction).toEqual([true]) + expect(receipt.adopted).toBe(0) + expect(ledgerAnswers()).toBe(0) + expect(statusWrites()).toBe(writesBefore) + expect(stored()).toEqual(before) + expect(journal.cursor()).toEqual(tip) + expect(journal.submission('send-1')).toBeUndefined() + expect(journal.snapshot()).toEqual(renderJournalState(loadTestJournal(root, SESSION)!.state)) + }) +}) + +// A failed COMMIT of this batch is in the fold-undo property, through the corpus's failed start. +describe("a failed start's rows", () => { + it('commit with one status write, from the fold that holds every row', async () => { + const journal = await openChat() + await send(journal, 'first') + await send(journal, 'second') + const writesBefore = statusWrites() + + await journal.appendLifecycleBatch(startFailureBatch()) + + expect(statusWrites() - writesBefore).toBe(1) + expect(stored()).toEqual(freshDerivation()) + expect(stored()).toMatchObject({ queuedSends: 0 }) + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-database-records-migration.test.ts b/src/main/native-chat/agent-session-journal/journal-database-records-migration.test.ts index 120588b306dd..83c806b36e84 100644 --- a/src/main/native-chat/agent-session-journal/journal-database-records-migration.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-database-records-migration.test.ts @@ -206,6 +206,42 @@ function openAsVersion3Build(path: string): { db: Database.Database; readOnly: b return { db: probe, readOnly: false } } +/** A build at version 4 that knows nothing of the status table: it latches only a higher version. */ +function openAsVersion4Build(path: string): { db: Database.Database; readOnly: boolean } { + const probe = new Database(path) + if (journalPragmaNumber(probe, 'user_version') > 4) { + probe.close() + return { db: new Database(path, { readonly: true, fileMustExist: true }), readOnly: true } + } + return { db: probe, readOnly: false } +} + +describe('each chat status, beside the version-4 tables', () => { + it('adds its table with no version bump, so a version-4 build still opens the database writable', () => { + seedVersion3() + openJournalDatabase(dbPath, importOf('session-a')).db.close() + inspect((db) => db.exec('DROP TABLE journal_session_state')) + + openJournalDatabase(dbPath, importOf('session-b')).db.close() + + inspect((db) => { + expect(journalPragmaNumber(db, 'user_version')).toBe(4) + expect(hasTable(db, 'journal_session_state')).toBe(true) + }) + const older = openAsVersion4Build(dbPath) + try { + expect(older.readOnly).toBe(false) + older.db + .prepare( + "INSERT INTO journal_sessions (session_id, workspace_id, epoch) VALUES ('s2', 'ws', 'e')" + ) + .run() + } finally { + older.db.close() + } + }) +}) + describe('a build from before the move', () => { it('opens a version-4 database read-only, so it never writes beside the records', () => { openJournalDatabase(dbPath, importOf('session-a')).db.close() diff --git a/src/main/native-chat/agent-session-journal/journal-database.ts b/src/main/native-chat/agent-session-journal/journal-database.ts index a306ad6e564e..9b1125cc7efa 100644 --- a/src/main/native-chat/agent-session-journal/journal-database.ts +++ b/src/main/native-chat/agent-session-journal/journal-database.ts @@ -15,6 +15,7 @@ import { } from './journal-database-schema' import { JournalUnreleasedSchemaError } from './journal-open-failure' import { ensureQueuedMessagesTable } from './queued-message-schema' +import { ensureJournalSessionStatusTable } from './journal-session-state' export const JOURNAL_BUSY_TIMEOUT_MS = 5000 /** Bounds the WAL a checkpoint leaves behind; SQLite truncates it back to this after a reset. */ @@ -127,9 +128,10 @@ export function openJournalDatabase( configureJournalPragmas(probe, stored) const legacyRecordImportOwed = migrateJournalSchema(probe, stored, legacyRecords) // Outside `migrateJournalSchema` on purpose: its early return skips a db - // already at the current version, and this table must exist at EVERY + // already at the current version, and these tables must exist at EVERY // writable open with no `user_version` bump (see `ensureQueuedMessagesTable`). ensureQueuedMessagesTable(probe) + ensureJournalSessionStatusTable(probe) hardenSqliteDatabaseFiles(dbPath) transferred = true return { db: probe, readOnly: false, legacyRecordImportOwed } diff --git a/src/main/native-chat/agent-session-journal/journal-derived-turn-scope.ts b/src/main/native-chat/agent-session-journal/journal-derived-turn-scope.ts index 0fbadede3bc7..0954a42da44f 100644 --- a/src/main/native-chat/agent-session-journal/journal-derived-turn-scope.ts +++ b/src/main/native-chat/agent-session-journal/journal-derived-turn-scope.ts @@ -20,6 +20,13 @@ import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' export class JournalDerivedTurnScope { private openTurnItemId: string | null = null + /** For an append's undo: the scope as it stood before the row. */ + clone(): JournalDerivedTurnScope { + const copy = new JournalDerivedTurnScope() + copy.openTurnItemId = this.openTurnItemId + return copy + } + /** The scope for a row being created now that states none. A turn record belongs to no turn. */ scopeFor(body: AgentJournalItemBody): AgentJournalTurnScope { return this.openTurnItemId === null || readAgentJournalTurn(body) diff --git a/src/main/native-chat/agent-session-journal/journal-dispatch-reducer.ts b/src/main/native-chat/agent-session-journal/journal-dispatch-reducer.ts index 305b7383ad9e..1c1cf806b61a 100644 --- a/src/main/native-chat/agent-session-journal/journal-dispatch-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-dispatch-reducer.ts @@ -20,11 +20,14 @@ export function applyJournalDispatchRow( state: JournalReducerState, row: Extract ): void { - const submission = state.submissions.get(row.clientMessageId) + const current = state.submissions.get(row.clientMessageId) // Shared with the queued-draft returned hook: a row ignored here must not alter a draft. - if (!submission || !journalDispatchRowApplies(submission)) { + if (!current || !journalDispatchRowApplies(current)) { return } + // Replaced, never changed in place: a holder of the old object keeps what had committed. + const submission = { ...current } + state.submissions.set(row.clientMessageId, submission) submission.fence = row.fence submission.dispatchState = row.state submission.providerItemId = row.providerItemId diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts index 3a79762ee2dd..488623d0dd39 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts @@ -5,7 +5,7 @@ import type { import type { JournalHostDatabase } from './journal-host-database' import { replaceJournalEpoch, type JournalReplacementItem } from './journal-epoch-replacement' import type { JournalQueuePauseRestatement } from './queued-message-pause' -import { publishNewEpoch } from './journal-epoch-rollover' +import { publishNewEpoch, type JournalEpochStateWriter } from './journal-epoch-rollover' import type { JournalLoad } from './journal-open' import type { AgentJournalEpochReason } from './journal-row-schema' import { assertJournalFence, assertJournalWritable } from './journal-write-guards' @@ -25,6 +25,8 @@ export class JournalEpochController { queuePauseRestatement: () => JournalQueuePauseRestatement cursor: () => AgentJournalCursor adopt: (loaded: JournalLoad) => void + /** The chat's status for the new epoch, inside its transaction. */ + writeState: JournalEpochStateWriter } ) {} @@ -36,6 +38,7 @@ export class JournalEpochController { reason, fence, now: this.deps.now(), + writeState: this.deps.writeState, onPublished: this.deps.adopt }) } @@ -72,6 +75,7 @@ export class JournalEpochController { queuePause: this.deps.queuePauseRestatement(), now: this.deps.now, mintEpoch: this.deps.mintEpoch, + writeState: this.deps.writeState, onPublished: this.deps.adopt }) return this.deps.cursor() diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts index 6337b00417bf..eec8431e43f8 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts @@ -71,6 +71,7 @@ function replace(input: { queuePause: { lifted: false, liveStop: null }, now, mintEpoch: () => `epoch-${clock}`, + writeState: () => undefined, onPublished: input.onPublished ?? (() => undefined) }) } diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts index da53b2e9a11d..64938bb24983 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts @@ -1,7 +1,7 @@ // Republishing a live item set into a fresh epoch. // // One transaction: discard the old epoch's rows, insert the epoch row plus the -// replacement items, and move the session projection. +// replacement items, move the session projection, and write the chat's status for the new epoch. import type { AgentJournalItemBody, @@ -13,6 +13,7 @@ import type { import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import type { JournalHostDatabase } from './journal-host-database' import type { JournalLoad } from './journal-open' +import type { JournalEpochStateWriter } from './journal-epoch-rollover' import { applyJournalRow, createJournalReducerState } from './journal-reducer' import { buildJournalItemRow, journalRowBase } from './journal-row-builders' import { @@ -49,6 +50,7 @@ export function replaceJournalEpoch(input: { queuePause: JournalQueuePauseRestatement now: () => number mintEpoch: () => string + writeState: JournalEpochStateWriter /** Called the instant the transaction commits, before any fallible follow-up. */ onPublished: (loaded: JournalLoad) => void }): void { @@ -100,6 +102,7 @@ export function replaceJournalEpoch(input: { insertJournalRow(db, sessionId, row) } publishJournalSessionEpoch(db, input.identity, epoch) + input.writeState(db, state) }) // COMMIT landed: on disk the superseded rows are gone and this epoch is the diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts b/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts index 477a79f11ed1..9e4f056d1a27 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts @@ -1,15 +1,20 @@ // Opening a new epoch. // // One transaction: discard every row of the superseded epoch, insert the new -// epoch row at sequence 1, and move the session projection onto it. Superseded -// rows are DELETED rather than retained — nothing would ever shed them. +// epoch row at sequence 1, move the session projection onto it, and write the chat's status for +// the new epoch. Superseded rows are DELETED rather than retained — nothing would ever shed them. import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import type { JournalHostDatabase } from './journal-host-database' import type { JournalLoad } from './journal-open' -import { applyJournalRow, createJournalReducerState } from './journal-reducer' +import type Database from '../../sqlite/sync-database' +import { + applyJournalRow, + createJournalReducerState, + type JournalReducerState +} from './journal-reducer' import { deleteJournalEpochRows, insertJournalRow, @@ -18,6 +23,10 @@ import { } from './journal-row-table' import type { AgentJournalEpochReason, JournalRow } from './journal-row-schema' +/** Writes the chat's status from the fold an epoch transaction is about to publish. A throw fails + * the transaction. */ +export type JournalEpochStateWriter = (db: Database.Database, state: JournalReducerState) => void + export function publishNewEpoch(input: { database: JournalHostDatabase identity: AgentSessionJournalIdentity @@ -25,6 +34,7 @@ export function publishNewEpoch(input: { reason: AgentJournalEpochReason fence: number now: number + writeState: JournalEpochStateWriter /** Called the instant the transaction commits, before any fallible follow-up. */ onPublished: (loaded: JournalLoad) => void }): void { @@ -41,6 +51,10 @@ export function publishNewEpoch(input: { } const { sessionId } = input.identity + // Folded first, so the transaction stores the status of the epoch it publishes. + const state = createJournalReducerState(sessionId, input.epoch) + applyJournalRow(state, row) + state.oldestSequence = 1 input.database.transaction((db) => { const retired = readJournalSessionEpoch(db, sessionId) if (retired !== null) { @@ -48,13 +62,11 @@ export function publishNewEpoch(input: { } insertJournalRow(db, sessionId, row) publishJournalSessionEpoch(db, input.identity, input.epoch) + input.writeState(db, state) }) // COMMIT landed: on disk the superseded prefix is gone and this epoch is the // live one. The caller adopts that immediately, or a later failure leaves the // store writing into an epoch that no longer exists. - const state = createJournalReducerState(sessionId, input.epoch) - applyJournalRow(state, row) - state.oldestSequence = 1 input.onPublished({ state, newer: null, damage: null }) } diff --git a/src/main/native-chat/agent-session-journal/journal-fold-holder.ts b/src/main/native-chat/agent-session-journal/journal-fold-holder.ts new file mode 100644 index 000000000000..35020c5ff164 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-fold-holder.ts @@ -0,0 +1,44 @@ +// A journal store's fold and the one way to read it. A failed append whose undo could not put the +// fold back marks it stale; the next read re-folds it from what committed, so no reader or writer +// gets a fold the disk never held. + +import type { JournalHostDatabase } from './journal-host-database' +import { replayJournal } from './journal-open' +import { createJournalReducerState, type JournalReducerState } from './journal-reducer' + +export class JournalFoldHolder { + private fold: JournalReducerState + private stale = false + + constructor( + private readonly sessionId: string, + private readonly database: JournalHostDatabase + ) { + this.fold = createJournalReducerState(sessionId, '') + } + + get(): JournalReducerState { + if (this.stale) { + const { db } = this.database + if (db.isTransaction) { + // A transaction would read its own uncommitted rows back as the fold. + throw new Error(`agent-session journal ${this.sessionId} is stale mid-transaction`) + } + const reloaded = replayJournal(db, this.sessionId) + if (!reloaded) { + throw new Error(`agent-session journal ${this.sessionId} is gone`) + } + this.set(reloaded.state) + } + return this.fold + } + + set(fold: JournalReducerState): void { + this.fold = fold + this.stale = false + } + + markStale(): void { + this.stale = true + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts b/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts new file mode 100644 index 000000000000..ff1961d7b6ca --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts @@ -0,0 +1,335 @@ +// The write path, as a property: every append first runs with a failing COMMIT, then for real, and +// after each the in-memory fold equals a fresh replay of the disk, container by container with key +// order, scalars and turn scope, and the stored status equals a fresh derivation. At the end of each +// chat, startup's selection reads the pinned answer. +// +// The harness wraps the store's private row writer: a writer refactor breaks it loudly. + +import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { activeStructuredAgentSessionTurnIdBySequence } from '../../../shared/structured-agent-session-live-turn' +import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { + AgentJournalItemIdentity, + AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import { + createTrackedJournalOpener, + loadTestJournal, + openTestJournalHostDatabase, + readTestJournalSessionStatus +} from './journal-host-database-test-support' +import type { JournalFoldHolder } from './journal-fold-holder' +import { JournalQueuedMessages } from './journal-queued-messages' +import { + MAX_JOURNAL_APPLIED_SETTLEMENT_IDS, + renderJournalState, + type JournalReducerState +} from './journal-reducer' +import { + deriveJournalSessionStatus, + isUnsettledJournalSessionStatus, + readUnsettledJournalSessionIds +} from './journal-session-state' +import { + CORPUS_FENCE, + CORPUS_UNSETTLED, + JOURNAL_SESSION_STATE_CASES, + JOURNAL_SESSION_STATE_CORPUS, + type JournalSessionStateCase +} from './journal-session-state-test-corpus' +import type { AgentSessionJournal } from './journal-store' + +const journals = createTrackedJournalOpener() +let root: string +let clock = 1_000 + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-fold-undo-')) + clock = 1_000 +}) + +afterEach(async () => { + vi.restoreAllMocks() + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +function identity(sessionId: string): AgentSessionJournalIdentity { + return { + sessionId, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: codexProviderHandle(`thread-${sessionId}`) + } +} + +function open(sessionId: string): Promise { + return journals.open({ + identity: identity(sessionId), + stateDirectory: root, + now: () => (clock += 1), + mintEpoch: () => `epoch-${sessionId}-${clock}`, + currentFence: () => CORPUS_FENCE + }) +} + +function failNextCommit(): () => void { + const connection = openTestJournalHostDatabase(root).db + const exec = connection.exec.bind(connection) + let failing = true + const spy = vi.spyOn(connection, 'exec').mockImplementation((sql: string) => { + if (failing && sql === 'COMMIT') { + failing = false + throw new Error('COMMIT failed: disk I/O error') + } + return exec(sql) + }) + return () => spy.mockRestore() +} + +type JournalInternals = { + fold: JournalFoldHolder + rowWriter: { enqueue: (build: unknown, hook?: unknown) => Promise } +} + +function internals(journal: AgentSessionJournal): JournalInternals { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the store's private fold and row writer, read to compare against a replay; both fields exist on every store. + return journal as unknown as JournalInternals +} + +/** Every scalar, every container in key order, and the turn scope's open turn. */ +function comparableFold(state: JournalReducerState) { + const { derivedTurnScope, ...rest } = state + return { + ...Object.fromEntries( + Object.entries(rest).map(([key, value]) => [ + key, + value instanceof Map || value instanceof Set ? [...value.entries()] : value + ]) + ), + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the scope's private open turn, the one field a clone carries. + openTurn: (derivedTurnScope as unknown as { openTurnItemId: string | null }).openTurnItemId + } +} + +/** The fold as the store serves it: a stale fold is re-read first. */ +function servedFold(journal: AgentSessionJournal): JournalReducerState { + return internals(journal).fold.get() +} + +function expectFoldIsTheDisk(journal: AgentSessionJournal, sessionId: string): void { + const loaded = loadTestJournal(root, sessionId)! + expect(comparableFold(servedFold(journal))).toEqual(comparableFold(loaded.state)) + expect(readTestJournalSessionStatus(root, sessionId)).toEqual( + deriveJournalSessionStatus(loaded.state, { currentFence: CORPUS_FENCE }) + ) +} + +/** Every append is first tried with a failing COMMIT, then for real, one append at a time. + * Answers every check that failed, so a case that swallows an append's error still reports it. */ +function failEveryAppendOnce(journal: AgentSessionJournal, sessionId: string): unknown[] { + const violations: unknown[] = [] + const check = (run: () => void) => { + try { + run() + } catch (error) { + violations.push(error) + throw error + } + } + const writer = internals(journal).rowWriter + const enqueue = writer.enqueue.bind(writer) + let tail: Promise = Promise.resolve() + const twice = async (build: unknown, hook?: unknown) => { + const statusBefore = readTestJournalSessionStatus(root, sessionId) + const restore = failNextCommit() + const failed = await enqueue(build, hook).then( + () => new Error('the failing COMMIT committed'), + (error: unknown) => error + ) + restore() + check(() => { + expect(readTestJournalSessionStatus(root, sessionId)).toEqual(statusBefore) + expectFoldIsTheDisk(journal, sessionId) + }) + if (!String(failed).includes('COMMIT failed')) { + // Refused before its COMMIT (a revision rule, a held id): nothing to retry. + throw failed + } + const committed = await enqueue(build, hook) + check(() => expectFoldIsTheDisk(journal, sessionId)) + return committed + } + writer.enqueue = (build, hook) => { + const run = tail.then(() => twice(build, hook)) + tail = run.catch(() => undefined) + return run + } + return violations +} + +function assistant(text: string) { + return { + kind: 'message' as const, + role: 'assistant' as const, + blocks: [{ type: 'text' as const, text }] + } +} + +function codexItem(turnId: string, ordinal: number): AgentJournalItemIdentity { + return { provider: 'codex', threadId: 'thread-corpus', turnId, ordinal } +} + +const THREAD = { fence: CORPUS_FENCE, turnScope: { kind: 'thread' as const } } + +/** What startup reads of the chat's final row: selection, the partial index, the active turn, and a + * settled summary no fence changes (T3, T15a). */ +function expectStartupReadsTheRow(name: JournalSessionStateCase): void { + const status = readTestJournalSessionStatus(root, name)! + const { state } = loadTestJournal(root, name)! + expect(isUnsettledJournalSessionStatus(status)).toBe(CORPUS_UNSETTLED[name]) + expect(readUnsettledJournalSessionIds(openTestJournalHostDatabase(root).db)).toEqual( + CORPUS_UNSETTLED[name] ? [name] : [] + ) + // The active turn the facts pass finds is the by-sequence reader's. + expect(status.activeTurnId).toBe( + activeStructuredAgentSessionTurnIdBySequence(state.items.values()) + ) + if (CORPUS_UNSETTLED[name]) { + return + } + const snapshot = renderJournalState(state) + for (const fence of [undefined, 0, 3, 4, 100]) { + expect( + projectStructuredAgentSessionStatusState(snapshot.items, snapshot.submissions, fence).summary + ).toEqual(status.summary) + } +} + +describe('a failed append leaves the fold equal to a replay of the disk', () => { + it.each(JOURNAL_SESSION_STATE_CASES)('%s', async (name) => { + const journal = await open(name) + const violations = failEveryAppendOnce(journal, name) + await JOURNAL_SESSION_STATE_CORPUS[name](journal) + expect(violations).toEqual([]) + expectStartupReadsTheRow(name) + }) + + it('through every case on one chat, a tombstone and a recreate', async () => { + const journal = await open('mixed') + const violations = failEveryAppendOnce(journal, 'mixed') + for (const name of JOURNAL_SESSION_STATE_CASES) { + // Cases reuse ids; a revision rule refusing a stale one is a write path too. + await JOURNAL_SESSION_STATE_CORPUS[name](journal).catch(() => undefined) + } + const id = codexItem('t-x', 0) + await journal.appendItem(id, assistant('a'), THREAD) + await journal.appendTombstone(id, { fence: CORPUS_FENCE }) + await journal.appendItem(id, assistant('b'), THREAD) + expect(violations).toEqual([]) + }) + + it('keeps map order when the failed append tombstoned an item that is not the newest', async () => { + const journal = await open('order') + for (const ordinal of [0, 1]) { + await journal.appendItem(codexItem('t-1', ordinal), assistant('x'), THREAD) + } + const violations = failEveryAppendOnce(journal, 'order') + + await journal.appendTombstone(codexItem('t-1', 0), { fence: CORPUS_FENCE }) + expect(violations).toEqual([]) + }) + + it("keeps a person's Stop and the queue's Resume as a replay reads them", async () => { + const journal = await open('stop') + await journal.appendItem(codexItem('t-1', 0), assistant('x'), THREAD) + const violations = failEveryAppendOnce(journal, 'stop') + + await journal.appendStopEvent({ reason: 'user-stop', turnId: 't-1' }, CORPUS_FENCE) + await journal.appendQueueResume(CORPUS_FENCE) + await journal.appendStopEvent({ reason: 'user-stop' }, CORPUS_FENCE) + expect(violations).toEqual([]) + }) + + it('through every case on one chat, with a bookkeeping savepoint that writes and then fails', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + let written = 0 + vi.spyOn(JournalQueuedMessages.prototype, 'onRowInTransaction').mockImplementation((db) => { + // A complete row, so only the savepoint's rollback keeps it off the disk. + db.prepare( + `INSERT INTO journal_session_state (session_id, lifecycle, active_turn_id, handed_over_sends, + queued_sends, live_child_work, summary_json, last_activity_at, rules_version, epoch, tip_seq, + tip_ts) VALUES ('savepoint-write', 'running', NULL, 0, 0, 0, '{}', 0, 1, 'epoch-x', 1, 1)` + ).run() + written += 1 + throw new Error('bookkeeping failed') + }) + const journal = await open('mixed') + const violations = failEveryAppendOnce(journal, 'mixed') + for (const name of JOURNAL_SESSION_STATE_CASES) { + await JOURNAL_SESSION_STATE_CORPUS[name](journal).catch(() => undefined) + } + expect(violations).toEqual([]) + + expect(written).toBeGreaterThan(0) + const savepointWrite = openTestJournalHostDatabase(root) + .db.prepare("SELECT 1 FROM journal_session_state WHERE session_id = 'savepoint-write'") + .get() + expect(savepointWrite).toBeUndefined() + }) +}) + +describe('a failed append that evicted the oldest settlement id (R3W-1)', () => { + it('forgets the same ids a replay forgets, so a re-sent evicted settle is skipped live too', async () => { + const journal = await open('cap') + const note = (id: string) => ({ + kind: 'item' as const, + identity: { provider: 'orca' as const, clientMessageId: id }, + body: { kind: 'status' as const, text: id }, + turnScope: { kind: 'thread' as const } + }) + for (let index = 0; index < MAX_JOURNAL_APPLIED_SETTLEMENT_IDS; index += 1) { + await journal.appendLifecycleBatch({ + settlementId: `s-${index}`, + fence: CORPUS_FENCE, + mutations: [note(`n-${index}`)] + }) + } + const restore = failNextCommit() + await expect( + journal.appendLifecycleBatch({ + settlementId: 's-new', + fence: CORPUS_FENCE, + mutations: [note('n-new')] + }) + ).rejects.toThrow('COMMIT failed') + restore() + expectFoldIsTheDisk(journal, 'cap') + + // One more commit evicts the oldest, as a replay does; then a settle re-sent under that id. + await journal.appendLifecycleBatch({ + settlementId: 's-after', + fence: CORPUS_FENCE, + mutations: [note('n-after')] + }) + await journal.appendLifecycleBatch({ + settlementId: 's-1', + fence: CORPUS_FENCE, + mutations: [ + { + kind: 'item', + identity: codexItem('turn-1', 7), + body: { kind: 'tool-call', name: 'shell', input: { command: 'x' }, state: 'running' }, + turnScope: { kind: 'thread' } + } + ] + }) + + expectFoldIsTheDisk(journal, 'cap') + }, 120_000) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-fold-undo.ts b/src/main/native-chat/agent-session-journal/journal-fold-undo.ts new file mode 100644 index 000000000000..f6397b24a78d --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-fold-undo.ts @@ -0,0 +1,132 @@ +// One append's undo of the fold. An append folds its row inside its transaction, so the status +// written with it describes it; if the transaction fails, what the row touched is put back. +// Entries are replaced on change, never edited, so an entry a reader holds is never one the failed +// row changed. A replaced or added key goes back in place; a removed one cannot, since re-adding it +// puts it last and key order is part of the fold (eviction is oldest-first, ties go to map order), +// so a unit that removed an entry declines the undo and its caller re-reads the fold instead. + +import type { JournalReducerState } from './journal-reducer' + +type Undo = () => void +/** Null for a change no undo can put back in place: the removal of an entry. */ +type Recorder = (undo: Undo | null) => void + +/** A fold container that, while a unit is open, tells it each change it is about to make. */ +export class JournalFoldMap extends Map { + onChange: Recorder | null = null + + override set(key: K, value: V): this { + this.record(key) + return super.set(key, value) + } + + override delete(key: K): boolean { + if (this.onChange && super.has(key)) { + this.onChange(null) + } + return super.delete(key) + } + + private record(key: K): void { + if (!this.onChange) { + return + } + if (super.has(key)) { + const previous = super.get(key) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: `has` just answered true, so `get` returned the key's own value, `undefined` included when V allows it. + this.onChange(() => super.set(key, previous as V)) + } else { + this.onChange(() => super.delete(key)) + } + } +} + +export class JournalFoldSet extends Set { + onChange: Recorder | null = null + + override add(value: T): this { + if (this.onChange && !super.has(value)) { + this.onChange(() => super.delete(value)) + } + return super.add(value) + } + + override delete(value: T): boolean { + if (this.onChange && super.has(value)) { + this.onChange(null) + } + return super.delete(value) + } +} + +export type JournalFoldUndo = { + /** COMMIT landed: the fold as it stands is adopted. */ + commit: () => void + /** The transaction failed: every change since the unit began is put back, newest first. False, + * changing nothing, when the unit removed an entry; the caller then re-reads the fold. */ + rollback: () => boolean +} + +/** Null when the fold's containers cannot record their changes; the caller re-reads it instead. */ +export function beginJournalFoldUndo(state: JournalReducerState): JournalFoldUndo | null { + const containers: readonly object[] = [ + state.items, + state.itemFences, + state.tombstones, + state.submissions, + state.receipts, + state.aliases, + state.appliedSettlementIds + ] + const recording: { onChange: Recorder | null }[] = [] + for (const container of containers) { + if (!(container instanceof JournalFoldMap || container instanceof JournalFoldSet)) { + return null + } + recording.push(container) + } + const undos: Undo[] = [] + let restorable = true + const record: Recorder = (undo) => { + if (undo) { + undos.push(undo) + } else { + restorable = false + } + } + const scalars = { + epoch: state.epoch, + lastSequence: state.lastSequence, + lastActivityAt: state.lastActivityAt, + oldestSequence: state.oldestSequence, + highestFence: state.highestFence, + latestAcceptedTurnSequence: state.latestAcceptedTurnSequence + } + const derivedTurnScope = state.derivedTurnScope.clone() + // Edited in place by the fold; its fields are replaced, never mutated, so a shallow copy restores it. + const queuePauseMarks = { ...state.queuePauseMarks } + for (const container of recording) { + container.onChange = record + } + const detach = () => { + for (const container of recording) { + container.onChange = null + } + } + return { + commit: detach, + rollback: () => { + detach() + if (!restorable) { + return false + } + for (const undo of undos.toReversed()) { + undo() + } + Object.assign(state, scalars) + state.derivedTurnScope = derivedTurnScope + Object.assign(state.queuePauseMarks, queuePauseMarks) + return true + } + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts b/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts index ad151d10ec72..aafa32b183e9 100644 --- a/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts +++ b/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts @@ -7,6 +7,7 @@ import { NO_LEGACY_JOURNAL_RECORDS } from './journal-database' import { JournalHostDatabase } from './journal-host-database' import { replayJournal, type JournalLoad } from './journal-open' import { serializeJournalRow, type JournalRow } from './journal-row-schema' +import { readJournalSessionStatuses, type JournalSessionStatus } from './journal-session-state' import { iterateJournalEpochRows, publishJournalSessionEpoch, @@ -80,6 +81,17 @@ export function createTrackedJournalOpener(): TrackedJournalOpener { } } +/** The chat's stored status, or null when it has none. */ +export function readTestJournalSessionStatus( + stateDirectory: string, + sessionId: string +): JournalSessionStatus | null { + const [stored] = readJournalSessionStatuses(openTestJournalHostDatabase(stateDirectory).db, [ + sessionId + ]) + return stored?.status ?? null +} + /** What a fresh open of the chat would replay, read from the test state directory's database. */ /** What an open of a chat a newer Orca saved is refused with. */ export const SAVED_BY_NEWER_ORCA = { diff --git a/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts new file mode 100644 index 000000000000..0e67d8baebe5 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts @@ -0,0 +1,144 @@ +// What an open owes a chat whose earlier host process is gone, one rule per kind of entity. +// +// The settlement plan (`structured-agent-session-open-settlement.ts`) and each chat's stored status +// (`journal-session-state.ts`) both read these same rules, so the status shows exactly the work a +// settle would revise. Each rule revises its entity out of the state that selected it, so once the +// plan commits the stored status reads settled. + +import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import type { + AgentJournalItemBody, + AgentJournalItemIdentity, + AgentJournalRenderItem, + AgentJournalSubmission, + AgentJournalTurnLifecycle +} from '../../../shared/agent-session-journal-types' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import type { AgentJournalRunningCallEnd } from '../../../shared/agent-journal-tool-call-lifecycle' +import { + endedUnseenMessageBody, + requiresTerminalSettlement, + terminalAgentJournalBody +} from './journal-terminal-settlement' +import type { JournalReducerState } from './journal-reducer' +import { staleSubagentRosterRevision } from './journal-subagent-liveness' + +/** The key a settlement revises an item under. One that will not parse cannot be revised, so the + * item owes nothing: a fresh identity would duplicate it rather than settle it. */ +export function openSettlementItemIdentity( + item: Pick +): AgentJournalItemIdentity | null { + return parseAgentJournalItemKey(item.itemId) +} + +/** A running tool call ends as `end` says (its turn's row, or the death evidence), and a pending + * approval or question is cancelled. An open reasoning row is ended too, but is not unfinished + * work: its running turn already says so. */ +export function openSettlementTerminalBody( + item: Pick, + end: AgentJournalRunningCallEnd +): AgentJournalItemBody | null { + return endedUnseenMessageBody(item.body) ?? terminalAgentJournalBody(item.body, end) +} + +/** Every turn record still `running` gets the verdict for its own writer. */ +export function isRunningJournalTurn(item: Pick): boolean { + return readAgentJournalTurn(item.body)?.state === 'running' +} + +/** A handed-over send nothing answered: it becomes a recovered `unknown`. A queued one is not. */ +export function owesRecoveredDispatch(submission: AgentJournalSubmission): boolean { + return ( + !isQueuedAgentJournalSubmission(submission) && + (submission.dispatchState === 'pending' || + (submission.dispatchState === 'unknown' && submission.recovered !== true)) + ) +} + +/** What a chat's settle would have to revise, counted from the same rules the settle applies, and + * the turn the chat is running. */ +export type JournalSettlementFacts = { + /** A turn record or tool call still `running`, under a key a settle can revise. */ + runningWork: boolean + /** A pending approval or question, under a key a settle can revise. */ + pendingPrompts: boolean + /** Handed-over sends nothing answered (`owesRecoveredDispatch`). */ + handedOverSends: number + /** Sends accepted and never handed over. */ + queuedSends: number + /** A working subagent roster or live background-task row. */ + liveChildWork: boolean + /** The newest turn record by sequence, when it is running (`activeStructuredAgentSessionTurnIdBySequence`). */ + activeTurnId: string | null +} + +type ItemFacts = { + running: boolean + prompt: boolean + roster: boolean + turn: AgentJournalTurnLifecycle | null +} + +// Per item object: a write replaces the entry it changes and never edits one, so an object's facts +// never change. Each append still visits every item, but computes facts only for new objects. +const ITEM_FACTS = new WeakMap() + +function itemFacts(item: AgentJournalRenderItem): ItemFacts { + let facts = ITEM_FACTS.get(item) + if (!facts) { + const running = isRunningJournalTurn(item) || isRunningToolCall(item) + const prompt = !running && requiresTerminalSettlement(item.body) + const settlable = (running || prompt) && openSettlementItemIdentity(item) !== null + facts = { + running: settlable && running, + prompt: settlable && prompt, + roster: staleSubagentRosterRevision(item) !== null, + turn: readAgentJournalTurn(item.body) + } + ITEM_FACTS.set(item, facts) + } + return facts +} + +export function journalSettlementFacts( + fold: Pick +): JournalSettlementFacts { + let handedOverSends = 0 + let queuedSends = 0 + for (const submission of fold.submissions.values()) { + if (isQueuedAgentJournalSubmission(submission)) { + queuedSends += 1 + } else if (owesRecoveredDispatch(submission)) { + handedOverSends += 1 + } + } + let runningWork = false + let pendingPrompts = false + let liveChildWork = false + let newestSequence = 0 + let newest: AgentJournalTurnLifecycle | null = null + for (const item of fold.items.values()) { + const facts = itemFacts(item) + runningWork ||= facts.running + pendingPrompts ||= facts.prompt + liveChildWork ||= facts.roster + // Ties go to the later item, as the by-sequence reader decides them. + if (facts.turn && item.sequence >= newestSequence) { + newestSequence = item.sequence + newest = facts.turn + } + } + return { + runningWork, + pendingPrompts, + handedOverSends, + queuedSends, + liveChildWork, + activeTurnId: newest?.state === 'running' ? newest.turnId : null + } +} + +function isRunningToolCall(item: Pick): boolean { + return item.body.kind === 'tool-call' && item.body.state === 'running' +} diff --git a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts index 9e9b1f6c57cd..7ec9f7bdaaa6 100644 --- a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts +++ b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts @@ -2,6 +2,7 @@ import type { AgentJournalDispatchRejection } from '../../../shared/agent-sessio import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { DISPATCH_DOUBT_HOST_RESTARTED } from './journal-dispatch-doubt-reasons' +import { owesRecoveredDispatch } from './journal-open-settlement-plan' import type { JournalReducerState } from './journal-reducer' import { journalDispatchRowBuilder } from './journal-row-builders' import type { JournalRow } from './journal-row-schema' @@ -15,14 +16,7 @@ export async function markJournalPendingSubmissionsUnknown( fence: number, reason: string = DISPATCH_DOUBT_HOST_RESTARTED ): Promise { - const unresolved = journal - .submissions() - .filter( - (entry) => - !isQueuedAgentJournalSubmission(entry) && - (entry.dispatchState === 'pending' || - (entry.dispatchState === 'unknown' && entry.recovered !== true)) - ) + const unresolved = journal.submissions().filter(owesRecoveredDispatch) for (const entry of unresolved) { // An earlier reason already names a sharper fact than "the host restarted". const resolvedReason = @@ -46,14 +40,7 @@ export async function rejectJournalPendingSubmissions( fence: number, rejection: AgentJournalDispatchRejection ): Promise { - const unwritten = journal - .submissions() - .filter( - (entry) => - !isQueuedAgentJournalSubmission(entry) && - (entry.dispatchState === 'pending' || - (entry.dispatchState === 'unknown' && entry.recovered !== true)) - ) + const unwritten = journal.submissions().filter(owesRecoveredDispatch) for (const entry of unwritten) { await journal.resolveDispatch({ clientMessageId: entry.clientMessageId, diff --git a/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts b/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts index 53397f87044b..8a08660467ea 100644 --- a/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts @@ -27,10 +27,12 @@ import { loadTestJournal, openTestJournalHostDatabase, readTestJournalRows, + readTestJournalSessionStatus, SAVED_BY_NEWER_ORCA } from './journal-host-database-test-support' import { journalDirectoryFor, legacyJournalDatabaseFile } from './journal-paths' import { importPerSessionJournal, previewPerSessionJournal } from './journal-per-session-import' +import { deriveJournalSessionStatus, JOURNAL_SESSION_STATUS_RULES } from './journal-session-state' import { readJournalSessionEpoch, type JournalStoredRow } from './journal-row-table' import { createStructuredAgentSessionLogger } from '../agent-session-wire/structured-agent-session-logger' import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' @@ -214,6 +216,38 @@ describe('importing a per-chat journal', () => { expect(await leftovers()).toEqual([]) }) + // T10: the copy's publish writes the chat's status for the rows it publishes, in the same + // transaction, replacing whatever described the rows it replaced. + it('drops the status of the rows the copy replaced, and the open after it writes the status again', async () => { + const { epoch, rows } = await historyRows() + await writeLegacyJournal(epoch, rows) + const { db } = openTestJournalHostDatabase(root) + // Named at the tip the copy publishes, so only the copy's own drop can retire it. + const tip = rows.at(-1)! + db.prepare( + `INSERT INTO journal_session_state (session_id, lifecycle, active_turn_id, handed_over_sends, + queued_sends, live_child_work, summary_json, last_activity_at, rules_version, epoch, tip_seq, + tip_ts) + VALUES (?, 'running', NULL, 3, 0, 0, '{"status":"working","latestPrompt":"stale"}', 1, ?, ?, ?, + ?)` + ).run(IDENTITY.sessionId, JOURNAL_SESSION_STATUS_RULES, epoch, tip.seq, tip.ts) + + await importPerSessionJournal({ + database: openTestJournalHostDatabase(root), + identity: IDENTITY, + legacyDirectory: legacyDir() + }) + expect(readTestJournalSessionStatus(root, IDENTITY.sessionId)).toBeNull() + + // As the chat's open does after its replay. + ;(await openChat()).sessionStatus.backfill() + + const loaded = loadTestJournal(root, IDENTITY.sessionId)! + expect(readTestJournalSessionStatus(root, IDENTITY.sessionId)).toEqual( + deriveJournalSessionStatus(loaded.state) + ) + }) + // T-B3: the upgrade restart is the restart that produced the offers. A new epoch or renumbered // rows would silently withdraw every "resume after update" offer. it('keeps a restart offer taken before the upgrade', async () => { diff --git a/src/main/native-chat/agent-session-journal/journal-per-session-import.ts b/src/main/native-chat/agent-session-journal/journal-per-session-import.ts index 66f1f139ea32..da255d1a2375 100644 --- a/src/main/native-chat/agent-session-journal/journal-per-session-import.ts +++ b/src/main/native-chat/agent-session-journal/journal-per-session-import.ts @@ -19,6 +19,7 @@ // an empty chat founded here would take a new epoch the next open's import could not reconcile. import { createHash } from 'node:crypto' +import { deleteJournalSessionStatus } from './journal-session-state' import { existsSync } from 'node:fs' import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' @@ -226,6 +227,9 @@ async function copyLegacyJournal( input.database.transaction((db) => { publishJournalSessionEpoch(db, input.identity, epoch) writePerSessionImportMarker(db, sessionId, legacy) + // It described the rows this copy replaced. The replay that follows the copy writes it again + // (the chat's open), without a second whole-chat replay inside this transaction. + deleteJournalSessionStatus(db, sessionId) }) } diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index 264933b7fb33..d366151a7f3a 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -24,6 +24,7 @@ import { } from '../../../shared/agent-session-journal-item-key' import { agentSessionSendBodyFingerprint } from '../../../shared/structured-agent-session-send-mutation' import { JournalDerivedTurnScope } from './journal-derived-turn-scope' +import { JournalFoldMap, JournalFoldSet } from './journal-fold-undo' import { removeJournalItem, statedOrDerivedTurnScope, upsertJournalItem } from './journal-item-fold' import { journalItemRevisionIsStale } from './journal-item-revision' import { isJournalStopOrResumeRow, type JournalRow } from './journal-row-schema' @@ -75,13 +76,14 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou lastActivityAt: 0, oldestSequence: 1, highestFence: 0, - items: new Map(), - itemFences: new Map(), - tombstones: new Map(), - submissions: new Map(), - receipts: new Map(), - aliases: new Map(), - appliedSettlementIds: new Set(), + // Recording containers: an append that fails puts back exactly what it changed. + items: new JournalFoldMap(), + itemFences: new JournalFoldMap(), + tombstones: new JournalFoldMap(), + submissions: new JournalFoldMap(), + receipts: new JournalFoldMap(), + aliases: new JournalFoldMap(), + appliedSettlementIds: new JournalFoldSet(), derivedTurnScope: new JournalDerivedTurnScope(), latestAcceptedTurnSequence: 0, queuePauseMarks: createJournalQueuePauseMarks() diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts index 3671ce46cfcf..e2e109ab844f 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts @@ -70,10 +70,13 @@ describe('journal row writer', () => { readOnly: () => readOnly, highestFence: () => 0, nextSequence: () => sequence, - commit: (committed) => { - committedRows.push(committed) - sequence = committed.seq + 1 - } + apply: () => undefined, + writeStatus: () => undefined, + committed: (rows) => { + committedRows.push(...rows) + sequence += rows.length + }, + recoverFold: () => undefined }) return { writer, committedRows } } diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.ts index 60db6e0cbd1e..260dc9ca3342 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.ts @@ -27,7 +27,15 @@ export type JournalRowWriterDeps = { readOnly: () => boolean highestFence: () => number nextSequence: () => number - commit: (row: JournalRow) => void + /** Folds the append's rows inside its transaction, so the status written with them describes them. */ + apply: (rows: readonly JournalRow[]) => void + /** The chat's status from the fold that now holds the rows. A throw fails the append. */ + writeStatus: (db: Database.Database, rows: readonly JournalRow[]) => void + /** After COMMIT: observers learn of the rows only once they are durable. */ + committed: (rows: readonly JournalRow[]) => void + /** A transaction that failed after `apply`: the fold is ahead of the disk, so it is folded + * again from what committed. */ + recoverFold: () => void /** Standing hook run for EVERY appended row — the queued-draft returned * transition rides here so no rejection path can bypass it. Bookkeeping: it * runs in its own savepoint, so its failure is reported and never vetoes the row. */ @@ -50,23 +58,7 @@ export class JournalRowWriter { assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) const row = build(this.deps.nextSequence(), this.deps.now()) assertJournalFence(row.fence, this.deps.highestFence()) - try { - // One INSERT: the chat's epoch pointer moves only when the epoch does. - this.deps.database().transaction((db) => { - insertJournalRow(db, this.deps.sessionId, row) - hook?.(db, row) - receipt?.write(db) - this.runBookkeeping(db, row) - }) - } catch (error) { - this.deps.rolledBack?.() - throw error - } - // COMMIT landed, so the row is durable: adopt it before anything that can - // fail. Rejecting here instead would leave the next append reusing a - // sequence the table already holds. The ledger first: it cannot throw, the fold can. - receipt?.committed() - this.deps.commit(row) + this.commitRows([row], hook, receipt) return row }) } @@ -91,20 +83,7 @@ export class JournalRowWriter { for (const row of rows) { assertJournalFence(row.fence, this.deps.highestFence()) } - try { - this.deps.database().transaction((db) => { - for (const row of rows) { - insertJournalRow(db, this.deps.sessionId, row) - this.runBookkeeping(db, row) - } - }) - } catch (error) { - this.deps.rolledBack?.() - throw error - } - for (const row of rows) { - this.deps.commit(row) - } + this.commitRows(rows) return rows } @@ -121,6 +100,39 @@ export class JournalRowWriter { })) } + /** One transaction: every row, then the receipt, then the fold and the status it gives. */ + private commitRows( + rows: readonly JournalRow[], + hook?: JournalRowTransactionHook, + receipt?: JournalOperationReceipt + ): void { + let applied = false + try { + // One INSERT per row: the chat's epoch pointer moves only when the epoch does. The hook and + // the bookkeeping read the fold before the rows; the status reads it after all of them. + this.deps.database().transaction((db) => { + for (const row of rows) { + insertJournalRow(db, this.deps.sessionId, row) + hook?.(db, row) + this.runBookkeeping(db, row) + } + receipt?.write(db) + applied = true + this.deps.apply(rows) + this.deps.writeStatus(db, rows) + }) + } catch (error) { + this.deps.rolledBack?.() + if (applied) { + this.deps.recoverFold() + } + throw error + } + // The ledger first: it cannot throw, observers can. + receipt?.committed() + this.deps.committed(rows) + } + private runBookkeeping(db: Database.Database, row: JournalRow): void { const hook = this.deps.inTransaction if (!hook) { diff --git a/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts new file mode 100644 index 000000000000..f7f260024390 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts @@ -0,0 +1,309 @@ +// Chats in every state a settlement plan can find, written through a real journal store, for the +// tests of the stored status and the plan that must agree about them. + +import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' +import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' +import { agentSessionHostStatusBody } from '../../../shared/agent-session-host-status-rows' +import { structuredAgentSessionPayloadFingerprint } from '../../../shared/structured-agent-session-mutation' +import type { AgentSessionJournal } from './journal-store' +import { + assistantMessage, + codexItem, + CORPUS_FENCE, + item, + roster, + runningTool, + send, + settledTurn, + startFailure, + userMessage +} from './journal-session-state-test-writes' + +export { CORPUS_FENCE } from './journal-session-state-test-writes' + +/** Each case, written onto a freshly opened (empty) journal. */ +export const JOURNAL_SESSION_STATE_CORPUS = { + empty: async () => undefined, + settled: (journal: AgentSessionJournal) => settledTurn(journal, 'turn-1'), + 'older turn running beside a newer one': async (journal: AgentSessionJournal) => { + await item(journal, codexItem('turn-0', 0), { + kind: 'turn', + turnId: 'turn-0', + state: 'running', + startedAt: 5 + }) + await settledTurn(journal, 'turn-1') + }, + 'working subagent roster': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item(journal, { provider: 'orca', clientMessageId: 'roster-1' }, roster('working')) + }, + 'live background task': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item( + journal, + { provider: 'orca', clientMessageId: 'claude-background-task:task-1' }, + { + kind: 'message', + role: 'system', + blocks: [ + { + type: 'background-task', + taskId: 'task-1', + kind: 'command', + label: 'sleep 20', + state: 'working', + startedAt: 10 + } + ] + } + ) + }, + 'pending prompt': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item(journal, codexItem('turn-1', 3), { + kind: 'approval', + title: 'Approve?', + detail: null, + options: [], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + }) + }, + 'running tool': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item(journal, codexItem('turn-1', 4), { + kind: 'tool-call', + name: 'shell', + input: { command: 'ls' }, + state: 'running' + }) + }, + 'pending send': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-pending', 'never answered') + }, + 'unknown send': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-unknown', 'ambiguous') + await journal.resolveDispatch({ + clientMessageId: 'send-unknown', + state: 'unknown', + reason: 'adapter_timeout', + fence: CORPUS_FENCE + }) + }, + 'queued leftover': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-queued', 'waiting its turn', true) + }, + 'unverifiable turn': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item(journal, codexItem('turn-2', 0), { + kind: 'turn', + turnId: 'turn-2', + state: 'unverifiable', + startedAt: 30 + }) + }, + 'settled roster': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item(journal, { provider: 'orca', clientMessageId: 'roster-1' }, roster('completed')) + }, + 'provider echo claiming a send': async (journal: AgentSessionJournal) => { + const body = userMessage('hello') + await journal.appendSubmission({ + clientMessageId: 'send-echo', + payloadFingerprint: structuredAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: journal.snapshot().sessionId, + fields: { body } + }), + body, + fence: CORPUS_FENCE + }) + await item(journal, codexItem('turn-2', 0), { + kind: 'turn', + turnId: 'turn-2', + state: 'running', + startedAt: 10 + }) + // The provider's copy of the user's own message, matched to the send by its fingerprint. + await item(journal, codexItem('turn-2', 1), userMessage('hello')) + await item(journal, codexItem('turn-2', 2), assistantMessage('working on it')) + }, + 'newest reply tombstoned': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item(journal, codexItem('turn-1', 3), assistantMessage('a later reply')) + await journal.appendTombstone(codexItem('turn-1', 3), { fence: CORPUS_FENCE }) + }, + 'queued send handed over': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-queued', 'waiting its turn', true) + await journal.resolveDispatch({ + clientMessageId: 'send-queued', + state: 'pending', + turnScope: AGENT_JOURNAL_THREAD_SCOPE, + fence: CORPUS_FENCE + }) + }, + 'refused send': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-refused', 'nope') + await journal.resolveDispatch({ + clientMessageId: 'send-refused', + state: 'rejected', + ...agentSessionFailureWords(agentSessionFailureFact('hostRestarted'), { + surface: 'rejection' + }), + fence: CORPUS_FENCE + }) + }, + // The rejected send moves to where it was rejected, after the turn: it is the latest request. + 'send rejected after a later turn': async (journal: AgentSessionJournal) => { + await send(journal, 'send-early', 'the early one') + await settledTurn(journal, 'turn-1') + await journal.resolveDispatch({ + clientMessageId: 'send-early', + state: 'rejected', + ...agentSessionFailureWords(agentSessionFailureFact('hostRestarted'), { + surface: 'rejection' + }), + fence: CORPUS_FENCE + }) + }, + 'running turn on a legacy status row': async (journal: AgentSessionJournal) => { + await item(journal, codexItem('turn-9', 0), { + kind: 'status', + text: 'Codex is working', + turnLifecycle: { turnId: 'turn-9', state: 'running', startedAt: 10 } + }) + }, + 'running work settled by a batch': async (journal: AgentSessionJournal) => { + await item(journal, codexItem('turn-1', 0), { + kind: 'turn', + turnId: 'turn-1', + state: 'running', + startedAt: 10 + }) + await item(journal, codexItem('turn-1', 4), runningTool) + await journal.appendLifecycleBatch({ + settlementId: 'settle-1', + fence: CORPUS_FENCE, + recovered: true, + mutations: [ + { + kind: 'item', + identity: codexItem('turn-1', 4), + body: { ...runningTool, state: 'failed' }, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }, + { + kind: 'item', + identity: codexItem('turn-1', 0), + body: { + kind: 'turn', + turnId: 'turn-1', + state: 'interrupted', + startedAt: 10, + completedAt: 20 + }, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + } + ] + }) + }, + 'unknown send recovered': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-recovered', 'lost in a crash') + await journal.markPendingSubmissionsUnknown(CORPUS_FENCE) + }, + // A person's Stop found the turn running, then Orca died before the provider ended it. + "running turn a person's Stop found": async (journal: AgentSessionJournal) => { + await item(journal, codexItem('turn-2', 0), { + kind: 'turn', + turnId: 'turn-2', + state: 'running', + startedAt: 10 + }) + await journal.appendStopEvent({ reason: 'user-stop', turnId: 'turn-2' }, CORPUS_FENCE) + }, + // A failed start rejects every send still queued behind it, in the batch that records it. + 'failed start rejects queued': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-first', 'first', true) + await send(journal, 'send-second', 'second', true) + await startFailure(journal) + }, + // An older build's repair left its notice newest; nothing here owes a rebuild, so the roster counts. + 'working roster under a former repair notice': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item(journal, { provider: 'orca', clientMessageId: 'roster-1' }, roster('working')) + await item( + journal, + { provider: 'orca', clientMessageId: 'journal-malformed-lines' }, + agentSessionHostStatusBody('history-repaired') + ) + } +} satisfies Record Promise> + +export type JournalSessionStateCase = keyof typeof JOURNAL_SESSION_STATE_CORPUS + +/** Whether each case's stored status shows work a gone process left: what startup selects. */ +export const CORPUS_UNSETTLED: Record = { + empty: false, + settled: false, + 'older turn running beside a newer one': true, + 'working subagent roster': true, + 'live background task': true, + 'pending prompt': true, + 'running tool': true, + 'pending send': true, + 'unknown send': true, + 'queued leftover': true, + // Its verdict was decided when it settled; startup never revises it. + 'unverifiable turn': false, + 'settled roster': false, + 'provider echo claiming a send': true, + 'newest reply tombstoned': false, + 'queued send handed over': true, + 'refused send': false, + 'send rejected after a later turn': false, + 'running turn on a legacy status row': true, + 'running work settled by a batch': false, + 'unknown send recovered': false, + "running turn a person's Stop found": true, + 'failed start rejects queued': false, + 'working roster under a former repair notice': true +} + +export const JOURNAL_SESSION_STATE_CASES = Object.keys(JOURNAL_SESSION_STATE_CORPUS).filter( + (name): name is JournalSessionStateCase => name in JOURNAL_SESSION_STATE_CORPUS +) + +/** Death evidence as a record can carry it: none, an older build's (no owner), naming the + * corpus's writer, and naming another owner. */ +export const CORPUS_DEATH_EVIDENCE: Record = { + none: null, + 'older build, no owner': { kind: 'exit-observed', detail: 'exit', observedAt: 50 }, + 'names the writer': { + kind: 'pid-absent', + detail: 'gone', + observedAt: 60, + ownerFence: CORPUS_FENCE + }, + 'names another owner': { + kind: 'exit-observed', + detail: 'exit', + observedAt: 70, + ownerFence: CORPUS_FENCE + 1 + }, + // Observed after every journal clock in the corpus: after a Stop event, too. + 'names the writer, after a Stop': { + kind: 'pid-absent', + detail: 'gone', + observedAt: 10_000_000, + ownerFence: CORPUS_FENCE + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts b/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts new file mode 100644 index 000000000000..2df5a33600ea --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts @@ -0,0 +1,118 @@ +// How the chat-status corpus writes each chat, through a real journal store. + +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemBody, + type AgentJournalItemIdentity, + type AgentJournalMessageItem +} from '../../../shared/agent-session-journal-types' +import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' +import type { AgentSessionJournal } from './journal-store' + +/** The fence every case writes its content under; an `unverifiable` turn's writer. */ +export const CORPUS_FENCE = 3 + +const THREAD = 'thread-corpus' + +export function codexItem(turnId: string, ordinal: number): AgentJournalItemIdentity { + return { provider: 'codex', threadId: THREAD, turnId, ordinal } +} + +export async function item( + journal: AgentSessionJournal, + identity: AgentJournalItemIdentity, + body: AgentJournalItemBody +): Promise { + await journal.appendItem(identity, body, { + fence: CORPUS_FENCE, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) +} + +export async function send( + journal: AgentSessionJournal, + clientMessageId: string, + text: string, + handoverRecorded?: true +): Promise { + await journal.appendSubmission({ + clientMessageId, + payloadFingerprint: `fp-${clientMessageId}`, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, + fence: CORPUS_FENCE, + ...(handoverRecorded ? { handoverRecorded } : {}) + }) +} + +export async function settledTurn(journal: AgentSessionJournal, turnId: string): Promise { + await send(journal, `send-${turnId}`, `asked ${turnId}`) + await journal.resolveDispatch({ + clientMessageId: `send-${turnId}`, + state: 'accepted', + providerIdentity: codexItem(turnId, 1), + fence: CORPUS_FENCE + }) + await item(journal, codexItem(turnId, 0), { + kind: 'turn', + turnId, + state: 'completed', + outcome: 'success', + startedAt: 10, + completedAt: 20 + }) + await item(journal, codexItem(turnId, 2), { + kind: 'message', + role: 'assistant', + blocks: [{ type: 'text', text: `answered ${turnId}` }] + }) +} + +export function userMessage(text: string): AgentJournalMessageItem { + return { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] } +} + +export function assistantMessage(text: string): AgentJournalItemBody { + return { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text }] } +} + +export const runningTool: Extract = { + kind: 'tool-call', + name: 'shell', + input: { command: 'ls' }, + state: 'running' +} + +export function roster(state: 'working' | 'completed'): AgentJournalItemBody { + return { + kind: 'message', + role: 'system', + blocks: [ + { + type: 'subagent-group', + groupId: 'group-1', + agents: [{ id: 'child-1', label: 'reads', state, startedAt: 10 }] + } + ] + } +} + +/** A failed agent start: its notice, and every send still queued rejected, in one batch. */ +export async function startFailure(journal: AgentSessionJournal): Promise { + await journal.appendLifecycleBatch({ + settlementId: 'start-failure:gen-1', + fence: CORPUS_FENCE, + recovered: true, + mutations: [ + { + kind: 'item', + identity: { provider: 'orca', clientMessageId: 'start-failure:gen-1' }, + body: { kind: 'status', tone: 'error', text: 'Codex did not start.' }, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + } + ], + rejectsQueued: agentSessionFailureWords(agentSessionFailureFact('providerStartFailed'), { + surface: 'rejection' + }) + }) +} diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts new file mode 100644 index 000000000000..afd2d0ad89d0 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts @@ -0,0 +1,349 @@ +// Each chat's stored status: written in the same transaction as every journal write, equal to what a +// fresh replay derives, for every chat state; a failed write leaves the fold equal to the disk. + +import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + AGENT_SESSION_JOURNAL_SCHEMA_VERSION, + type AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import { journalDatabasePath } from './journal-host-database' +import Database from '../../sqlite/sync-database' +import { + createTrackedJournalOpener, + liveTestJournalRows, + loadTestJournal, + openTestJournalHostDatabase, + readTestJournalSessionStatus, + updateTestJournalRowJson +} from './journal-host-database-test-support' +import * as JournalFoldUndo from './journal-fold-undo' +import * as JournalOpen from './journal-open' +import { renderJournalState } from './journal-reducer' +import { + deriveJournalSessionStatus, + readUnsettledJournalSessionIds, + type JournalSessionStatus +} from './journal-session-state' +import { + CORPUS_FENCE, + JOURNAL_SESSION_STATE_CORPUS, + type JournalSessionStateCase +} from './journal-session-state-test-corpus' +import type { AgentSessionJournal } from './journal-store' + +vi.mock('./journal-open', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, replayJournal: vi.fn(actual.replayJournal) } +}) +vi.mock('./journal-fold-undo', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, beginJournalFoldUndo: vi.fn(actual.beginJournalFoldUndo) } +}) + +const journals = createTrackedJournalOpener() +let root: string +let clock = 1_000 + +function identity(sessionId: string): AgentSessionJournalIdentity { + return { + sessionId, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: codexProviderHandle(`thread-${sessionId}`) + } +} + +function open(sessionId: string): Promise { + return journals.open({ + identity: identity(sessionId), + stateDirectory: root, + now: () => (clock += 1), + mintEpoch: () => `epoch-${sessionId}-${clock}`, + // As production passes the record's fence. + currentFence: () => CORPUS_FENCE + }) +} + +function db() { + return openTestJournalHostDatabase(root).db +} + +/** What a fresh open of the chat would derive, read back from disk. */ +function freshDerivation(sessionId: string): JournalSessionStatus { + const loaded = loadTestJournal(root, sessionId) + if (!loaded) { + throw new Error(`no journal for ${sessionId}`) + } + return deriveJournalSessionStatus(loaded.state, { currentFence: CORPUS_FENCE }) +} + +const stored = (sessionId: string) => readTestJournalSessionStatus(root, sessionId) + +async function write(name: JournalSessionStateCase): Promise { + const journal = await open(name) + await JOURNAL_SESSION_STATE_CORPUS[name](journal) + return journal +} + +function note(journal: AgentSessionJournal, id: string) { + return journal.appendItem( + { provider: 'orca', clientMessageId: id }, + { kind: 'status', text: id }, + { fence: 3, turnScope: { kind: 'thread' } } + ) +} + +function reply(journal: AgentSessionJournal, text: string) { + return journal.appendItem( + { provider: 'orca', clientMessageId: 'reply' }, + { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text }] }, + { fence: 3, turnScope: { kind: 'thread' } } + ) +} + +/** Fails the next COMMIT the connection runs, once. */ +function failNextCommit(): void { + const connection = db() + const exec = connection.exec.bind(connection) + let failing = true + vi.spyOn(connection, 'exec').mockImplementation((sql: string) => { + if (failing && sql === 'COMMIT') { + failing = false + throw new Error('COMMIT failed: disk I/O error') + } + return exec(sql) + }) +} + +function refuseStatusWrites(): void { + db().exec(`CREATE TEMP TRIGGER fail_status_write BEFORE UPDATE ON main.journal_session_state + BEGIN SELECT RAISE(ABORT, 'status write refused'); END`) +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-journal-session-state-')) + clock = 1_000 +}) + +afterEach(async () => { + vi.restoreAllMocks() + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +describe('observers hear of a write only once it is committed (T13)', () => { + it('tells observers after COMMIT, when another connection already reads the row and its status', async () => { + const journal = await open('committed') + const seen: { inTransaction: boolean; rows: unknown; status: unknown }[] = [] + journal.observeCommits(() => { + const other = new Database(journalDatabasePath(root), { readonly: true }) + try { + seen.push({ + inTransaction: db().isTransaction, + rows: other + .prepare('SELECT COUNT(*) AS n FROM journal_rows WHERE session_id = ?') + .get('committed'), + status: other + .prepare('SELECT lifecycle FROM journal_session_state WHERE session_id = ?') + .get('committed') + }) + } finally { + other.close() + } + }) + + await note(journal, 'note-1') + + expect(seen).toEqual([{ inTransaction: false, rows: { n: 2 }, status: { lifecycle: 'idle' } }]) + }) +}) + +describe('an open that fails on its rows drops the stored status (T10)', () => { + // Each rewrites a row in place, so the stored row still names the chat's tip and is selected. + it.each([ + ['damage', (rowJson: string) => `${rowJson.slice(0, 8)}`], + [ + "a newer build's row", + (rowJson: string) => + JSON.stringify({ ...JSON.parse(rowJson), v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION + 1 }) + ] + ] as const)('%s', async (_case, rewrite) => { + const name = 'running tool' + const journal = await write(name) + const tip = journal.cursor() + await journal.close() + expect(readUnsettledJournalSessionIds(db())).toContain(name) + const row = liveTestJournalRows(db(), name).find((stored) => stored.seq === 2)! + updateTestJournalRowJson(db(), name, 2, rewrite(row.rowJson)) + expect(readUnsettledJournalSessionIds(db())).toContain(name) + + await expect(open(name)).rejects.toThrow() + + // Nothing selects it again, and no stale status is seeded; every row stays. + expect(stored(name)).toBeNull() + expect(readUnsettledJournalSessionIds(db())).not.toContain(name) + expect(liveTestJournalRows(db(), name)).toHaveLength(tip.sequence) + }) +}) + +describe('a failed write leaves the fold equal to the disk (T1)', () => { + it('puts back what a failed COMMIT folded, stores nothing, and the next append takes the sequence', async () => { + const journal = await write('settled') + const tip = journal.cursor() + const before = stored('settled') + failNextCommit() + vi.mocked(JournalOpen.replayJournal).mockClear() + + await expect(reply(journal, 'lost')).rejects.toThrow('COMMIT failed') + + // Undone in memory: the chat is not read from disk again. + expect(JournalOpen.replayJournal).not.toHaveBeenCalled() + expect(journal.cursor()).toEqual(tip) + expect(journal.snapshot()).toEqual(renderJournalState(loadTestJournal(root, 'settled')!.state)) + expect(stored('settled')).toEqual(before) + await expect(reply(journal, 'kept')).resolves.toMatchObject({ + cursor: { sequence: tip.sequence + 1 } + }) + // Not the projection of the reply that rolled back, which held the same sequence. + expect(stored('settled')).toEqual(freshDerivation('settled')) + expect(stored('settled')?.summary.lastAssistantMessage).toBe('kept') + }) + + it('fails the append when its status write fails: the row and its status land together or not at all', async () => { + const journal = await write('settled') + const tip = journal.cursor() + refuseStatusWrites() + + await expect(note(journal, 'refused')).rejects.toThrow('status write refused') + + expect(loadTestJournal(root, 'settled')?.state.lastSequence).toBe(tip.sequence) + expect(journal.cursor()).toEqual(tip) + expect(journal.snapshot()).toEqual(renderJournalState(loadTestJournal(root, 'settled')!.state)) + db().exec('DROP TRIGGER fail_status_write') + await expect(note(journal, 'kept')).resolves.toMatchObject({ + cursor: { sequence: tip.sequence + 1 } + }) + }) + + it('never shows a held submission the change that rolled back', async () => { + const journal = await open('held') + await journal.appendSubmission({ + clientMessageId: 'send-1', + payloadFingerprint: 'fp', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }, + fence: 3 + }) + const held = journal.submission('send-1')! + expect(held.dispatchState).toBe('pending') + failNextCommit() + + await expect( + journal.resolveDispatch({ + clientMessageId: 'send-1', + state: 'accepted', + providerIdentity: { + provider: 'codex', + threadId: 'thread-held', + turnId: 'turn-1', + ordinal: 1 + }, + fence: 3 + }) + ).rejects.toThrow('COMMIT failed') + + expect(held.dispatchState).toBe('pending') + expect(journal.submission('send-1')).toMatchObject({ dispatchState: 'pending' }) + }) + + it('re-reads before next use a fold whose undo failed, and keeps the chat open', async () => { + const journal = await write('settled') + const tip = journal.cursor() + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.mocked(JournalFoldUndo.beginJournalFoldUndo).mockImplementationOnce(() => ({ + commit: () => undefined, + rollback: () => { + throw new Error('undo failed') + } + })) + failNextCommit() + + await expect(note(journal, 'lost')).rejects.toThrow('COMMIT failed') + + // The next read re-reads the disk instead of serving the fold that held the lost row. + expect(journal.cursor()).toEqual(tip) + expect(journal.snapshot()).toEqual(renderJournalState(loadTestJournal(root, 'settled')!.state)) + await expect(note(journal, 'kept')).resolves.toMatchObject({ + cursor: { sequence: tip.sequence + 1 } + }) + }) +}) + +describe('a stale fold is never read back inside a transaction (R2W-3)', () => { + it('keeps a stale fold on the committed epoch through a roll whose COMMIT fails', async () => { + const journal = await write('settled') + const committedEpoch = journal.epoch + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + // An append fails and its undo fails too: the fold is stale. + vi.mocked(JournalFoldUndo.beginJournalFoldUndo).mockImplementationOnce(() => ({ + commit: () => undefined, + rollback: () => { + throw new Error('undo failed') + } + })) + failNextCommit() + await expect(note(journal, 'lost')).rejects.toThrow('COMMIT failed') + // The next write is a roll, and its COMMIT fails too. + vi.restoreAllMocks() + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + failNextCommit() + + await expect(journal.rollEpoch('handle_forked', 3)).rejects.toThrow('COMMIT failed') + + expect(journal.epoch).toBe(committedEpoch) + expect(loadTestJournal(root, 'settled')?.state.epoch).toBe(committedEpoch) + // An acknowledged append lands where every replay reads it. + await note(journal, 'after-roll') + expect( + [...loadTestJournal(root, 'settled')!.state.items.keys()].some((id) => + id.includes('after-roll') + ) + ).toBe(true) + }) +}) + +describe('epoch writes carry the status (T10)', () => { + it('writes it for a new epoch, a roll and a replacement, in their transactions', async () => { + const journal = await open('epochs') + expect(stored('epochs')).toMatchObject({ lifecycle: 'idle', summary: { status: null } }) + await JOURNAL_SESSION_STATE_CORPUS['running tool'](journal) + expect(stored('epochs')).toMatchObject({ lifecycle: 'running' }) + + await journal.rollEpoch('unreconcilable_prefix', 3) + expect(stored('epochs')).toEqual(freshDerivation('epochs')) + expect(stored('epochs')).toMatchObject({ lifecycle: 'idle' }) + + await journal.replaceEpochItems('legacy_import', 3, [ + { + identity: { provider: 'orca', clientMessageId: 'rebuilt-1' }, + body: { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'rebuilt' }] } + } + ]) + expect(stored('epochs')).toEqual(freshDerivation('epochs')) + }) + + it('fails a roll whose status write fails, leaving the epoch where it was', async () => { + const journal = await write('settled') + const epoch = journal.epoch + refuseStatusWrites() + + await expect(journal.rollEpoch('handle_forked', 3)).rejects.toThrow('status write refused') + + expect(journal.epoch).toBe(epoch) + expect(loadTestJournal(root, 'settled')?.state.epoch).toBe(epoch) + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.ts b/src/main/native-chat/agent-session-journal/journal-session-state.ts new file mode 100644 index 000000000000..4e4ad0fd08f7 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-state.ts @@ -0,0 +1,304 @@ +// Each chat's status, stored beside its journal. +// +// One row per chat, written in the SAME transaction as every journal write that changes the chat, +// from the fold that write produces: whether work is running or a prompt is waiting, which sends +// were handed over and never answered or are still queued, live child work, and the status summary +// a session list shows. A failed status write fails the write it describes, so a row is always the +// chat's current state. Startup selects the chats a gone process left with work from these rows, +// and seeds every other chat's status from them. +// +// The table needs no schema version: an older build opens the database as before, ignores the table, +// and can append history without a status. So each row records the history tip it was derived from +// and the rules it was derived by, and a row whose tip is not the chat's tip now, or whose rules are +// other, reads as missing: it is derived again exactly as a chat with no row is, when the chat opens +// (a listed one in the background restore after the listing). + +import type Database from '../../sqlite/sync-database' +import { isAgentTurnOutcome } from '../../../shared/agent-turn-outcome' +import { + projectStructuredAgentSessionStatusState, + type StructuredAgentSessionStatusProjection +} from '../../../shared/structured-agent-session-projection' +import { journalSettlementFacts } from './journal-open-settlement-plan' +import { renderJournalState, type JournalReducerState } from './journal-reducer' + +/** What the stored status is derived by: the derivation below and the shared projection it reads. + * Bump it with any change to what either produces for the same journal; the corpus digest test + * fails until it is bumped. */ +export const JOURNAL_SESSION_STATUS_RULES = 3 + +/** Work a gone process can have left: running work, a waiting prompt, unanswered or queued sends, + * or live child work. Startup settles exactly these chats. */ +const UNSETTLED = `lifecycle <> 'idle' OR handed_over_sends > 0 OR queued_sends > 0 OR live_child_work = 1` + +/** A row whose recorded tip is still the chat's tip: no row was written past it or in its place. + * `st` is the status row; each check is one primary-key lookup on `journal_rows`. */ +const AT_TIP = `EXISTS (SELECT 1 FROM journal_sessions live JOIN journal_rows tip + ON tip.session_id = live.session_id AND tip.epoch = live.epoch AND tip.seq = st.tip_seq + WHERE live.session_id = st.session_id AND live.epoch = st.epoch AND tip.ts = st.tip_ts) + AND NOT EXISTS (SELECT 1 FROM journal_rows later WHERE later.session_id = st.session_id + AND later.epoch = st.epoch AND later.seq > st.tip_seq)` + +/** Created idempotently at every writable open, with no `user_version` bump, as + * `ensureQueuedMessagesTable` is: an older build stays writable and ignores it. */ +export function ensureJournalSessionStatusTable(db: Database.Database): void { + db.exec(` +CREATE TABLE IF NOT EXISTS journal_session_state ( + session_id TEXT PRIMARY KEY, + lifecycle TEXT NOT NULL, + active_turn_id TEXT, + handed_over_sends INTEGER NOT NULL, + queued_sends INTEGER NOT NULL, + live_child_work INTEGER NOT NULL, + summary_json TEXT NOT NULL, + last_activity_at INTEGER NOT NULL, + rules_version INTEGER NOT NULL, + epoch TEXT NOT NULL, + tip_seq INTEGER NOT NULL, + tip_ts INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS journal_session_state_unsettled ON journal_session_state (session_id) + WHERE ${UNSETTLED}; +`) +} + +export type JournalSessionStatus = { + /** What a settle would revise: work still running (a starting turn included: the journal records + * no separate start) or a prompt waiting. Not the status a session list shows, which is + * `summary.status`: that follows the newest turn and counts unanswered sends as working. */ + lifecycle: 'idle' | 'running' | 'attention' + activeTurnId: string | null + handedOverSends: number + queuedSends: number + liveChildWork: boolean + summary: StructuredAgentSessionStatusProjection + lastActivityAt: number +} + +export type JournalSessionStatusInput = { + currentFence?: number + /** The fold's status summary, when the caller already projects this tip. */ + statusSummary?: () => StructuredAgentSessionStatusProjection +} + +export function deriveJournalSessionStatus( + state: JournalReducerState, + input: JournalSessionStatusInput = {} +): JournalSessionStatus { + const facts = journalSettlementFacts(state) + return { + lifecycle: facts.runningWork ? 'running' : facts.pendingPrompts ? 'attention' : 'idle', + activeTurnId: facts.activeTurnId, + handedOverSends: facts.handedOverSends, + queuedSends: facts.queuedSends, + liveChildWork: facts.liveChildWork, + summary: input.statusSummary + ? input.statusSummary() + : projectSummary(state, input.currentFence), + lastActivityAt: state.lastActivityAt + } +} + +function projectSummary( + state: JournalReducerState, + fence: number | undefined +): StructuredAgentSessionStatusProjection { + const snapshot = renderJournalState(state) + return projectStructuredAgentSessionStatusState(snapshot.items, snapshot.submissions, fence) + .summary +} + +export function isUnsettledJournalSessionStatus(status: JournalSessionStatus): boolean { + return ( + status.lifecycle !== 'idle' || + status.handedOverSends > 0 || + status.queuedSends > 0 || + status.liveChildWork + ) +} + +// The tip is read inside the statement, so a row always records the history it was written with. +const UPSERT_STATUS = `INSERT INTO journal_session_state (session_id, lifecycle, active_turn_id, + handed_over_sends, queued_sends, live_child_work, summary_json, last_activity_at, rules_version, + epoch, tip_seq, tip_ts) +SELECT s.session_id, ?, ?, ?, ?, ?, ?, ?, ?, s.epoch, r.seq, r.ts +FROM journal_sessions s JOIN journal_rows r ON r.session_id = s.session_id AND r.epoch = s.epoch +WHERE s.session_id = ? ORDER BY r.seq DESC LIMIT 1 +ON CONFLICT(session_id) DO UPDATE SET + lifecycle = excluded.lifecycle, active_turn_id = excluded.active_turn_id, + handed_over_sends = excluded.handed_over_sends, queued_sends = excluded.queued_sends, + live_child_work = excluded.live_child_work, summary_json = excluded.summary_json, + last_activity_at = excluded.last_activity_at, rules_version = excluded.rules_version, + epoch = excluded.epoch, tip_seq = excluded.tip_seq, tip_ts = excluded.tip_ts` +const STATUS_COLUMNS = `st.lifecycle AS lifecycle, st.active_turn_id AS active_turn_id, + st.handed_over_sends AS handed_over_sends, st.queued_sends AS queued_sends, + st.live_child_work AS live_child_work, st.summary_json AS summary_json, + st.last_activity_at AS last_activity_at` + +/** Inside the transaction that wrote the rows it describes. */ +export function writeJournalSessionStatus( + db: Database.Database, + sessionId: string, + status: JournalSessionStatus +): void { + const written = db + .prepare(UPSERT_STATUS) + .run( + status.lifecycle, + status.activeTurnId, + status.handedOverSends, + status.queuedSends, + status.liveChildWork ? 1 : 0, + JSON.stringify(status.summary), + status.lastActivityAt, + JOURNAL_SESSION_STATUS_RULES, + sessionId + ) + if (written.changes === 0) { + throw new Error(`no journal history to store the status of ${sessionId} beside`) + } +} + +export function deleteJournalSessionStatus(db: Database.Database, sessionId: string): void { + db.prepare('DELETE FROM journal_session_state WHERE session_id = ?').run(sessionId) +} + +/** Whether the chat has a current row: by these rules, at its tip. An open writes one for a chat an + * older build last wrote, or wrote by other rules. */ +export function hasJournalSessionStatus(db: Database.Database, sessionId: string): boolean { + return ( + db + .prepare( + `SELECT 1 FROM journal_session_state st WHERE st.session_id = ? AND st.rules_version = ? + AND ${AT_TIP}` + ) + .get(sessionId, JOURNAL_SESSION_STATUS_RULES) !== undefined + ) +} + +/** A chat's stored status for the startup seed: null when the chat has a journal but no current + * row. */ +export type StoredJournalSessionStatus = { + sessionId: string + status: JournalSessionStatus | null +} + +// Bounded well under SQLite's host-parameter limit. +const IN_LIST_CHUNK = 500 + +/** One query per chunk of ids; ids with no journal yet are not returned. */ +export function readJournalSessionStatuses( + db: Database.Database, + sessionIds: readonly string[] +): StoredJournalSessionStatus[] { + const results: StoredJournalSessionStatus[] = [] + for (let start = 0; start < sessionIds.length; start += IN_LIST_CHUNK) { + const chunk = sessionIds.slice(start, start + IN_LIST_CHUNK) + const rows = db + .prepare( + `SELECT s.session_id AS session_id, ${STATUS_COLUMNS} +FROM journal_sessions s LEFT JOIN journal_session_state st + ON st.session_id = s.session_id AND st.rules_version = ? AND ${AT_TIP} +WHERE s.session_id IN (${chunk.map(() => '?').join(', ')})` + ) + .all(JOURNAL_SESSION_STATUS_RULES, ...chunk) + for (const row of rows) { + if (typeof row.session_id === 'string') { + results.push({ sessionId: row.session_id, status: parseStatusRow(row) }) + } + } + } + return results +} + +/** Every chat whose current row says a gone process left it work, through the partial index. A row + * left behind by a chat that is gone or has moved on is never selected. */ +export function readUnsettledJournalSessionIds(db: Database.Database): string[] { + return db + .prepare( + `SELECT st.session_id AS session_id FROM journal_session_state st +WHERE st.rules_version = ? AND (${UNSETTLED}) AND ${AT_TIP}` + ) + .all(JOURNAL_SESSION_STATUS_RULES) + .flatMap((row) => (typeof row.session_id === 'string' ? [row.session_id] : [])) +} + +const LIFECYCLES = ['idle', 'running', 'attention'] as const + +function parseStatusRow(row: Record): JournalSessionStatus | null { + const lifecycle = LIFECYCLES.find((known) => known === row.lifecycle) + const summary = parseSummary(row.summary_json) + if ( + !lifecycle || + !summary || + typeof row.handed_over_sends !== 'number' || + typeof row.queued_sends !== 'number' || + typeof row.last_activity_at !== 'number' + ) { + return null + } + return { + lifecycle, + activeTurnId: typeof row.active_turn_id === 'string' ? row.active_turn_id : null, + handedOverSends: row.handed_over_sends, + queuedSends: row.queued_sends, + liveChildWork: row.live_child_work === 1, + summary, + lastActivityAt: row.last_activity_at + } +} + +function parseJson(value: unknown): unknown { + if (typeof value !== 'string') { + return null + } + try { + return JSON.parse(value) + } catch { + return null + } +} + +const SUMMARY_STATUSES = ['idle', 'working', 'attention'] as const + +function parseSummaryStatus( + value: unknown +): StructuredAgentSessionStatusProjection['status'] | undefined { + return value === null ? null : SUMMARY_STATUSES.find((status) => status === value) +} + +/** Known keys only. A value this build cannot read makes the row unreadable, so the chat is opened + * and publishes what its open derives rather than a row missing that field. */ +function parseSummary(value: unknown): StructuredAgentSessionStatusProjection | null { + const parsed = parseJson(value) + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + return null + } + const source = new Map(Object.entries(parsed)) + const status = parseSummaryStatus(source.get('status')) + const latestPrompt = source.get('latestPrompt') + if (status === undefined || typeof latestPrompt !== 'string') { + return null + } + const text = (key: string): string | undefined => { + const field = source.get(key) + return typeof field === 'string' ? field : undefined + } + const turnOutcome = source.get('turnOutcome') + if (turnOutcome !== undefined && !isAgentTurnOutcome(turnOutcome)) { + return null + } + const statusStartedAt = source.get('statusStartedAt') + const toolName = text('toolName') + const toolInput = text('toolInput') + const lastAssistantMessage = text('lastAssistantMessage') + return { + status, + latestPrompt, + ...(toolName !== undefined ? { toolName } : {}), + ...(toolInput !== undefined ? { toolInput } : {}), + ...(lastAssistantMessage !== undefined ? { lastAssistantMessage } : {}), + ...(isAgentTurnOutcome(turnOutcome) ? { turnOutcome } : {}), + ...(typeof statusStartedAt === 'number' ? { statusStartedAt } : {}) + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-rules.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-rules.test.ts new file mode 100644 index 000000000000..e8a4df7fb1f7 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-status-rules.test.ts @@ -0,0 +1,65 @@ +// The stored status is a cache of its derivation: a row derived by other rules reads as missing and +// is derived again. This pins what the derivation writes for every corpus chat, so a change to it +// (or to the shared projection it reads) fails here until the rules version is bumped with it. + +import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { createHash } from 'node:crypto' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import { + createTrackedJournalOpener, + readTestJournalSessionStatus +} from './journal-host-database-test-support' +import { JOURNAL_SESSION_STATUS_RULES } from './journal-session-state' +import { + CORPUS_FENCE, + JOURNAL_SESSION_STATE_CASES, + JOURNAL_SESSION_STATE_CORPUS +} from './journal-session-state-test-corpus' + +// Recorded with the rules version: change both together, after checking the new rows are right. A +// new corpus case changes only the digest: record it, with no bump. +const DERIVED = { + rules: 3, + digest: 'f0118b4048c1f6e1feb53d2105557019564c08d4600c69f200a2abbd44ba1d15' +} + +const journals = createTrackedJournalOpener() +const roots: string[] = [] + +afterEach(async () => { + await journals.closeAll() + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) +}) + +it('derives the same row for every corpus chat as the recorded rules version', async () => { + const rows: Record = {} + for (const name of JOURNAL_SESSION_STATE_CASES) { + const root = await mkdtemp(join(tmpdir(), 'orca-status-rules-')) + roots.push(root) + let clock = 1_000 + const journal = await journals.open({ + identity: { + sessionId: name, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: codexProviderHandle(`thread-${name}`) + }, + stateDirectory: root, + now: () => (clock += 1), + mintEpoch: () => `epoch-${name}`, + currentFence: () => CORPUS_FENCE + }) + await JOURNAL_SESSION_STATE_CORPUS[name](journal) + rows[name] = readTestJournalSessionStatus(root, name) + } + const digest = createHash('sha256').update(JSON.stringify(rows)).digest('hex') + + expect( + { rules: JOURNAL_SESSION_STATUS_RULES, digest }, + 'The stored status derivation changed: bump JOURNAL_SESSION_STATUS_RULES and record the new digest' + ).toEqual(DERIVED) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts new file mode 100644 index 000000000000..4893a50b79fc --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts @@ -0,0 +1,175 @@ +// A stored status names the history tip it was derived from and the rules it was derived by. An +// older build writes history with no status, or deletes a chat, and the table carries no schema +// version to stop it: a row whose tip is not the chat's tip now, or whose rules are other, reads as +// missing, is derived again, and is never selected to settle. + +import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { + createTrackedJournalOpener, + deleteTestJournalRow, + insertTestJournalRowJson, + liveTestJournalRows, + loadTestJournal, + openTestJournalHostDatabase, + publishTestJournalEpoch, + readTestJournalSessionStatus +} from './journal-host-database-test-support' +import { + deriveJournalSessionStatus, + hasJournalSessionStatus, + JOURNAL_SESSION_STATUS_RULES, + readUnsettledJournalSessionIds +} from './journal-session-state' +import { CORPUS_FENCE, JOURNAL_SESSION_STATE_CORPUS } from './journal-session-state-test-corpus' +import { codexItem, item, runningTool } from './journal-session-state-test-writes' +import type { AgentSessionJournal } from './journal-store' +import type { SqliteRow } from '../../sqlite/sqlite-statement' + +const journals = createTrackedJournalOpener() +let root: string +let clock = 1_000 + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-journal-status-tip-')) + clock = 1_000 +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +function open(sessionId: string): Promise { + return journals.open({ + identity: { + sessionId, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: codexProviderHandle(`thread-${sessionId}`) + }, + stateDirectory: root, + now: () => (clock += 1), + mintEpoch: () => `epoch-${sessionId}-${clock}`, + currentFence: () => CORPUS_FENCE + }) +} + +const db = () => openTestJournalHostDatabase(root).db +const stored = (sessionId: string) => readTestJournalSessionStatus(root, sessionId) + +function freshDerivation(sessionId: string) { + return deriveJournalSessionStatus(loadTestJournal(root, sessionId)!.state, { + currentFence: CORPUS_FENCE + }) +} + +/** The stored row as SQLite holds it, every column. */ +function rawRow(sessionId: string): SqliteRow { + const row = db() + .prepare('SELECT * FROM journal_session_state WHERE session_id = ?') + .get(sessionId) + if (!row) { + throw new Error(`no status row for ${sessionId}`) + } + return row +} + +/** Puts a row back exactly as it was: what is left when a build that keeps no status writes on. */ +function putBack(row: SqliteRow): void { + const columns = Object.keys(row) + db() + .prepare( + `INSERT OR REPLACE INTO journal_session_state (${columns.join(', ')}) +VALUES (${columns.map(() => '?').join(', ')})` + ) + .run(...Object.values(row)) +} + +it('reads a row an older build wrote history past as missing, and derives it again', async () => { + const journal = await open('chat') + await JOURNAL_SESSION_STATE_CORPUS.settled(journal) + const before = rawRow('chat') + // The history an older build appends: a tool starts running, and the row stays as it was. + await item(journal, codexItem('turn-1', 4), runningTool) + await journal.close() + putBack(before) + + expect(stored('chat')).toBeNull() + expect(hasJournalSessionStatus(db(), 'chat')).toBe(false) + expect(readUnsettledJournalSessionIds(db())).toEqual([]) + + const reopened = await open('chat') + reopened.sessionStatus.backfill() + + expect(stored('chat')).toEqual(freshDerivation('chat')) + expect(stored('chat')).toMatchObject({ lifecycle: 'running' }) + expect(readUnsettledJournalSessionIds(db())).toEqual(['chat']) +}) + +it('reads a row as missing when a repair in the same epoch regrew the history to its tip', async () => { + const journal = await open('chat') + await JOURNAL_SESSION_STATE_CORPUS['running tool'](journal) + await journal.close() + const tip = liveTestJournalRows(db(), 'chat').at(-1)! + expect(stored('chat')).not.toBeNull() + + // An older build's repair dropped the tip and wrote a row at the same sequence, later. + deleteTestJournalRow(db(), 'chat', tip.seq) + insertTestJournalRowJson(db(), 'chat', tip.seq, tip.rowJson, tip.ts + 1) + + expect(stored('chat')).toBeNull() + expect(readUnsettledJournalSessionIds(db())).toEqual([]) +}) + +it('reads a row as missing when an older build moved the chat to a new epoch, even at the same tip', async () => { + const journal = await open('chat') + await JOURNAL_SESSION_STATE_CORPUS['running tool'](journal) + await journal.close() + const tip = liveTestJournalRows(db(), 'chat').at(-1)! + + // A new epoch whose newest row matches the old tip's sequence and time. + publishTestJournalEpoch(db(), 'chat', 'epoch-older-build') + insertTestJournalRowJson(db(), 'chat', tip.seq, tip.rowJson, tip.ts) + + expect(stored('chat')).toBeNull() + expect(readUnsettledJournalSessionIds(db())).toEqual([]) +}) + +it('never selects the row of a chat whose history is gone', async () => { + const journal = await open('chat') + await JOURNAL_SESSION_STATE_CORPUS['running tool'](journal) + await journal.close() + expect(readUnsettledJournalSessionIds(db())).toEqual(['chat']) + + // An older build deleted the chat and knew nothing of its status row. + db().prepare('DELETE FROM journal_rows WHERE session_id = ?').run('chat') + db().prepare('DELETE FROM journal_sessions WHERE session_id = ?').run('chat') + + expect(rawRow('chat')).toMatchObject({ lifecycle: 'running' }) + expect(readUnsettledJournalSessionIds(db())).toEqual([]) +}) + +it('reads a row derived by other rules as missing, never selects it, and derives it again', async () => { + const journal = await open('chat') + await JOURNAL_SESSION_STATE_CORPUS['running tool'](journal) + await journal.close() + db() + .prepare('UPDATE journal_session_state SET rules_version = ? WHERE session_id = ?') + .run(JOURNAL_SESSION_STATUS_RULES - 1, 'chat') + + expect(stored('chat')).toBeNull() + expect(hasJournalSessionStatus(db(), 'chat')).toBe(false) + expect(readUnsettledJournalSessionIds(db())).toEqual([]) + + const reopened = await open('chat') + reopened.sessionStatus.backfill() + + expect(rawRow('chat')).toMatchObject({ rules_version: JOURNAL_SESSION_STATUS_RULES }) + expect(stored('chat')).toEqual(freshDerivation('chat')) + expect(readUnsettledJournalSessionIds(db())).toEqual(['chat']) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-writer.ts b/src/main/native-chat/agent-session-journal/journal-session-status-writer.ts new file mode 100644 index 000000000000..5e2f238c6a85 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-status-writer.ts @@ -0,0 +1,154 @@ +// The chat's stored status (journal_session_state). Every append folds its rows inside its +// transaction and writes the status from that fold there, so a stored status never describes rows +// that did not commit; a failed transaction puts the fold back. An epoch's new fold, and a chat an +// older build last wrote, get theirs here too. + +import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' +import type Database from '../../sqlite/sync-database' +import { beginJournalFoldUndo, type JournalFoldUndo } from './journal-fold-undo' +import type { JournalHostDatabase } from './journal-host-database' +import type { JournalLoad } from './journal-open' +import { applyJournalRow, type JournalReducerState } from './journal-reducer' +import type { JournalRow } from './journal-row-schema' +import { + deleteJournalSessionStatus, + deriveJournalSessionStatus, + hasJournalSessionStatus, + writeJournalSessionStatus +} from './journal-session-state' +import { + JournalStatusProjection, + type JournalStatusProjectionState +} from './journal-status-projection' + +export type JournalSessionStatusHost = { + identity: AgentSessionJournalIdentity + state: () => JournalReducerState + database: () => JournalHostDatabase + /** A per-chat file's copy is still owed: the fold is not the database's yet. */ + importPending: () => boolean + /** The conversation's fence, which the stored status reads as the status feed does. */ + currentFence: () => number | undefined + /** The undo failed too: the fold is re-read before its next use, never served as it is. */ + markFoldStale: () => void + notifyCommitted: () => void +} + +/** What the store hands its readers: the projection, and the backfill an open runs. */ +export type JournalSessionStatusAccess = Pick + +export class JournalSessionStatusWriter { + private readonly projection: JournalStatusProjection + // The open append's undo: what its rows changed in the fold, put back if its transaction fails. + private undo: JournalFoldUndo | null = null + + constructor(private readonly host: JournalSessionStatusHost) { + this.projection = new JournalStatusProjection(host.state) + } + + /** The status projection at this tip, projected once per commit for every reader. */ + at = (fence: number | undefined): JournalStatusProjectionState => this.projection.at(fence) + + /** `live`: the store's own fold, whose projection the status feed shares; an epoch's new fold + * projects its own. Decided by the caller, so nothing reads the store's fold inside the + * transaction. */ + write(db: Database.Database, state: JournalReducerState, live: boolean): void { + const currentFence = this.host.currentFence() + writeJournalSessionStatus( + db, + this.host.identity.sessionId, + deriveJournalSessionStatus(state, { + currentFence, + ...(live ? { statusSummary: () => this.projection.at(currentFence).summary } : {}) + }) + ) + } + + /** Inside an append's transaction: folds its rows under one undo. */ + apply(rows: readonly JournalRow[]): void { + const state = this.host.state() + this.undo = beginJournalFoldUndo(state) + for (const row of rows) { + applyJournalRow(state, row) + } + } + + /** Inside an append's transaction, after `apply`: the status of the fold that holds its rows. */ + writeAppended(db: Database.Database): void { + this.write(db, this.host.state(), true) + } + + /** After the append's COMMIT. */ + committed(): void { + this.undo?.commit() + this.undo = null + this.host.notifyCommitted() + } + + /** + * An append whose transaction failed after its rows were folded: the undo puts back what they + * changed. If it cannot (it throws, or a row removed an entry, which no undo puts back in its + * place), the fold is marked stale and folded again from what committed before its next use (the + * host database rolls a stranded transaction back before it hands out the connection, so that + * re-read never sees the failed rows). + */ + recover(): void { + // What the projection read of the failed rows must not answer for the next ones at their seq. + this.projection.invalidate() + const failed = this.undo + this.undo = null + try { + if (failed?.rollback()) { + return + } + } catch (error) { + console.warn('[agent-session-journal] undoing a failed append failed', { + sessionId: this.host.identity.sessionId, + error + }) + } + this.host.markFoldStale() + } + + /** A load whose open fails (damage, or a newer build's rows) drops the chat's row, so no startup + * selects it again; an open that succeeds writes it back. Bookkeeping: a failure only logs. */ + forgetUnloadable = (load: JournalLoad): void => { + const database = this.host.database() + if ((!load.newer && !load.damage) || database.readOnly) { + return + } + try { + database.transaction((db) => deleteJournalSessionStatus(db, this.host.identity.sessionId)) + } catch (error) { + console.warn('[agent-session-journal] dropping an unloadable chat status failed', { + sessionId: this.host.identity.sessionId, + error + }) + } + } + + /** Writes the chat's status if it has no current row: one last written before stored status, by + * an older build or by other rules. Bookkeeping: a failure leaves the chat without a row, which + * its next open writes again. */ + backfill = (): void => { + const { host } = this + const database = host.database() + if (database.readOnly || host.importPending()) { + return + } + try { + // Read before the transaction: a stale fold is re-read from disk, never inside one. + const state = host.state() + database.transaction((db) => { + if (!hasJournalSessionStatus(db, host.identity.sessionId)) { + this.write(db, state, true) + } + }) + } catch (error) { + console.warn('[agent-session-journal] writing a chat status failed', { + sessionId: host.identity.sessionId, + error + }) + } + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-status-projection.ts b/src/main/native-chat/agent-session-journal/journal-status-projection.ts new file mode 100644 index 000000000000..3830ff20c07e --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-status-projection.ts @@ -0,0 +1,58 @@ +// A chat's status projection at its fold's tip, kept for one (fold, epoch, seq, fence): the status +// feed and the stored state (journal-session-state.ts) read the same one, so a commit renders and +// projects the whole chat once. + +import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' +import { renderJournalState, type JournalReducerState } from './journal-reducer' + +export type JournalStatusProjectionState = ReturnType< + typeof projectStructuredAgentSessionStatusState +> + +type Cached = { + fold: JournalReducerState + epoch: string + sequence: number + fence: number | undefined + state: JournalStatusProjectionState +} + +export class JournalStatusProjection { + private cached: Cached | null = null + + constructor(private readonly fold: () => JournalReducerState) {} + + /** The projection under this fence; the fence moves when a child ends, the tip does not. */ + at(fence: number | undefined): JournalStatusProjectionState { + const tip = this.atTip() + return tip && tip.fence === fence ? tip.state : this.project(fence) + } + + /** A failed append's projection read a row that never committed. */ + invalidate(): void { + this.cached = null + } + + private atTip(): Cached | null { + const fold = this.fold() + const cached = this.cached + return cached && + cached.fold === fold && + cached.epoch === fold.epoch && + cached.sequence === fold.lastSequence + ? cached + : null + } + + private project(fence: number | undefined): JournalStatusProjectionState { + const fold = this.fold() + const snapshot = renderJournalState(fold) + const state = projectStructuredAgentSessionStatusState( + snapshot.items, + snapshot.submissions, + fence + ) + this.cached = { fold, epoch: fold.epoch, sequence: fold.lastSequence, fence, state } + return state + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 395ad50a6ee4..514851d99963 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -22,6 +22,10 @@ import { restoreJournalStore } from './journal-store-restore' import { JournalSubmissionWriter } from './journal-submission-writer' import type { JournalRow } from './journal-row-schema' import type { AgentSessionJournal } from './journal-store' +import { journalRowsAfterReader } from './journal-open' +import { readJournalSince } from './journal-cursor' +import type { JournalReadSince } from './journal-store-contracts' +import { JournalSessionStatusWriter } from './journal-session-status-writer' import type { JournalWriteBody } from './journal-write-queue' export type JournalStoreHost = { @@ -44,7 +48,12 @@ export type JournalStoreHost = { readOnly: () => boolean cursor: () => AgentJournalCursor adopt: (loaded: JournalLoad) => void - commit: (row: JournalRow) => void + /** That re-read failed too: the fold is re-read before its next use, never served as it is. */ + markFoldStale: () => void + /** The conversation's fence, which the stored status reads as the status feed does. */ + currentFence: () => number | undefined + /** A per-chat file's copy is still owed: the fold is not the database's yet. */ + importPending: () => boolean journal: () => AgentSessionJournal enqueue: (build: (seq: number, ts: number) => JournalRow) => Promise } @@ -61,9 +70,12 @@ export type JournalStoreCollaborators = { /** Restores the store's state from disk. Owned here because it needs the same * collaborators the constructor just built. */ restore: () => Promise + readSince: (cursor: AgentJournalCursor, limit?: number) => JournalReadSince + sessionStatus: JournalSessionStatusWriter } export function createJournalStoreCollaborators(host: JournalStoreHost): JournalStoreCollaborators { + const sessionStatus = new JournalSessionStatusWriter(host) const epochController = new JournalEpochController({ identity: host.identity, now: host.now, @@ -78,7 +90,8 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal host.state().latestAcceptedTurnSequence ), cursor: host.cursor, - adopt: host.adopt + adopt: host.adopt, + writeState: (db, state) => sessionStatus.write(db, state, false) }) const queuedMessages = new JournalQueuedMessages({ sessionId: host.identity.sessionId, @@ -98,7 +111,10 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal readOnly: host.readOnly, highestFence: () => host.state().highestFence, nextSequence: () => host.state().lastSequence + 1, - commit: host.commit, + apply: (rows) => sessionStatus.apply(rows), + writeStatus: (db) => sessionStatus.writeAppended(db), + committed: () => sessionStatus.committed(), + recoverFold: () => sessionStatus.recover(), // Every rejection is a dispatch row through this one writer; the draft // returned-transition rides it so no path can bypass the hook. inTransaction: (db, row) => queuedMessages.onRowInTransaction(db, row), @@ -113,11 +129,27 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal return { epochController, queuedMessages, + sessionStatus, + // Here rather than on the store, which is at its length limit. + readSince: (cursor, limit) => + readJournalSince( + { + state: host.state(), + rowsAfter: journalRowsAfterReader( + host.database().db, + host.identity.sessionId, + host.state().epoch, + limit + ) + }, + cursor, + host.cursor + ), stopMarks: new JournalStopMarks({ state: host.state }), // Behind the stored fact: settles drafts whose consumed submission the loaded journal shows // refused (a downgrade wrote no hook), then prunes. Bookkeeping, never failing the open. restore: () => - restoreJournalStore(host, { epochController }).then(() => + restoreJournalStore(host, { epochController, sessionStatus }).then(() => queuedMessages.repairAndPruneAtOpen() ), rowWriter, diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index 2d7b56d06b1f..0374ee548b76 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -23,6 +23,8 @@ export type AgentSessionJournalOptions = { mintEpoch?: () => string /** A restore's open: see `AgentSessionJournal.whenImported`. */ deferPerSessionImport?: boolean + /** The conversation's fence, which the chat's stored status reads as the status feed does. */ + currentFence?: () => number | undefined } export type JournalReadSince = diff --git a/src/main/native-chat/agent-session-journal/journal-store-open.ts b/src/main/native-chat/agent-session-journal/journal-store-open.ts index cac1517c4091..d54e576755ee 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-open.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-open.ts @@ -9,7 +9,6 @@ import { } from './journal-file-format-remnant' import type { JournalLoad } from './journal-open' import { failLoadOnUnloadableJournal } from './journal-open-failure' -import { staleSubagentRosterRevisions } from './journal-subagent-liveness' type JournalDisclosure = ReturnType @@ -37,7 +36,6 @@ export async function openJournalStoreState(input: { } failLoadOnUnloadableJournal(input.sessionId, loaded) input.adopt(loaded) - await settleStaleSubagentRosters(input, loaded) // Founding the epoch and appending the row are two transactions, and a // committed epoch sends every later open down this branch instead. Anything // that interrupts between them — a quit during startup restore, a failed @@ -68,20 +66,3 @@ async function discloseFileFormatRemnant(input: { const disclosure = journalFileFormatRemnantDisclosure({ transcriptPath, agent: input.agent }) await input.appendItem(disclosure.identity, disclosure.body, input.highestFence()) } - -/** Retires a `working` subagent roster the previous host never got to settle. */ -async function settleStaleSubagentRosters( - input: { - appendItem: ( - identity: AgentJournalItemIdentity, - body: AgentJournalItemBody, - fence: number - ) => Promise - highestFence: () => number - }, - loaded: JournalLoad -): Promise { - for (const revision of staleSubagentRosterRevisions(loaded.state.items.values())) { - await input.appendItem(revision.identity, revision.body, input.highestFence()) - } -} diff --git a/src/main/native-chat/agent-session-journal/journal-store-restore.ts b/src/main/native-chat/agent-session-journal/journal-store-restore.ts index d8c44f964bdb..f44f393b9111 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-restore.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-restore.ts @@ -8,6 +8,7 @@ import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' import type { JournalEpochController } from './journal-epoch-controller' +import type { JournalSessionStatusWriter } from './journal-session-status-writer' import { replayJournal } from './journal-open' import { failLoadOnUnloadableJournal, journalOpenRefusalError } from './journal-open-failure' import type { JournalStoreHost } from './journal-store-collaborators' @@ -17,7 +18,10 @@ import { AgentSessionJournalError } from './journal-write-guards' export async function restoreJournalStore( host: JournalStoreHost, - collaborators: { epochController: JournalEpochController } + collaborators: { + epochController: JournalEpochController + sessionStatus: Pick + } ): Promise { const source = { database: host.database(), @@ -42,6 +46,7 @@ export async function restoreJournalStore( if (!imported) { throw new Error(`per-chat journal of ${host.identity.sessionId} was gone before its copy`) } + collaborators.sessionStatus.forgetUnloadable(imported) failLoadOnUnloadableJournal(host.identity.sessionId, imported) host.adopt(imported) }) @@ -52,7 +57,13 @@ export async function restoreJournalStore( return openJournalStoreState({ sessionId: host.identity.sessionId, legacyDirectory: host.legacyDirectory, - replay: () => preview ?? replayJournal(host.database().db, host.identity.sessionId), + replay: () => { + const loaded = preview ?? replayJournal(host.database().db, host.identity.sessionId) + if (loaded) { + collaborators.sessionStatus.forgetUnloadable(loaded) + } + return loaded + }, start: () => collaborators.epochController.start('session_created', 0), adopt: host.adopt, // File-format and roster notices are about the conversation, not any turn in it. diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 146c141bda07..49f7af7bd117 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -26,17 +26,14 @@ import { newestStructuredAgentSessionTurnBySequence } from '../../../shared/structured-agent-session-live-turn' import type { JournalReplacementItem } from './journal-epoch-replacement' -import { readJournalSince } from './journal-cursor' import type { JournalHostDatabase } from './journal-host-database' -import { journalRowsAfterReader, type JournalLoad } from './journal-open' +import type { JournalLoad } from './journal-open' import { markJournalPendingSubmissionsUnknown, rejectJournalPendingSubmissions, rejectJournalQueuedSubmissions } from './journal-pending-submission-recovery' import { - applyJournalRow, - createJournalReducerState, renderJournalState, resolveJournalItemId, type JournalReducerState @@ -55,6 +52,7 @@ import type { ResolveDispatchInput } from './journal-store-contracts' import type { JournalQueuedMessages } from './journal-queued-messages' +import type { JournalSessionStatusAccess } from './journal-session-status-writer' import { journalQueueResumeRowBuilder, journalStopEventRowBuilder @@ -73,6 +71,7 @@ import type { JournalItemAppender } from './journal-item-appender' import type { JournalLifecycleBatchAppender } from './journal-lifecycle-batch-appender' import type { JournalStepWriter } from './journal-step-writer' import type { JournalStopMarks } from './journal-stop-marks' +import { JournalFoldHolder } from './journal-fold-holder' export { AgentSessionJournalError } from './journal-write-guards' @@ -82,7 +81,7 @@ export class AgentSessionJournal { private readonly now: () => number private readonly mintEpoch: () => string - private state: JournalReducerState + private readonly fold: JournalFoldHolder private openedThrough: AgentJournalCursor = { epoch: '', sequence: 0 } private onCommitted: (() => void) | null = null private readonly queue: JournalWriteQueue @@ -93,6 +92,9 @@ export class AgentSessionJournal { private readonly submissionWriter: JournalSubmissionWriter private readonly stepWriter: JournalStepWriter private readonly restore: () => Promise + readonly readSince: (cursor: AgentJournalCursor, limit?: number) => JournalReadSince + /** The chat's status: projected once per commit for every reader, and stored with each write. */ + readonly sessionStatus: JournalSessionStatusAccess /** Draft rows queued while the agent works; never reducer input or owed work. */ readonly queuedMessages: JournalQueuedMessages readonly stopMarks: JournalStopMarks @@ -102,7 +104,7 @@ export class AgentSessionJournal { this.database = options.database this.now = options.now ?? (() => Date.now()) this.mintEpoch = options.mintEpoch ?? randomUUID - this.state = createJournalReducerState(options.identity.sessionId, '') + this.fold = new JournalFoldHolder(options.identity.sessionId, options.database) // Serializes sequence assignment with the durable write behind it. this.queue = new JournalWriteQueue(options.identity.sessionId) const collaborators = createJournalStoreCollaborators({ @@ -122,10 +124,9 @@ export class AgentSessionJournal { this.adoptLoadedJournal(loaded) this.onCommitted?.() }, - commit: (row) => { - applyJournalRow(this.state, row) - this.onCommitted?.() - }, + markFoldStale: () => this.fold.markStale(), + currentFence: options.currentFence ?? (() => undefined), + importPending: () => this.queue.owing, notifyCommitted: () => this.onCommitted?.(), journal: () => this, enqueue: (build) => this.rowWriter.enqueue(build) @@ -139,6 +140,16 @@ export class AgentSessionJournal { this.queuedMessages = collaborators.queuedMessages this.stopMarks = collaborators.stopMarks this.restore = collaborators.restore + this.readSince = collaborators.readSince + this.sessionStatus = collaborators.sessionStatus + } + + private get state(): JournalReducerState { + return this.fold.get() + } + + private set state(state: JournalReducerState) { + this.fold.set(state) } get epoch(): string { @@ -246,6 +257,9 @@ export class AgentSessionJournal { /** Includes revisions and completion tombstones, whose timestamps disappear from render items. */ lastActivityAt = (): number => this.state.lastActivityAt + /** The highest fence any row carries: a writer that revises what an earlier one left. */ + highestFence = (): number => this.state.highestFence + /** Fence of the writer that created the item, while it is in the timeline. */ itemFence = (itemId: string): number | undefined => this.state.itemFences.get(itemId) @@ -265,15 +279,7 @@ export class AgentSessionJournal { /** Reads the fold with every write issued before this call committed, and none issued after: at * once unless writes still wait behind an owed import or a running write. */ - readInOrder(read: () => T): Promise { - return this.queue.readInOrder(read) - } - - readSince(cursor: AgentJournalCursor, limit?: number): JournalReadSince { - const { sessionId } = this.identity - const rowsAfter = journalRowsAfterReader(this.database.db, sessionId, this.state.epoch, limit) - return readJournalSince({ state: this.state, rowsAfter }, cursor, () => this.cursor()) - } + readInOrder = (read: () => T): Promise => this.queue.readInOrder(read) /** Upsert by stable identity. The revision is assigned here so a caller * cannot accidentally publish a revision the reducer will drop. */ diff --git a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts index 4da76e0ba429..fa938a1420e9 100644 --- a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts @@ -21,6 +21,10 @@ import { import type { openAgentSessionJournal } from './journal-store-factory' import { createTrackedJournalOpener } from './journal-host-database-test-support' import { staleSubagentRosterRevisions } from './journal-subagent-liveness' +import { + appendOpenSettlement, + planOpenSettlement +} from '../agent-session-wire/structured-agent-session-open-settlement' import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' const IDENTITY: AgentSessionJournalIdentity = { @@ -53,6 +57,15 @@ async function open(overrides: Partial { + throw error + }) + return journal +} + /** The row as the producer writes it: the structured block plus its twin. */ function rosterRow(agents: NativeChatSubagentEntry[]) { return { @@ -237,7 +250,7 @@ describe('journal reopen after the writing host is gone', () => { // The host dies without ever settling them — no `ended`, so no session sweep. await live.close() - const reopened = await open() + const reopened = await reopenAfterHostGone() const afterRestart = reopened.snapshot().items.at(-1)! expect(afterRestart.itemId).toBe(beforeRestart.itemId) expect(rosterOf(afterRestart.body)).toMatchObject([ @@ -257,7 +270,7 @@ describe('journal reopen after the writing host is gone', () => { const before = live.snapshot().items.length await live.close() - const reopened = await open() + const reopened = await reopenAfterHostGone() expect(reopened.snapshot().items).toHaveLength(before) expect(reopened.snapshot().items.at(-1)?.revision).toBe(2) }) @@ -274,7 +287,7 @@ describe('journal reopen after the writing host is gone', () => { expect(twinOf(beforeRestart.body)).toBe('Started background command "sleep 20"') await live.close() - const reopened = await open() + const reopened = await reopenAfterHostGone() const afterRestart = reopened.snapshot().items.at(-1)! expect(afterRestart.itemId).toBe(beforeRestart.itemId) expect(taskOf(afterRestart.body)).toMatchObject({ state: 'unverifiable' }) @@ -290,11 +303,11 @@ describe('journal reopen after the writing host is gone', () => { }) await live.close() - const once = await open() + const once = await reopenAfterHostGone() const revision = once.snapshot().items.at(-1)?.revision await once.close() - const twice = await open() + const twice = await reopenAfterHostGone() expect(twice.snapshot().items.at(-1)?.revision).toBe(revision) }) }) diff --git a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts index ea4cebab1504..c771d909dda5 100644 --- a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts +++ b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts @@ -52,21 +52,31 @@ export function staleSubagentRosterRevisions( ): JournalSubagentLivenessRevision[] { const revisions: JournalSubagentLivenessRevision[] = [] for (const item of items) { - const body = lostLiveWorkJournalBody(item.body) - if (!body) { - continue + const revision = staleSubagentRosterRevision(item) + if (revision) { + revisions.push(revision) } - // A key that will not parse cannot be re-addressed, and appending under a - // fresh identity would duplicate the row rather than revise it. - const identity = parseAgentJournalItemKey(item.itemId) - if (!identity || agentJournalItemKey(identity) !== item.itemId) { - continue - } - revisions.push({ identity, body }) } return revisions } +/** One row's revision, or null when it claims no live child. */ +export function staleSubagentRosterRevision( + item: Pick +): JournalSubagentLivenessRevision | null { + const body = lostLiveWorkJournalBody(item.body) + if (!body) { + return null + } + // A key that will not parse cannot be re-addressed, and appending under a + // fresh identity would duplicate the row rather than revise it. + const identity = parseAgentJournalItemKey(item.itemId) + if (!identity || agentJournalItemKey(identity) !== item.itemId) { + return null + } + return { identity, body } +} + /** The row once the host lost the session running its live work: every working child and * in-flight background task `unverifiable`, plain-text twins restated. Null when none is live. */ export function lostLiveWorkJournalBody(body: AgentJournalItemBody): AgentJournalItemBody | null { diff --git a/src/main/native-chat/agent-session-journal/journal-submission-fold.ts b/src/main/native-chat/agent-session-journal/journal-submission-fold.ts index 7b03b4f457f8..a0bf7df0e882 100644 --- a/src/main/native-chat/agent-session-journal/journal-submission-fold.ts +++ b/src/main/native-chat/agent-session-journal/journal-submission-fold.ts @@ -104,12 +104,15 @@ export function acceptSubmissionFromProviderItem( if (providerItemId === resolvedItemId) { return } - const submission = [...state.submissions.values()].find( + const current = [...state.submissions.values()].find( (candidate) => agentJournalSubmissionKey(candidate.clientMessageId) === resolvedItemId ) - if (!submission || !journalDispatchRowApplies(submission)) { + if (!current || !journalDispatchRowApplies(current)) { return } + // Replaced, never changed in place, as the dispatch reducer does. + const submission = { ...current } + state.submissions.set(current.clientMessageId, submission) submission.fence = row.fence submission.dispatchState = 'accepted' noteTurnAccepted(state, submission) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts index 28a675ee83dd..750118006a19 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts @@ -683,6 +683,8 @@ describe('Stop withdraws what is queued', () => { await journal.appendSubmission({ ...earlierSubmission('leftover', 'l', true), fence }) }) + // Stop's own open appends the chat's settlement plan, which settles what a gone process left + // queued before Stop acts: nothing is handed over, and the reason names the restart. expect(await stop()).toMatchObject({ ok: true }) // A person's message is kept as a held card, which no Stop withdraws; the rest is rejected. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts index e801a99042da..7a5b1bce3f89 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts @@ -42,6 +42,7 @@ import { withAgentSessionSpan, type AgentSessionCreatePhaseRecorder } from '../../observability/agent-session-instrumentation' +import { serializeStructuredAgentSessionCommand } from './structured-agent-session-command-entry' export type StructuredAgentSessionAttachOptions = { hostLaunchDirectory?: string @@ -81,7 +82,7 @@ export function attachStructuredAgentSession( // evicts, so no child is spawned behind the eviction and orphaned. const run = (recordPhase?: AgentSessionCreatePhaseRecorder) => context.tasks.trackAttach( - context.serialize(sessionId, () => + serializeStructuredAgentSessionCommand(context, sessionId, () => runAttach(context, callerKey, params, { ...options, recordPhase }) ) ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts index 12ced4eacf81..473e70d2f591 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts @@ -34,6 +34,7 @@ import { } from './structured-agent-session-stop-wind-down' import { isMainAgentWorking, performCancel } from './structured-agent-session-turns-cancel' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' +import { serializeStructuredAgentSessionCommand } from './structured-agent-session-command-entry' type ChatStopOutcome = TurnOutcome @@ -181,8 +182,9 @@ export function mutateWithChatStop( }, openForWrite(context, envelope) ) - // Queued in the mutation's own tick, so a send made meanwhile lands behind the child's end. - void context.serialize(sessionId, async () => { + // Queued in the mutation's own tick, through the same command entry, so a send made meanwhile + // lands behind the child's end, startup's wait included. + void serializeStructuredAgentSessionCommand(context, sessionId, async () => { if (windDown) { const { owed, ctx } = windDown await endStoppedStructuredAgentSession( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts index c369ce1a5602..f8f63beb72e5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts @@ -15,8 +15,11 @@ import type { } from './structured-agent-session-host-types' import { AGENT_SESSION_NOT_ATTACHED } from './structured-agent-session-mutation-admission' import { StructuredAgentSessionSendSettlement } from './structured-agent-session-send-settlement' -import type { StructuredAgentSessionStatusSubscriber } from './structured-agent-session-status-feed' import { createStructuredAgentSessionHostStatusFeed } from './structured-agent-session-host-status-feed' +import type { + StructuredAgentSessionStatusFeed, + StructuredAgentSessionStatusSubscriber +} from './structured-agent-session-status-feed' import { StructuredAgentSessionTurnCompletionFeed, type StructuredAgentSessionTurnCompletionSubscriber @@ -119,6 +122,10 @@ export class StructuredAgentSessionClientDelivery { this.turnCompletionFeed.observe(sessionId, undefined, { historical: true }) } + /** A row from the state stored beside a journal nobody has opened. */ + seedStatus: StructuredAgentSessionStatusFeed['seed'] = (record, stored) => + this.statusFeed.seed(record, stored) + subscribeStatus = (subscriber: StructuredAgentSessionStatusSubscriber): (() => void) => this.statusFeed.subscribe(subscriber) forgetStatus = (sessionId: string): void => this.statusFeed.forget(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts index a027d9b1caf7..af9c55a061a5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts @@ -119,9 +119,9 @@ async function runningTurn(): Promise { } /** What the settle wrote, read back from the journal the next reader opens. */ -async function settledTurn() { - await host.restoreReadableSessions([SESSION]) - const { items } = await host.journalSnapshot(SESSION) +async function settledTurn(reader = host) { + await reader.restoreReadableSessions([SESSION]) + const { items } = await reader.journalSnapshot(SESSION) const turn = items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) const [settled] = [...selectStructuredAgentSettledTurns(items).values()] const label = settled && describeNativeChatTurnStatus({ elapsedSeconds: 0, ...settled }).key @@ -132,6 +132,27 @@ async function settledTurn() { return { turn, settled, label, notices } } +/** What the next launch reads: a host that has begun quitting opens no chat. */ +async function settledTurnAfterQuit() { + const state = hostTestState() + const reader = new StructuredAgentSessionHost({ + agents: NO_STRUCTURED_AGENTS, + logger: createStructuredAgentSessionLogger(), + store: state.store, + adapter: adapter(), + journalDatabase: openTestJournalHostDatabase(state.root), + recoveryCapsule: new AgentSessionRecoveryCapsule(state.root), + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + now: () => HOST_TEST_NOW + }) + try { + return await settledTurn(reader) + } finally { + await reader.flushAllStreamedEvents() + } +} + /** A turn nobody stopped reads like a finished one, with exactly one row saying it stopped. */ const ONE_NOTICE = [ 'Codex stopped while this response was in progress. You can continue in this conversation.' @@ -260,7 +281,7 @@ describe('a turn cut short by closing its provider', () => { expect(statuses.findLast((event) => event.type === 'status')).toMatchObject({ session: { status: 'idle', turnOutcome: 'interruption' } }) - const { label, notices } = await settledTurn() + const { label, notices } = await settledTurnAfterQuit() expect(label).toBe('workedFor') expect(notices).toEqual(ONE_NOTICE) }) @@ -319,7 +340,7 @@ describe('a turn cut short by closing its provider', () => { await host.flushAllStreamedEvents({ trigger: 'quit' }) - const { turn, notices } = await settledTurn() + const { turn, notices } = await settledTurnAfterQuit() expect(turn).toMatchObject({ state: 'interrupted' }) expect(turn).not.toHaveProperty('outcome') expect(notices).toEqual(ONE_NOTICE) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-command-entry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-command-entry.ts new file mode 100644 index 000000000000..388c9aae94ce --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-command-entry.ts @@ -0,0 +1,23 @@ +// The one way a chat command reaches a chat: host startup's settle first, then the chat's lock. +// The settle takes chats' locks too, so the wait comes before the lock, never under it. Nothing +// that holds a chat's lock calls this; /clear's attach of its replacement, under the source +// chat's lock, is the one call that does, and it can, because its own command already passed the +// wait and the wait never closes again. + +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' + +type CommandEntryContext = { + deps: Pick + serialize: (sessionId: string, task: () => Promise) => Promise +} + +/** Queued on the lock at once when startup is done, so commands keep the order they arrived in. */ +export function serializeStructuredAgentSessionCommand( + context: CommandEntryContext, + sessionId: string, + task: () => Promise +): Promise { + const startup = context.deps.commandsReady?.() + const run = () => context.serialize(sessionId, task) + return startup ? startup.then(run) : run() +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index 28393709a641..c2102eecdd38 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -24,6 +24,7 @@ import { import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { StructuredAgentSessionIdleSweep } from './structured-agent-session-idle-sweep' import { AGENT_SESSION_NOT_ATTACHED } from './structured-agent-session-mutation-admission' +import { sessionTabListed } from './structured-agent-session-host-tabs' import { deferredStructuredAgentSessionLogger } from './structured-agent-session-logger' export type StructuredAgentSessionConversationLifetime = ReturnType< @@ -65,11 +66,8 @@ export function createStructuredAgentSessionConversationLifetime(host: { return closeStructuredAgentSessionConversationUnderSerialize( { sessions, - closeStatus: (id) => { - const tabs = deps().store.getVisibleSessionTabIndex() - // A legacy store cannot say, so the row stays; restart is the boundary that forgets. - host.closeStatus(id, { listed: !tabs.present || tabs.sessionIds.includes(id) }) - } + // A legacy store cannot say, so the row stays; restart is the boundary that forgets. + closeStatus: (id) => host.closeStatus(id, { listed: sessionTabListed(deps().store, id) }) }, sessionId ) @@ -104,6 +102,7 @@ export function createStructuredAgentSessionConversationLifetime(host: { return { idleSweep, stopAgent, + isDisposed: (): boolean => disposed, /** Quit has begun: nothing opens a conversation or sweeps one after this. */ dispose: (): void => { disposed = true @@ -116,6 +115,13 @@ export function createStructuredAgentSessionConversationLifetime(host: { * drop it after. */ conversation: async (sessionId: string): Promise => { + // Reads (history, subscribe, snapshot, reveal) wait for startup's settle here, as commands do + // at `serializeStructuredAgentSessionCommand`: opening a crash-left chat would settle it ahead + // of startup's lease resolution. Before this chat's lock; no caller holds one here. + const startup = deps().commandsReady?.() + if (startup) { + await startup + } const open = sessions.get(sessionId) if (open) { // Restore left its per-chat file uncopied; a reader gets the chat from the one database. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts index 9c556916a39b..5bc186f28a2f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -1,18 +1,17 @@ // The one way a conversation's journal becomes open on this host: for a send, for a reader, and // for an attach that finds none open. // -// A damaged journal fails the open, which refuses it as unloadable. It marks what an earlier host -// process handed over and left unanswered as in doubt, and settles what it left running — the -// crash boundary. That -// needs no lease: provider history decides such a row later, under a won lease, in the attach. A -// row an earlier process accepted and never handed over (it quit or crashed first) is settled here -// too, before any reader, command or child sees it: a person's message is kept as a held card, the -// rest rejected (`journal-unsent-send-hold.ts`). Nothing here starts a provider child. +// A damaged journal fails the open, which refuses it as unloadable. It appends the chat's open +// settlement plan (structured-agent-session-open-settlement.ts): what an earlier host process +// handed over and left unanswered becomes doubt, what it accepted and never handed over is kept as +// a held card when it is a person's message and rejected otherwise (`journal-unsent-send-hold.ts`), +// and what it left running is settled — the crash boundary. That needs no lease: provider history +// decides a doubtful row later, under a won lease, in the attach. Nothing here starts a provider +// child. The plan's appends keep the chat's stored state current, and the open re-derives it when +// something else (an older build, an import) left it behind. import type { JournalHostDatabase } from '../agent-session-journal/journal-host-database' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' -import { holdUnsentSends } from '../agent-session-journal/journal-unsent-send-hold' import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { @@ -21,9 +20,13 @@ import { type AgentSessionAttachParams } from './structured-agent-session-attach' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' -import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' +import { + appendGoneGenerationSettlement, + appendOpenSettlement, + planOpenSettlement, + type OpenSettlementRecordFacts +} from './structured-agent-session-open-settlement' import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' -import { structuredAgentSessionHostInstance } from './structured-agent-session-queued-pause' import type { StructuredAgentSessionHostDeps, StructuredAgentSessionHostSession @@ -79,7 +82,8 @@ export async function openStructuredAgentSessionConversation( /** The open itself, indexed by nobody yet: the caller adopts the result. */ export async function openStructuredAgentSessionConversationJournal( - deps: Omit, + deps: Omit & + Partial>, record: AgentSessionRecord, options: StructuredAgentSessionConversationOpenOptions = {} ): Promise { @@ -93,40 +97,24 @@ export async function openStructuredAgentSessionConversationJournal( const journal = await openAgentSessionJournal({ identity, database: deps.journalDatabase, - deferPerSessionImport: options.deferPerSessionImport + deferPerSessionImport: options.deferPerSessionImport, + // The fence the status feed reads, which a child's end moves after this open. + currentFence: () => deps.store?.getRecord(sessionId)?.lease.runtimeFence ?? fence }) - try { - // A handed-over row found here is only doubt, which provider history decides under a won lease. - await journal.markPendingSubmissionsUnknown(fence) - } catch (error) { - deps.logger.warn('marking pending sends unknown on open failed', { - scope: 'open-pending-unknown', - sessionId, - error - }) - } - try { - // Before a Stop this open serves can withdraw one: a Stop never withdraws a card. - await holdUnsentSends(journal, { - fence, - hostInstance: structuredAgentSessionHostInstance(), - hold: { cause: 'hostRestarted' } - }) - } catch (error) { - // The row stays queued. The delivery loop's first step tries again before it hands anything - // over; if that fails too, the loop fails and rejects every queued send. - deps.logger.warn('settling sends an earlier process left queued failed on open', { - scope: 'open-leftover-sends', + // No child in this process writes to a journal nobody had open, so whatever it shows running + // belongs to a generation that is gone, whatever the lease still claims. Settled before any + // reader or child sees it; an acquisition settles against the generation it takes instead. + const plan = planOpenSettlement(journal, openSettlementRecordFacts(record), { + acquisition: options.acquisition + }) + await appendOpenSettlement(journal, plan, fence, (error) => + deps.logger.warn("settling a gone agent's work on open failed", { + scope: 'open-dead-generation', sessionId, error }) - } - // No child in this process writes to a journal nobody had open, so whatever it shows running - // belongs to a generation that is gone, whatever the lease still claims. Settled before any - // reader or child sees it. - if (!options.acquisition) { - await settleGoneGeneration(deps, record, journal) - } + ) + journal.sessionStatus.backfill() return { session: { journal, params, child: null } } } @@ -141,35 +129,37 @@ export async function resettleOpenStructuredAgentSessionConversation( session: StructuredAgentSessionHostSession | undefined ): Promise { const record = deps.store.getRecord(sessionId) - if (session && record?.lease.deathEvidence) { - await settleGoneGeneration(deps, record, session.journal) + if (!session || !record?.lease.deathEvidence) { + return } -} - -async function settleGoneGeneration( - deps: Pick, - record: AgentSessionRecord, - journal: AgentSessionJournal -): Promise { + const { goneGeneration } = planOpenSettlement(session.journal, openSettlementRecordFacts(record)) try { - await settleStaleStructuredAgentSessionState({ - journal, - sessionId: record.sessionId, - fence: record.lease.runtimeFence, - acquisitionGeneration: null, - deathEvidence: record.lease.deathEvidence ?? null, - failureTextContext: structuredAgentSessionFailureWordsContext(record) - }) + if (goneGeneration && goneGeneration.mutations.length > 0) { + await appendGoneGenerationSettlement( + session.journal, + goneGeneration, + record.lease.runtimeFence + ) + } } catch (error) { // Best effort: the next open or acquire re-derives it. deps.logger.warn("settling a gone agent's work on open failed", { scope: 'open-dead-generation', - sessionId: record.sessionId, + sessionId, error }) } } +export function openSettlementRecordFacts(record: AgentSessionRecord): OpenSettlementRecordFacts { + return { + sessionId: record.sessionId, + fence: record.lease.runtimeFence, + deathEvidence: record.lease.deathEvidence ?? null, + failureTextContext: structuredAgentSessionFailureWordsContext(record) + } +} + export function attachParamsForRecord( record: AgentSessionRecord, input: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index ce11c2dc34c9..8ce346e31b4b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -7,7 +7,6 @@ import { agentJournalItemKey, parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' -import { STALE_SESSION_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { AGENT_JOURNAL_THREAD_SCOPE, @@ -33,23 +32,18 @@ import { } from './structured-agent-session-start-failure-row' import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import { - endedByPersonsStop, - provenUnverifiableTurnRevisions, - provenUnverifiedToolCallRevisions, runningTurnLifecycleRevisions, - stopFoundTurnLiveAt, - turnVerdictFromDeathEvidence, watchedExitRevisions, type StructuredAgentSessionTurnVerdict, type StructuredAgentSessionWatchedExit } from './structured-agent-session-stale-turn-verdict' +import { exitedRootTurnScope } from './structured-agent-session-exit-turn-scope' import { - exitedRootTurnScope, - runningRootTurnScope -} from './structured-agent-session-exit-turn-scope' + appendGoneGenerationSettlement, + planGoneGenerationSettlement +} from './structured-agent-session-open-settlement' import { hasUnfinishedStructuredAgentSessionWork, - isInProgressStructuredAgentSessionItem, type DeadGenerationJournal } from './structured-agent-session-unfinished-work' import { withdrawCodexSendsNoTurnOpenedFor } from './structured-agent-session-unopened-send-withdrawal' @@ -173,12 +167,9 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { } /** - * Settles whatever a generation with no child in this process left running: found when a new child - * is acquired, or when a chat is reopened for reading. Derived from the journal and the lease's - * death evidence each time, so nothing is owed in between. Proven death ends the turn interrupted, - * and a proof written after an earlier settle revises what that settle left `unverifiable`, the - * turn and the calls it closed alike. Must run before a new child's buffered events land, or a - * live turn would be judged. + * Settles whatever a generation with no child in this process left running, against the + * generation an acquisition just took (see `planGoneGenerationSettlement`, the open's own rule). + * Must run before a new child's buffered events land, or a live turn would be judged. */ export async function settleStaleStructuredAgentSessionState(input: { journal: AgentSessionJournal @@ -190,79 +181,18 @@ export async function settleStaleStructuredAgentSessionState(input: { failureTextContext?: AgentSessionFailureWordsContext }): Promise { const { journal } = input - const items = journal.snapshot().items - // Each turn is judged by the evidence only if it names that turn's owner. - const verdictFor = (item: AgentJournalRenderItem) => - turnVerdictFromDeathEvidence( - input.deathEvidence, - journal.itemFence(item.itemId), - stopFoundTurnLiveAt(journal, item) - ) - // Per attempt: a retry re-partitions only what is left, and a reused chunk id would skip it. - const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` - const settlementId = `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:${input.fence}:${generation}` - // Calls an earlier settle closed with no proof, revised once a proof names their owner. - const mutations = provenUnverifiedToolCallRevisions(items, input.deathEvidence, journal) - for (const item of items) { - const identity = parseAgentJournalItemKey(item.itemId) - // A turn already settled (a person's Stop) ends its calls as it ended; only a turn still running - // leaves them to the evidence. - const end = runningCallEnd(item.turnScope, (id) => journal.itemBody(id), verdictFor(item).state) - const body = endedUnseenMessageBody(item.body) ?? terminalAgentJournalBody(item.body, end) - if (identity && body) { - mutations.push({ - kind: 'item', - identity, - body, - turnScope: item.turnScope ?? AGENT_JOURNAL_THREAD_SCOPE - }) - } - } - const proven = provenUnverifiableTurnRevisions(items, input.deathEvidence, journal) - const turnEnds = [ - ...items.flatMap((item) => runningTurnLifecycleRevisions([item], verdictFor(item))), - ...proven - ] - mutations.push(...turnEnds) - const evidence = input.deathEvidence - if ( - evidence && - (proven.length > 0 || - items.some( - (item) => - isInProgressStructuredAgentSessionItem(item) && verdictFor(item).state === 'interrupted' - )) && - !endedByPersonsStop(journal, turnEnds) - ) { - mutations.unshift({ - kind: 'item', - // Named by the death it explains, so a retry after a partly written settle adds no second row. - identity: { - provider: 'orca', - clientMessageId: `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:death-${evidence.ownerFence ?? 'unowned'}-${evidence.observedAt}` - }, - // The death evidence is Orca's log text, never a sentence for a person: the row says only - // that the provider stopped. - body: { - kind: 'status', - ...agentSessionFailureWords(agentSessionFailureFact('providerExited'), { - ...input.failureTextContext, - surface: 'row' - }), - tone: 'error' - }, - turnScope: runningRootTurnScope(items) - }) - } - for (const chunk of partitionJournalLifecycleMutations(settlementId, mutations)) { - await journal.appendLifecycleBatch({ - settlementId: chunk.settlementId, - fence: input.fence, - recovered: true, - mutations: chunk.mutations - }) - } - return mutations.length + const settlement = planGoneGenerationSettlement({ + items: journal.snapshot().items, + journal, + sessionId: input.sessionId, + fence: input.fence, + // Per attempt: a retry re-partitions only what is left, and a reused chunk id would skip it. + generation: input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}`, + deathEvidence: input.deathEvidence, + failureTextContext: input.failureTextContext + }) + await appendGoneGenerationSettlement(journal, settlement, input.fence) + return settlement.mutations.length } function withRevisions( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts index 2fc2bc28d735..e8675d1e68f3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts @@ -38,62 +38,64 @@ export function setStructuredAgentSessionTabVisibility( return host.deps.store.setSessionTabVisibility(sessionId, visible, tabId) } +/** Whether the chat still has its tab. A legacy store with no tab index cannot say, so yes. */ +export function sessionTabListed( + store: { getVisibleSessionTabIndex: () => { present: boolean; sessionIds: string[] } }, + sessionId: string +): boolean { + const tabs = store.getVisibleSessionTabIndex() + return !tabs.present || tabs.sessionIds.includes(sessionId) +} + export type StructuredAgentSessionTab = { sessionId: string workspaceId: string agent: AgentSessionRecord['provider'] } -export function listStructuredAgentSessionTabs( - sessions: ReadonlyMap< - string, - { params: { location: { workspaceId: string }; provider: AgentSessionRecord['provider'] } } - > +/** Tabs from durable state alone: each requested id, once, in the order given, that has a record. + * Reading a chat needs no adapter, so a workspace this host cannot start an agent in still lists + * its tabs. Opens nothing: a chat whose history is unreadable keeps its tab and its read + * says why; one whose history is missing keeps its tab and reads empty. */ +export function listPersistedSessionTabs( + deps: { + store: { getRecord: (sessionId: string) => AgentSessionRecord | null } + }, + sessionIds: readonly string[] ): StructuredAgentSessionTab[] { - return [...sessions.entries()].map(([sessionId, session]) => ({ - sessionId, - workspaceId: session.params.location.workspaceId, - agent: session.params.provider - })) + const tabs = new Map() + for (const sessionId of sessionIds) { + const record = deps.store.getRecord(sessionId) + if (!record || tabs.has(sessionId)) { + continue + } + tabs.set(sessionId, { + sessionId, + workspaceId: record.location.workspaceId, + agent: record.provider + }) + } + return [...tabs.values()] } -type TabSessions = ReadonlyMap< - string, - { - child?: unknown - params: { location: { workspaceId: string }; provider: AgentSessionRecord['provider'] } - } -> +type TabSessions = ReadonlyMap /** The host's chat-tab surface; reads `host.deps` per call, so it sees the host's wrapped deps. */ export function createStructuredAgentSessionTabSurface( host: Parameters[0] & { - deps: { + deps: Parameters[0] & { store: Pick< AgentSessionRecordStore, - 'getVisibleSessionTabIndex' | 'getSessionTabId' | 'showSessionTabs' | 'getRecord' + 'getVisibleSessionTabIndex' | 'getSessionTabId' | 'showSessionTabs' > } }, sessions: TabSessions, forgetStatus: (sessionId: string) => void ) { - // Restored chats that could not be opened. Each keeps its tab, whose read says why. - const unopened = new Set() return { - listSessionTabs: (): StructuredAgentSessionTab[] => [ - ...listStructuredAgentSessionTabs(sessions), - ...[...unopened].flatMap((sessionId) => { - const record = sessions.has(sessionId) ? null : host.deps.store.getRecord(sessionId) - return record - ? [{ sessionId, workspaceId: record.location.workspaceId, agent: record.provider }] - : [] - }) - ], - /** A restore target with a record whose open failed. */ - markUnopened: (sessionId: string): void => { - unopened.add(sessionId) - }, + /** From the record store and the given tab ids; opens no conversation. */ + listSessionTabs: (ids: readonly string[]) => listPersistedSessionTabs(host.deps, ids), getPersistedVisibleSessionTabIndex: () => host.deps.store.getVisibleSessionTabIndex(), getSessionTabId: (sessionId: string): string | null => host.deps.store.getSessionTabId(sessionId), @@ -107,11 +109,8 @@ export function createStructuredAgentSessionTabSurface( options?: { deferHiddenNotice?: boolean } ): Promise => { await setStructuredAgentSessionTabVisibility(host, sessionId, visible, tabId) - if (!visible) { - unopened.delete(sessionId) - if (!options?.deferHiddenNotice) { - notifyTabHidden(host, sessionId) - } + if (!visible && !options?.deferHiddenNotice) { + notifyTabHidden(host, sessionId) } // The tab edge of the row's lifetime; the handle close is the other. if (!visible && !sessions.get(sessionId)?.child) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index 2230efa15c9b..9957b437fbc3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -141,6 +141,8 @@ export type StructuredAgentSessionHostDeps = { now?: () => number /** The idle sweep's period and window. Tests drive these; production takes the defaults. */ idleSweep?: { intervalMs?: number; idleMs?: number } + /** How long startup waits on each chat's recovery. Tests shorten it; production takes the default. */ + startupRecoveryBudgetMs?: number /** Whether an orchestration dispatch still owns this session's worker; absent answers no. */ hasOpenDispatch?: (record: AgentSessionRecord) => boolean /** A chat tab left the screen: closed, or its workspace removed. Advisory; a throw is logged. */ @@ -148,6 +150,9 @@ export type StructuredAgentSessionHostDeps = { /** Where every failure the host carries on past is reported. Required: a host without one would * drop exactly the failures nobody sees in the UI. */ logger: StructuredAgentSessionLogger + /** Pending while host startup settles the chats a gone process left with work: every chat + * command waits for it, and none is refused. Null or absent once open. */ + commandsReady?: () => Promise | null /** Every status projection this host publishes. `replay` marks a re-projection of state the host * already knew (restore, an arriving subscriber) rather than a fresh journal edge. */ onSessionStatusChanged?: ( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts index e92b33aac1c1..c3f76162979f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts @@ -618,7 +618,7 @@ describe('restart', () => { const restoreReads = listRecords.mock.calls.length await host.restoreReadableSessions() - expect(host.listSessionTabs()).toEqual([ + expect(host.listSessionTabs([SESSION])).toEqual([ { sessionId: SESSION, workspaceId: 'workspace-1', agent: 'codex' } ]) const history = await host.history({ sessionId: SESSION, direction: 'tail' }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 835f9f9e204f..10e3637a8896 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -85,7 +85,9 @@ export class StructuredAgentSessionHost { private readonly tasks = new StructuredAgentSessionTaskQueue() private readonly runtimeState: StructuredAgentSessionHostRuntimeState private readonly reconcileLeases: ReturnType - private readonly restore: ReturnType + /** Startup from each chat's stored state (`structured-agent-session-startup-state`), and the + * history restore after the listing. */ + readonly startup: ReturnType private readonly lifetime: StructuredAgentSessionConversationLifetime private readonly conversationDelivery: conversation.StructuredAgentSessionConversationDelivery private readonly eventRecovery: StructuredAgentSessionEventRecovery @@ -123,15 +125,16 @@ export class StructuredAgentSessionHost { agentStart.ensureStructuredAgentSessionAgent(this.attachContext(), sessionId, startedFor), clientDelivery: this.clientDelivery }) - this.restore = reveal.createStructuredAgentSessionHostRestore(deps, { + this.startup = reveal.createStructuredAgentSessionHostRestore(deps, { reconcileLeases: this.reconcileLeases, resolveRecovery: (sessionId) => this.runtimeState.resolveRecovery(sessionId), serialize: (sessionId, task) => this.serialize(sessionId, task), hasSession: this.hasSession, + isDisposed: () => this.lifetime.isDisposed(), // Site 10: cannot overwrite a live entry — the restorer returns early on // `hasSession` inside the same serialized step as this `set`. onReadable: this.conversationDelivery.adoptOpened, - onUnopened: (sessionId) => this.tabs.markUnopened(sessionId) + seedStatus: this.clientDelivery.seedStatus }) this.eventRecovery = new StructuredAgentSessionEventRecovery({ deps, @@ -235,10 +238,9 @@ export class StructuredAgentSessionHost { holdsSessions = (): boolean => this.deps.store.holdsRecords() || this.legacyRecordImportOwed() onSessionsHeld = (listener: () => void): (() => void) => this.deps.store.onFirstRecord(listener) - reconcileRestartLeases = (): Promise => this.restore.reconcileRestartLeases() + reconcileRestartLeases = (): Promise => this.startup.reconcileRestartLeases() - restoreReadableSessions = (sessionIds?: readonly string[]): Promise => - this.restore.restoreReadableSessions(sessionIds) + restoreReadableSessions = (ids?: readonly string[]) => this.startup.restoreReadableSessions(ids) /** Make one persisted session addressable again; see `structured-agent-session-reveal`. */ revealSession = (sessionId: string): Promise => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts index 099b356cb5e3..17e19fbaa37c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts @@ -1,5 +1,6 @@ // The context every client mutation of a session runs with, and the one path each takes: admit the -// envelope against the lease, then run its plan inside the session's serialize. +// envelope against the lease, then run its plan inside the session's serialize, entered through +// the one chat-command entry (startup's settle first). import type { AgentSessionMutationEnvelope, @@ -12,6 +13,7 @@ import { type AgentSessionMutationSessionPreparation } from './structured-agent-session-mutation-admission' import type { MutationPlan } from './structured-agent-session-mutation-plans' +import { serializeStructuredAgentSessionCommand } from './structured-agent-session-command-entry' import type { StructuredAgentSessionStopEnding } from './structured-agent-session-host-lifetime' import type { StructuredAgentSessionAcquireAborts } from './structured-agent-session-acquire-aborts' import type { @@ -59,7 +61,7 @@ export function mutateStructuredAgentSession( plan: MutationPlan, prepareSession?: AgentSessionMutationRequest['prepareSession'] ): Promise> { - return context.serialize(envelope.sessionId, () => + return serializeStructuredAgentSessionCommand(context, envelope.sessionId, () => admitAndRunAgentSessionMutation({ store: context.deps.store, adapter: context.deps.adapter, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts new file mode 100644 index 000000000000..f3c080209aa6 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts @@ -0,0 +1,233 @@ +// A chat's stored status and its settlement plan are two readings of one set of rules: startup +// selects a chat exactly when its settle would write something, the settle writes exactly the plan, +// and once the plan commits the status reads settled, so nothing is selected twice. Death evidence +// changes only which verdict a running turn gets, never whether the chat is selected. + +import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + AGENT_SESSION_JOURNAL_SCHEMA_VERSION, + type AgentJournalItemBody, + type AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import { + createTrackedJournalOpener, + insertTestJournalRow, + liveTestJournalRows, + openTestJournalHostDatabase, + readTestJournalSessionStatus +} from '../agent-session-journal/journal-host-database-test-support' +import { isUnsettledJournalSessionStatus } from '../agent-session-journal/journal-session-state' +import { + CORPUS_DEATH_EVIDENCE, + CORPUS_FENCE, + CORPUS_UNSETTLED, + JOURNAL_SESSION_STATE_CASES, + JOURNAL_SESSION_STATE_CORPUS +} from '../agent-session-journal/journal-session-state-test-corpus' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + appendOpenSettlement, + planOpenSettlement, + type OpenSettlementPlan, + type OpenSettlementRecordFacts +} from './structured-agent-session-open-settlement' + +function openSettlementPlanIsEmpty(plan: OpenSettlementPlan): boolean { + return ( + plan.recoveredDispatches.length === 0 && + plan.leftoverQueued.length === 0 && + (plan.goneGeneration?.mutations.length ?? 0) === 0 && + plan.rosters.length === 0 + ) +} + +const journals = createTrackedJournalOpener() +let root: string +let clock = 1_000 +let chats = 0 + +function open(sessionId: string): Promise { + const identity: AgentSessionJournalIdentity = { + sessionId, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: codexProviderHandle(`thread-${sessionId}`) + } + return journals.open({ + identity, + stateDirectory: root, + now: () => (clock += 1), + mintEpoch: () => `epoch-${sessionId}` + }) +} + +function storedStatus(sessionId: string) { + const row = readTestJournalSessionStatus(root, sessionId) + if (!row) { + throw new Error(`no stored status for ${sessionId}`) + } + return row +} + +const selected = (sessionId: string) => isUnsettledJournalSessionStatus(storedStatus(sessionId)) + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-open-settlement-')) + clock = 1_000 +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +const COMBINATIONS = JOURNAL_SESSION_STATE_CASES.flatMap((name) => + Object.keys(CORPUS_DEATH_EVIDENCE).map((evidence) => [name, evidence] as const) +) + +describe('the stored status and the plan agree (T4)', () => { + it.each(COMBINATIONS)('%s, death evidence %s', async (name, evidenceName) => { + const sessionId = `chat-${(chats += 1)}` + const writer = await open(sessionId) + await JOURNAL_SESSION_STATE_CORPUS[name](writer) + await writer.close() + // The open after the process that wrote it is gone. + const journal = await open(sessionId) + const deathEvidence = CORPUS_DEATH_EVIDENCE[evidenceName] ?? null + const record: OpenSettlementRecordFacts = { sessionId, fence: CORPUS_FENCE, deathEvidence } + + // Pinned per case, so agreement between the status and the plan cannot hide both being wrong. + expect(selected(sessionId)).toBe(CORPUS_UNSETTLED[name]) + const plan = planOpenSettlement(journal, record) + // Evidence naming an `unverifiable` turn's writer lets an open revise it; startup never selects + // a chat for that (main's acquisition path does it when the chat is next used). + const revisesVerdictOnly = + name === 'unverifiable turn' && deathEvidence?.ownerFence === CORPUS_FENCE + expect(selected(sessionId)).toBe(!openSettlementPlanIsEmpty(plan) && !revisesVerdictOnly) + if (name === 'queued leftover') { + // A queued leftover is owed with no record to read, too. + expect(planOpenSettlement(journal, null).leftoverQueued).toEqual(['send-queued']) + } + + // The open appends exactly the plan: a row per roster, per recovered send and per rejected + // leftover, and one lifecycle batch for what the gone generation left. + const before = liveTestJournalRows(openTestJournalHostDatabase(root).db, sessionId).length + await appendOpenSettlement(journal, plan, CORPUS_FENCE, (error) => { + throw error + }) + const appended = + liveTestJournalRows(openTestJournalHostDatabase(root).db, sessionId).length - before + expect(appended).toBe( + plan.rosters.length + + plan.recoveredDispatches.length + + plan.leftoverQueued.length + + ((plan.goneGeneration?.mutations.length ?? 0) > 0 ? 1 : 0) + ) + + // Every entry revised its entity out of the state that selected it (T4b, T15b). + for (const clientMessageId of plan.leftoverQueued) { + expect(journal.submission(clientMessageId)).toMatchObject({ dispatchState: 'rejected' }) + } + expect(selected(sessionId)).toBe(false) + expect(openSettlementPlanIsEmpty(planOpenSettlement(journal, record))).toBe(true) + }) + + it("ends a turn a person's Stop found as theirs, with no row saying the provider stopped", async () => { + const providerStoppedRows = (plan: ReturnType) => + (plan.goneGeneration?.mutations ?? []).filter( + (mutation) => + mutation.kind === 'item' && + mutation.identity.provider === 'orca' && + mutation.identity.clientMessageId.includes(':death-') + ) + const settle = async (evidenceName: string) => { + const sessionId = `chat-${(chats += 1)}` + const journal = await open(sessionId) + await JOURNAL_SESSION_STATE_CORPUS["running turn a person's Stop found"](journal) + const deathEvidence = CORPUS_DEATH_EVIDENCE[evidenceName] ?? null + const record: OpenSettlementRecordFacts = { sessionId, fence: CORPUS_FENCE, deathEvidence } + const plan = planOpenSettlement(journal, record) + await appendOpenSettlement(journal, plan, CORPUS_FENCE, (error) => { + throw error + }) + return { sessionId, journal, plan } + } + + // The process was found dead after the Stop: the Stop decides how the turn ends. + const after = await settle('names the writer, after a Stop') + expect(providerStoppedRows(after.plan)).toEqual([]) + expect(after.plan.goneGeneration?.mutations).toContainEqual( + expect.objectContaining({ + body: expect.objectContaining({ kind: 'turn', state: 'interrupted' }) + }) + ) + expect(selected(after.sessionId)).toBe(false) + expect(storedStatus(after.sessionId).summary).toEqual( + after.journal.sessionStatus.at(undefined).summary + ) + + // Found dead before the Stop event's time: the death explains the end, and says so. + const before = await settle('names the writer') + expect(providerStoppedRows(before.plan)).toHaveLength(1) + }) + + it('selects nothing for an item whose key will not parse, which no plan can revise (R1J-4)', async () => { + const first = await open('unkeyed') + await JOURNAL_SESSION_STATE_CORPUS.settled(first) + const tip = first.cursor() + await first.close() + const unkeyed = (seq: number, itemId: string, body: AgentJournalItemBody) => + insertTestJournalRow(openTestJournalHostDatabase(root).db, 'unkeyed', { + v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION, + kind: 'item', + itemId, + revision: 1, + body, + epoch: tip.epoch, + seq, + fence: CORPUS_FENCE, + ts: 5_000 + }) + unkeyed(tip.sequence + 1, 'legacy-tool-1', { + kind: 'tool-call', + name: 'shell', + input: { command: 'ls' }, + state: 'running' + }) + unkeyed(tip.sequence + 2, 'legacy-turn-2', { + kind: 'turn', + turnId: 't2', + state: 'running', + startedAt: 40 + }) + unkeyed(tip.sequence + 3, 'legacy-turn-3', { + kind: 'turn', + turnId: 't3', + state: 'unverifiable', + startedAt: 50 + }) + // Written beside the journal, as before the status table: the open writes the status back. + openTestJournalHostDatabase(root) + .db.prepare('DELETE FROM journal_session_state WHERE session_id = ?') + .run('unkeyed') + const journal = await open('unkeyed') + // As the chat's open does after its settle. + journal.sessionStatus.backfill() + expect(readTestJournalSessionStatus(root, 'unkeyed')).not.toBeNull() + const deathEvidence = CORPUS_DEATH_EVIDENCE['names the writer'] ?? null + const plan = planOpenSettlement(journal, { + sessionId: 'unkeyed', + fence: CORPUS_FENCE, + deathEvidence + }) + + expect(openSettlementPlanIsEmpty(plan)).toBe(true) + expect(storedStatus('unkeyed')).toMatchObject({ lifecycle: 'idle' }) + expect(selected('unkeyed')).toBe(false) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts new file mode 100644 index 000000000000..45e4ea20d7e9 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts @@ -0,0 +1,287 @@ +// Everything a non-acquisition open writes because an earlier host process is gone, as one plan. +// +// Built from the per-entity rules in journal-open-settlement-plan.ts, which each chat's stored +// status counts with too, so startup selects a chat when this plan has work beyond revising an +// earlier verdict. The open appends exactly this plan, and each entry revises its entity out of +// the state that selected it. + +import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { + agentSessionFailureWords, + type AgentSessionFailureWordsContext +} from '../../../shared/agent-session-failure-words' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalRenderItem +} from '../../../shared/agent-session-journal-types' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' +import { STALE_SESSION_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' +import { partitionJournalLifecycleMutations } from '../agent-session-journal/journal-lifecycle-batch-partition' +import { runningCallEnd } from '../agent-session-journal/journal-terminal-settlement' +import { holdUnsentSends } from '../agent-session-journal/journal-unsent-send-hold' +import { structuredAgentSessionHostInstance } from './structured-agent-session-queued-pause' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import { + isRunningJournalTurn, + openSettlementItemIdentity, + openSettlementTerminalBody, + owesRecoveredDispatch +} from '../agent-session-journal/journal-open-settlement-plan' +import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + staleSubagentRosterRevisions, + type JournalSubagentLivenessRevision +} from '../agent-session-journal/journal-subagent-liveness' +import { runningRootTurnScope } from './structured-agent-session-exit-turn-scope' +import { + endedByPersonsStop, + provenUnverifiableTurnRevisions, + provenUnverifiedToolCallRevisions, + runningTurnLifecycleRevisions, + stopFoundTurnLiveAt, + turnVerdictFromDeathEvidence +} from './structured-agent-session-stale-turn-verdict' + +/** What the record says about the gone process, or null for the journal-only reading. */ +export type OpenSettlementRecordFacts = { + sessionId: string + fence: number + deathEvidence: AgentSessionDeathEvidence | null + /** Who the exit row names. */ + failureTextContext?: AgentSessionFailureWordsContext +} + +export type GoneGenerationSettlement = { + settlementId: string + mutations: JournalLifecycleMutationInput[] +} + +export type OpenSettlementPlan = { + /** Handed-over sends nothing answered: each becomes a recovered `unknown`. */ + recoveredDispatches: string[] + /** Sends accepted and never handed over: each is rejected as the host having restarted. */ + leftoverQueued: string[] + /** What the gone generation left running. Null on an acquisition's own open, which settles it + * against the generation it acquires. */ + goneGeneration: GoneGenerationSettlement | null + /** Rosters and background tasks still claiming a live child. */ + rosters: JournalSubagentLivenessRevision[] +} + +export type OpenSettlementJournal = Pick< + AgentSessionJournal, + 'snapshot' | 'submissions' | 'itemFence' | 'cursor' | 'stopMarks' | 'itemBody' | 'wroteBeforeOpen' +> + +export type OpenSettlementOptions = { + acquisition?: boolean +} + +export function planOpenSettlement( + journal: OpenSettlementJournal, + record: OpenSettlementRecordFacts | null, + options: OpenSettlementOptions = {} +): OpenSettlementPlan { + const { items } = journal.snapshot() + const submissions = journal.submissions() + return { + recoveredDispatches: submissions + .filter(owesRecoveredDispatch) + .map((submission) => submission.clientMessageId), + // What `holdUnsentSends` settles: queued by a process before this open. + leftoverQueued: submissions + .filter( + (submission) => + isQueuedAgentJournalSubmission(submission) && + journal.wroteBeforeOpen(submission.acceptedSequence) + ) + .map((submission) => submission.clientMessageId), + goneGeneration: options.acquisition + ? null + : planGoneGenerationSettlement({ + items, + journal, + sessionId: record?.sessionId ?? '', + fence: record?.fence ?? 0, + // Per attempt: a retry re-partitions only what is left, and a reused chunk id would skip it. + generation: `seq-${journal.cursor().sequence}`, + deathEvidence: record?.deathEvidence ?? null, + failureTextContext: record?.failureTextContext + }), + rosters: staleSubagentRosterRevisions(items) + } +} + +/** + * Settles whatever a generation with no child in this process left running: found when a new child + * is acquired, or when a chat is reopened. Derived from the journal and the lease's death evidence + * each time, so nothing is owed in between. Proven death ends the turn interrupted, and a proof + * written after an earlier settle revises what that settle left `unverifiable`, the turn and the + * calls it closed alike. + */ +export function planGoneGenerationSettlement(input: { + items: readonly AgentJournalRenderItem[] + /** Who wrote each item, what the latest Stop event says about the turn it found, and each + * call's turn row. */ + journal: Pick + sessionId: string + fence: number + generation: string + deathEvidence: AgentSessionDeathEvidence | null + failureTextContext?: AgentSessionFailureWordsContext +}): GoneGenerationSettlement { + const { items } = input + // Each turn is judged by the evidence only if it names that turn's owner. + const verdictFor = (item: AgentJournalRenderItem) => + turnVerdictFromDeathEvidence( + input.deathEvidence, + input.journal.itemFence(item.itemId), + stopFoundTurnLiveAt(input.journal, item) + ) + // Calls an earlier settle closed with no proof, revised once a proof names their owner. + const mutations: JournalLifecycleMutationInput[] = provenUnverifiedToolCallRevisions( + items, + input.deathEvidence, + input.journal + ) + for (const item of items) { + const identity = openSettlementItemIdentity(item) + // A turn already settled (a person's Stop) ends its calls as it ended; only a turn still running + // leaves them to the evidence. + const end = runningCallEnd( + item.turnScope, + (id) => input.journal.itemBody(id), + verdictFor(item).state + ) + const body = openSettlementTerminalBody(item, end) + if (identity && body) { + mutations.push({ + kind: 'item', + identity, + body, + turnScope: item.turnScope ?? AGENT_JOURNAL_THREAD_SCOPE + }) + } + } + const proven = provenUnverifiableTurnRevisions(items, input.deathEvidence, input.journal) + const turnEnds = [ + ...items.flatMap((item) => runningTurnLifecycleRevisions([item], verdictFor(item))), + ...proven + ] + mutations.push(...turnEnds) + const evidence = input.deathEvidence + // A turn a person's Stop ended reads as theirs, with no row saying the provider stopped. + if ( + evidence && + (proven.length > 0 || + items.some((item) => isInProgressItem(item) && verdictFor(item).state === 'interrupted')) && + !endedByPersonsStop(input.journal, turnEnds) + ) { + mutations.unshift({ + kind: 'item', + // Named by the death it explains, so a retry after a partly written settle adds no second row. + identity: { + provider: 'orca', + clientMessageId: `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:death-${evidence.ownerFence ?? 'unowned'}-${evidence.observedAt}` + }, + // The death evidence is Orca's log text, never a sentence for a person: the row says only + // that the provider stopped. + body: { + kind: 'status', + ...agentSessionFailureWords(agentSessionFailureFact('providerExited'), { + ...input.failureTextContext, + surface: 'row' + }), + tone: 'error' + }, + turnScope: runningRootTurnScope(items) + }) + } + return { + settlementId: `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:${input.fence}:${input.generation}`, + mutations + } +} + +/** Appends a gone generation's settlement, in bounded chunks each retry-safe by its id. */ +export async function appendGoneGenerationSettlement( + journal: Pick, + settlement: GoneGenerationSettlement, + fence: number +): Promise { + for (const chunk of partitionJournalLifecycleMutations( + settlement.settlementId, + settlement.mutations + )) { + await journal.appendLifecycleBatch({ + settlementId: chunk.settlementId, + fence, + recovered: true, + mutations: chunk.mutations + }) + } +} + +/** The open's writes: exactly the plan, in today's row shapes. Each part is best effort on its own: + * what fails stays owed, and the chat's next open or the next startup plans it again. */ +export async function appendOpenSettlement( + journal: AgentSessionJournal, + plan: OpenSettlementPlan, + fence: number, + onError: (error: unknown) => void +): Promise { + try { + for (const roster of plan.rosters) { + // The conversation's row, revised by whichever writer is newest. + await journal.appendItem(roster.identity, roster.body, { + fence: journal.highestFence(), + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) + } + } catch (error) { + onError(error) + } + try { + // A handed-over row is only doubt, which provider history decides under a won lease. + if (plan.recoveredDispatches.length > 0) { + await journal.markPendingSubmissionsUnknown(fence) + } + } catch (error) { + onError(error) + } + // A queued row found here was accepted by a process that is gone: a person's message is kept as + // a held card, the rest rejected. Before a Stop this open serves can withdraw one. + if (plan.leftoverQueued.length > 0) { + try { + await holdUnsentSends(journal, { + fence, + hostInstance: structuredAgentSessionHostInstance(), + hold: { cause: 'hostRestarted' } + }) + } catch (error) { + // The row stays queued. The delivery loop's first step tries again before it hands anything + // over; if that fails too, the loop fails and rejects every queued send. + onError(error) + } + } + if (plan.goneGeneration && plan.goneGeneration.mutations.length > 0) { + try { + await appendGoneGenerationSettlement(journal, plan.goneGeneration, fence) + } catch (error) { + // Best effort: the next open or acquire re-derives it. + onError(error) + } + } +} + +/** Work that means the provider was MID-RESPONSE, and that this settle revises. A pending approval + * or question is the provider waiting on the user, so dying while one sits there interrupted + * nothing. */ +function isInProgressItem(item: AgentJournalRenderItem): boolean { + return ( + (isRunningJournalTurn(item) || + (item.body.kind === 'tool-call' && item.body.state === 'running')) && + openSettlementItemIdentity(item) !== null + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts index 48fb03b44c13..3e6548ccc39c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts @@ -20,6 +20,7 @@ import type { StructuredAgentSessionHostDeps } from './structured-agent-session- import { structuredAgentSessionOptionModels } from './structured-agent-session-option-models' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' import type { StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' +import { serializeStructuredAgentSessionCommand } from './structured-agent-session-command-entry' type RestingOptions = Pick @@ -111,7 +112,7 @@ export async function readStructuredAgentSessionOptions( sessionId: string ): Promise { const { adapter, agents, store } = context.deps - const started = await context.serialize(sessionId, async () => { + const started = await serializeStructuredAgentSessionCommand(context, sessionId, async () => { const session = await context.openConversation(sessionId).catch((error: unknown) => { throw journalOpenReadRefusal(error, context.deps.logger, sessionId) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts new file mode 100644 index 000000000000..1555ad0ad724 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts @@ -0,0 +1,130 @@ +// A restarted host lists a chat's tab from its record and the tab table, never from whether its +// history opened: a chat with no history on disk, and one whose history cannot be read, both keep +// their tab. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { isAgentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { + createRestTestRig, + restTestChat, + sendRestTestMessage, + type RestTestRig +} from './structured-agent-session-rest-test-rig' +import { restTestOpens } from './structured-agent-session-rest-test-observations' + +let rig: RestTestRig + +beforeEach(async () => { + rig = await createRestTestRig() +}) + +afterEach(async () => { + await rig.dispose() + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +const tab = (sessionId: string, workspaceId = 'workspace-1') => ({ + sessionId, + workspaceId, + agent: 'codex' +}) + +describe('the tab list after a restart', () => { + it('lists every chat it is given without opening one', async () => { + await restTestChat(rig, 'session-a', { message: 'one' }) + await restTestChat(rig, 'session-b', { message: 'two' }) + await rig.crash() + const host = await rig.boot() + + expect(host.listSessionTabs(['session-a', 'session-b'])).toEqual([ + tab('session-a'), + tab('session-b') + ]) + expect(rig.journalOpens).not.toHaveBeenCalled() + }) + + it('keeps a chat with no history on disk, which then reads empty and takes a send (T3)', async () => { + // Its first start failed before the conversation opened, so no history was ever written. + rig.adapter.acquire.mockRejectedValueOnce(new Error('spawn failed')) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + await expect(restTestChat(rig, 'session-new')).rejects.toThrow(/attach refused/) + await rig.store.setSessionTabVisibility('session-new', true) + await restTestChat(rig, 'session-old', { message: 'hello' }) + await rig.crash() + const host = await rig.boot() + const ids = ['session-new', 'session-old'] + + await host.restoreReadableSessions(ids) + + expect(host.listSessionTabs(ids)).toEqual([tab('session-new'), tab('session-old')]) + // The restore founds no history for it; the tab stands anyway. + expect(restTestOpens(rig, 'session-new')).toBe(0) + const page = await host.history({ sessionId: 'session-new', direction: 'tail' }) + expect(page.ok && page.page.items).toEqual([]) + const sent = await sendRestTestMessage(rig, 'session-new', 'first words') + expect(sent).toMatchObject({ ok: true }) + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledOnce(), { timeout: 10_000 }) + const after = await host.history({ sessionId: 'session-new', direction: 'tail' }) + expect(JSON.stringify(after.ok && after.page)).toContain('first words') + }) + + it('keeps a chat whose history cannot be opened, and its read says why (T4)', async () => { + await restTestChat(rig, 'session-bad', { message: 'lost' }) + await restTestChat(rig, 'session-good', { message: 'kept' }) + await rig.crash() + const host = await rig.boot() + rig.journalOpens.mockImplementation(async (sessionId) => { + if (sessionId === 'session-bad') { + throw new Error('EACCES: permission denied') + } + }) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const ids = ['session-bad', 'session-good'] + + await host.restoreReadableSessions(ids).catch(() => undefined) + + expect(host.listSessionTabs(ids)).toEqual([tab('session-bad'), tab('session-good')]) + const read = await host.history({ sessionId: 'session-bad', direction: 'tail' }).then( + () => null, + (error: unknown) => error + ) + expect(isAgentSessionRefusalError(read) && read.refusal.code).toBe( + 'agent_session_journal_unreadable' + ) + }) + + it('lists in the order it is given, once each, whatever order the records are in (T7)', async () => { + await restTestChat(rig, 'session-c') + await restTestChat(rig, 'session-a') + await restTestChat(rig, 'session-b') + await rig.crash() + const host = await rig.boot() + + expect(host.listSessionTabs(['session-b', 'session-c', 'session-a', 'session-b'])).toEqual([ + tab('session-b'), + tab('session-c'), + tab('session-a') + ]) + }) + + it('lists no chat without a record, and lists and reads one in a workspace this host cannot start an agent in (T10)', async () => { + await restTestChat(rig, 'session-served', { message: 'here' }) + await restTestChat(rig, 'session-gated', { workspaceId: 'workspace-gated', message: 'there' }) + await rig.crash() + const host = await rig.boot() + rig.unsupportedWorkspaceIds.add('workspace-gated') + const ids = ['session-served', 'session-gated', 'session-missing'] + + await host.restoreReadableSessions(ids) + + // Reading needs no adapter: only starting the agent is gated. + expect(host.listSessionTabs(ids)).toEqual([ + tab('session-served'), + tab('session-gated', 'workspace-gated') + ]) + const page = await host.history({ sessionId: 'session-gated', direction: 'tail' }) + expect(page.ok).toBe(true) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-restore.test.ts index 09421a97cc56..1656b17429c0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-restore.test.ts @@ -104,7 +104,7 @@ describe('structured session provider restore', () => { await restarted.restoreReadableSessions() - expect(restarted.listSessionTabs()).toEqual([ + expect(restarted.listSessionTabs([HOST_TEST_SESSION])).toEqual([ { sessionId: HOST_TEST_SESSION, workspaceId: 'workspace-1', agent: 'claude' } ]) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts index cf80396d8204..fd02951188ea 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts @@ -1,6 +1,8 @@ import { existsSync } from 'node:fs' import { findJournalFileFormatRemnant } from '../agent-session-journal/journal-file-format-remnant' import { legacyJournalDatabaseFile } from '../agent-session-journal/journal-paths' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { JournalHostDatabase } from '../agent-session-journal/journal-host-database' import { readJournalSessionEpoch } from '../agent-session-journal/journal-row-table' import { openStructuredAgentSessionConversationJournal, @@ -22,21 +24,29 @@ export async function restoreStructuredAgentSessionRead( return null } const database = deps.journalDatabase - if (readJournalSessionEpoch(database.db, sessionId) === null) { - // Not in the host's database yet: its history may still sit in a per-chat file the open - // imports, or in the pre-SQLite format the open explains. - const legacyDirectory = database.legacyDirectoryFor({ - workspaceId: record.location.workspaceId, - sessionId - }) - if ( - !existsSync(legacyJournalDatabaseFile(legacyDirectory)) && - !findJournalFileFormatRemnant(legacyDirectory) - ) { - return null - } + if ( + readJournalSessionEpoch(database.db, sessionId) === null && + !hasHistoryOutsideJournalDatabase(database, record) + ) { + return null } return openStructuredAgentSessionConversationJournal(deps, record, { deferPerSessionImport: true }) } + +/** Not in the host's database yet, its history may still sit in a per-chat file the open imports, + * or in the pre-SQLite format the open explains. */ +export function hasHistoryOutsideJournalDatabase( + database: Pick, + record: Pick +): boolean { + const legacyDirectory = database.legacyDirectoryFor({ + workspaceId: record.location.workspaceId, + sessionId: record.sessionId + }) + return ( + existsSync(legacyJournalDatabaseFile(legacyDirectory)) || + findJournalFileFormatRemnant(legacyDirectory) !== null + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts index b6c8444a28bc..22531e9ab336 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts @@ -43,6 +43,8 @@ describe('StructuredAgentSessionReadableRestorer', () => { resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: () => false, + isDisposed: () => false, + isListed: () => true, onReadable: () => undefined }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-journal-opens.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-journal-opens.ts new file mode 100644 index 000000000000..11b292a8fa88 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-journal-opens.ts @@ -0,0 +1,59 @@ +// Each rest rig's journal opens, seen at the store's open: the open counter the rest tests read, and +// a hook a test can hold to hold the open. + +import { resolve } from 'node:path' +import { vi } from 'vitest' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' + +// Each rig's open hook, by its state directory: one spy on the store's open serves every rig. +const openHooks = new Map Promise>() + +/** Where a journal opens: its chat and the state directory of its database. */ +type JournalOpenSite = { sessionId: string; directory: string } + +/** The store keeps both privately; read as the plain fields they are at runtime. */ +function journalOpenSite(journal: unknown): JournalOpenSite | null { + if ( + typeof journal !== 'object' || + journal === null || + !('identity' in journal) || + !('database' in journal) + ) { + return null + } + const { identity, database } = journal + if ( + typeof identity !== 'object' || + identity === null || + !('sessionId' in identity) || + typeof identity.sessionId !== 'string' || + typeof database !== 'object' || + database === null || + !('stateDirectory' in database) || + typeof database.stateDirectory !== 'string' + ) { + return null + } + return { sessionId: identity.sessionId, directory: database.stateDirectory } +} + +export function watchRestTestJournalOpens( + root: string, + hook: (sessionId: string) => Promise +): void { + openHooks.set(resolve(root), hook) + if (vi.isMockFunction(AgentSessionJournal.prototype.open)) { + return + } + const open = AgentSessionJournal.prototype.open + vi.spyOn(AgentSessionJournal.prototype, 'open').mockImplementation(async function ( + this: AgentSessionJournal + ) { + const site = journalOpenSite(this) + const hook = site ? openHooks.get(resolve(site.directory)) : undefined + if (site && hook) { + await hook(site.sessionId) + } + return open.call(this) + }) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-observations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-observations.ts new file mode 100644 index 000000000000..cd693c6bd528 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-observations.ts @@ -0,0 +1,24 @@ +// What a rest rig's run observed: how often each chat's journal opened, and the status rows its +// stream carried. + +import type { RestTestRig } from './structured-agent-session-rest-test-rig' + +export function restTestOpens(rig: RestTestRig, sessionId: string): number { + return rig.journalOpens.mock.calls.filter(([id]) => id === sessionId).length +} + +/** The newest row the status stream carried for a session. */ +export function latestRestTestStatus(rig: RestTestRig, sessionId: string) { + for (const event of rig.statusEvents.toReversed()) { + if (event.type === 'status' && event.session.sessionId === sessionId) { + return event.session + } + if (event.type === 'snapshot') { + const found = event.sessions.find((session) => session.sessionId === sessionId) + if (found) { + return found + } + } + } + return undefined +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts index c16fa93048ba..2e2ecf413de2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts @@ -1,6 +1,8 @@ // A real host over a real store and journal, with a scripted provider and a clock the test moves, -// for the tests of a conversation that outlives its agent. The idle sweep runs on its own short -// interval; a test moves `clock.now` past the idle window and waits for the outcome. +// for the tests of a conversation that outlives its agent and of what a restarted host owes. The +// idle sweep runs on its own short interval; a test moves `clock.now` past the idle window and +// waits for the outcome. Every journal open calls `journalOpens` once, so that mock is the open +// counter, and holding it holds the open. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -8,6 +10,8 @@ import { join } from 'node:path' import { expect, vi, type Mock } from 'vitest' import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' import type { AgentJournalMessageItem } from '../../../shared/agent-session-journal-types' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionMutationEnvelope, AgentSessionStatusEvent, @@ -24,6 +28,7 @@ import { StructuredAgentSessionHost } from './structured-agent-session-host' import { claudeAndCodexAgents } from './structured-agent-session-adapter-router-test-support' import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' import { + HOST_TEST_LOCATION, HOST_TEST_NOW, HOST_TEST_SESSION as SESSION, HOST_TEST_THREAD as THREAD, @@ -34,6 +39,7 @@ import { } from './structured-agent-session-host-test-data' import { STRUCTURED_AGENT_SESSION_IDLE_MS } from './structured-agent-session-idle-sweep' import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { watchRestTestJournalOpens } from './structured-agent-session-rest-test-journal-opens' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' @@ -57,6 +63,9 @@ export type RestTestRig = { store: AgentSessionRecordStore host: StructuredAgentSessionHost adapter: RestTestAdapter + /** Called once per journal open in this rig, with the session id; replace its implementation to + * hold one. */ + journalOpens: Mock<(sessionId: string) => Promise> clock: { now: number } statusEvents: AgentSessionStatusEvent[] /** `readChildWork` serves the session's child records, as the host's store does. */ @@ -65,36 +74,55 @@ export type RestTestRig = { forget: Mock readChildWork: Mock<(subject: unknown) => AgentChildWorkView[]> } + probeOwner: Mock<(record: AgentSessionRecord) => Promise> + /** Workspaces this host's adapter does not serve, as a platform gate would. */ + unsupportedWorkspaceIds: Set /** Opens a fresh host over the same store and journals: what a restart leaves behind. */ restart: (deps?: Partial) => Promise + /** A crash: nothing settles or flushes; the process's timers and handles simply stop. */ + crash: () => Promise + /** A new app run over the same files, with no flush first; the mocks and status stream start + * empty, so they hold what the new run saw and nothing the previous one did. */ + boot: (deps?: Partial) => Promise dispose: () => Promise } let ordinal = 0 let generations = 0 -export function acceptedDispatch(): AgentSessionDispatchOutcome { +/** The rig's own chat keeps the shared test thread; any other chat gets a thread of its own. */ +function restTestThread(sessionId: string): string { + return sessionId === SESSION ? THREAD : `thread-${sessionId}` +} + +export function acceptedDispatch(sessionId = SESSION): AgentSessionDispatchOutcome { ordinal += 1 return { state: 'accepted', - providerIdentity: { provider: 'codex', threadId: THREAD, turnId: `turn-${ordinal}`, ordinal } + providerIdentity: { + provider: 'codex', + threadId: restTestThread(sessionId), + turnId: `turn-${ordinal}`, + ordinal + } } } export function restTestSend( text: string, - fence = 1 + fence: number | null = 1, + sessionId = SESSION ): { envelope: AgentSessionMutationEnvelope; body: AgentJournalMessageItem } { const body = hostTestMessage(text) return { body, envelope: { - sessionId: SESSION, + sessionId, clientOperationId: hostTestOperationId(), expectedRuntimeFence: fence, payloadFingerprint: computeAgentSessionPayloadFingerprint({ method: 'agentSession.send', - sessionId: SESSION, + sessionId, fields: { body } }) } @@ -127,12 +155,14 @@ export function collectSubscriber(): { return { events, emit: (event) => events.push(event) } } +/** `root` is handed to the rig, which removes it on dispose; by default it makes its own. */ export async function createRestTestRig( - deps: Partial = {} + deps: Partial = {}, + options: { root?: string } = {} ): Promise { resetHostTestOperationIds() ordinal = 0 - const root = await mkdtemp(join(tmpdir(), 'orca-rest-')) + const root = options.root ?? (await mkdtemp(join(tmpdir(), 'orca-rest-'))) const clock = { now: HOST_TEST_NOW } const statusEvents: AgentSessionStatusEvent[] = [] const sink: RestTestRig['sink'] = { @@ -140,15 +170,19 @@ export async function createRestTestRig( forget: vi.fn(), readChildWork: vi.fn((): AgentChildWorkView[] => []) } - let store = await openTestAgentSessionRecordStore(root) + const probeOwner: RestTestRig['probeOwner'] = vi.fn(async () => ({ outcome: 'pid-absent' })) + const unsupportedWorkspaceIds = new Set() + const openStore = () => openTestAgentSessionRecordStore(root) + let store = await openStore() const adapter: RestTestAdapter = { - acquire: vi.fn(async ({ fence, spawnToken }) => ({ + acquire: vi.fn(async ({ fence, spawnToken, identity }) => ({ acquisitionGeneration: `generation-${++generations}`, process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, link: { - linkId: `link-${fence}`, - handle: codexProviderHandle(THREAD), - origin: store.getRecord(SESSION)?.providerHandleChain.length + linkId: + identity.sessionId === SESSION ? `link-${fence}` : `link-${identity.sessionId}-${fence}`, + handle: codexProviderHandle(restTestThread(identity.sessionId)), + origin: store.getRecord(identity.sessionId)?.providerHandleChain.length ? ('resumed' as const) : ('created' as const), mintedAtFence: fence, @@ -156,18 +190,22 @@ export async function createRestTestRig( } })), closeSession: vi.fn(async () => true), - dispatch: vi.fn(async () => acceptedDispatch()), + dispatch: vi.fn(async (input) => acceptedDispatch(input.sessionId)), acknowledgeSessionRelease: vi.fn(), holdsDispatch: vi.fn(() => false), readOptions: vi.fn(async () => ({ models: [], current: { model: 'gpt-live' } })) } - const hostFor = (overrides: Partial) => - new StructuredAgentSessionHost({ + const journalOpens = vi.fn(async (_sessionId: string): Promise => undefined) + watchRestTestJournalOpens(root, (sessionId) => journalOpens(sessionId)) + const hostFor = (overrides: Partial) => { + const host = new StructuredAgentSessionHost({ agents: claudeAndCodexAgents(), logger: createStructuredAgentSessionLogger(), store, adapter: { ...adapter, + supportsCreate: (location, agent) => + agent === 'codex' && !unsupportedWorkspaceIds.has(location.workspaceId), releaseAcquisition: vi.fn(async () => true), cancelTurn: async () => ({ cancelled: true }), answerPrompt: async ({ commit }) => commit(), @@ -176,50 +214,113 @@ export async function createRestTestRig( journalDatabase: openTestJournalHostDatabase(root), claimKeyId: 'key-1', mintSpawnToken: () => 'spawn-a', - probeOwner: async () => ({ outcome: 'pid-absent' }), + probeOwner, now: () => clock.now, statusSink: sink, idleSweep: { intervalMs: SWEEP_INTERVAL_MS }, ...deps, ...overrides }) + host.subscribeStatus({ id: 'status', emit: (event) => statusEvents.push(event) }) + return host + } + const reopen = async (overrides: Partial) => { + store = await openStore() + rig.store = store + rig.host = hostFor(overrides) + return rig.host + } const rig: RestTestRig = { root, store, host: hostFor({}), adapter, + journalOpens, clock, statusEvents, sink, + probeOwner, + unsupportedWorkspaceIds, restart: async (overrides = {}) => { await rig.host.flushAllStreamedEvents().catch(() => undefined) - store = await openTestAgentSessionRecordStore(root) - rig.store = store - rig.host = hostFor(overrides) - rig.host.subscribeStatus({ id: 'status', emit: (event) => statusEvents.push(event) }) - return rig.host + return reopen(overrides) + }, + crash: async () => { + const { runtimeState, lifetime, sessions } = rig.host.collaboratorsForTests() + await runtimeState.stopLeaseRenewal() + lifetime.dispose() + for (const session of sessions.values()) { + await session.journal.close() + } + sessions.clear() + }, + boot: (overrides = {}) => { + for (const mock of [journalOpens, adapter.acquire, adapter.dispatch, probeOwner]) { + mock.mockClear() + } + sink.publish.mockClear() + sink.forget.mockClear() + statusEvents.length = 0 + return reopen(overrides) }, dispose: async () => { await rig.host.flushAllStreamedEvents().catch(() => undefined) await rm(root, { recursive: true, force: true }) } } - rig.host.subscribeStatus({ id: 'status', emit: (event) => statusEvents.push(event) }) return rig } -/** Creates the chat, lists its tab, and sends one message so its journal is on disk. */ -export async function foundRestTestChat(rig: RestTestRig): Promise { - const attached = await rig.host.attach(REST_TEST_CALLER, hostTestAttachParams(null)) +/** Creates the chat, lists its tab unless told not to, and sends `message` when one is given, + * waiting for its dispatch. */ +export async function restTestChat( + rig: RestTestRig, + sessionId: string, + options: { workspaceId?: string; listed?: boolean; message?: string } = {} +): Promise { + const fence = rig.store.getRecord(sessionId)?.lease.runtimeFence ?? null + const attached = await rig.host.attach( + REST_TEST_CALLER, + hostTestAttachParams(fence, { + envelope: { + sessionId, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: fence, + payloadFingerprint: '' + }, + location: { ...HOST_TEST_LOCATION, workspaceId: options.workspaceId ?? 'workspace-1' }, + providerHandle: { kind: 'codex', threadId: restTestThread(sessionId) } + }) + ) if (!attached.ok) { throw new Error(`attach refused: ${attached.refusal.code}`) } - await rig.store.setSessionTabVisibility(SESSION, true) - const sent = await rig.host.send(REST_TEST_CALLER, restTestSend('hello', attached.fence)) + if (options.listed !== false) { + await rig.store.setSessionTabVisibility(sessionId, true) + } + if (options.message === undefined) { + return + } + const { dispatch } = rig.adapter + const dispatched = dispatch.mock.calls.length + const params = restTestSend(options.message, attached.fence, sessionId) + const sent = await rig.host.send(REST_TEST_CALLER, params) if (!sent.ok) { throw new Error(`send refused: ${sent.refusal.code}`) } - await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalled()) + await vi.waitFor(() => expect(dispatch.mock.calls.length).toBeGreaterThan(dispatched), { + timeout: 10_000 + }) +} + +/** Creates the rig's own chat, lists its tab, and sends one message so its journal is on disk. */ +export function foundRestTestChat(rig: RestTestRig): Promise { + return restTestChat(rig, SESSION, { message: 'hello' }) +} + +/** A send from a client that has not attached this run, so it names no fence. */ +export function sendRestTestMessage(rig: RestTestRig, sessionId: string, text: string) { + return rig.host.send(REST_TEST_CALLER, restTestSend(text, null, sessionId)) } /** Runs one sweep pass now, for a test that set `idleSweep.intervalMs` out of reach. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-startup.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-startup.ts new file mode 100644 index 000000000000..d1b03346ca77 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-startup.ts @@ -0,0 +1,34 @@ +// Host startup and a crash mid-turn on a rest rig, for the startup tests. + +import { restTestChat, type RestTestRig } from './structured-agent-session-rest-test-rig' + +/** What host startup runs, in order, on the ids the tab list names; answers the listed chats it + * left to the restore after the listing. */ +export async function runRestTestStartup( + rig: RestTestRig, + listed: readonly string[] = rig.store.getVisibleSessionTabIndex().sessionIds +): Promise { + await rig.host.reconcileRestartLeases() + const background = rig.host.startup.seedStoredStatuses(listed) + await rig.host.startup.settleOwedSessions(listed) + await rig.host.restoreReadableSessions(background) + return background +} + +/** A chat whose turn is running when Orca dies: startup selects it to settle. */ +export async function crashRestTestChatMidTurn( + rig: RestTestRig, + sessionId: string, + options: { listed?: boolean } = {} +): Promise { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, ...options }) + const { providerIdentity } = await rig.adapter.dispatch.mock.results.at(-1)!.value + await rig.host + .collaboratorsForTests() + .sessions.get(sessionId)! + .journal.appendItem( + { ...providerIdentity, ordinal: 0 }, + { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, + { fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, turnScope: { kind: 'thread' } } + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts index 7df838d04e82..7d6d706f51f6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts @@ -36,7 +36,7 @@ const NO_OPEN_DEPS = { describe('restart journal restoration', () => { beforeEach(() => restoreRead.mockReset()) - it('bounds historical journal parsing to four sessions at a time', async () => { + it('opens one chat at a time', async () => { const gate = Promise.withResolvers() let active = 0 let peak = 0 @@ -66,16 +66,21 @@ describe('restart journal restoration', () => { resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: () => false, + isDisposed: () => false, + isListed: () => true, onReadable: () => undefined }) - await vi.waitFor(() => expect(active).toBe(4)) - expect(restoreRead).toHaveBeenCalledTimes(4) + await vi.waitFor(() => expect(active).toBeGreaterThan(0)) + // Held long enough for a second lane, had there been one, to have started. + await new Promise((resolve) => setTimeout(resolve, 20)) + const startedWhileHeld = restoreRead.mock.calls.length gate.resolve() await restoration + expect(startedWhileHeld).toBe(1) expect(restoreRead).toHaveBeenCalledTimes(records.length) - expect(peak).toBe(4) + expect(peak).toBe(1) }) it('lets the event loop run between chats', async () => { @@ -107,6 +112,8 @@ describe('restart journal restoration', () => { resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: () => false, + isDisposed: () => false, + isListed: () => true, onReadable: () => undefined }) ticking = false @@ -156,6 +163,8 @@ describe('restart journal restoration', () => { }, serialize: async (_sessionId, task) => task(), hasSession: () => false, + isDisposed: () => false, + isListed: () => true, onReadable: (_sessionId, readable) => { calls.push(readable === restored ? 'onReadable:restored' : 'onReadable') } @@ -183,6 +192,8 @@ describe('restart journal restoration', () => { ...bookkeeping, serialize: async (_sessionId, task) => task(), hasSession: () => false, + isDisposed: () => false, + isListed: () => true, onReadable: () => undefined }) @@ -239,6 +250,8 @@ describe('restart journal restoration', () => { resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: () => false, + isDisposed: () => false, + isListed: () => true, onReadable: (sessionId) => { readable.push(sessionId) } @@ -248,7 +261,7 @@ describe('restart journal restoration', () => { expect(readable).toEqual(records.slice(1).map((record) => record.sessionId)) expect(log.entries).toEqual([ expect.objectContaining({ - fields: expect.objectContaining({ scope: 'restart-restore', sessionId: 'session-0' }) + fields: expect.objectContaining({ scope: 'history-restore-chat', sessionId: 'session-0' }) }) ]) }) @@ -266,6 +279,8 @@ describe('restart journal restoration', () => { resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: () => true, + isDisposed: () => false, + isListed: () => true, onReadable: () => undefined }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 6c6efc49d728..27124708c66f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -13,7 +13,7 @@ import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { mapWithConcurrency } from '../../../shared/map-with-concurrency' +import { mapSettledWithConcurrency } from '../../../shared/map-with-concurrency' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { OpenedStructuredAgentSessionConversation, @@ -21,7 +21,9 @@ import type { } from './structured-agent-session-conversation-open' import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' -const JOURNAL_RESTORE_CONCURRENCY = 4 +// One chat at a time: the restore is CPU-bound on the main thread, so more lanes only lengthen each +// event-loop turn that a user's read or send waits behind. +const JOURNAL_RESTORE_CONCURRENCY = 1 export type StructuredAgentSessionReadRestoreDeps = { openDeps: StructuredAgentSessionConversationOpenDeps & { @@ -34,8 +36,10 @@ export type StructuredAgentSessionReadRestoreDeps = { resolveRecovery: (sessionId: string) => Promise serialize: (sessionId: string, task: () => Promise) => Promise hasSession: (sessionId: string) => boolean - /** A chat whose open failed: it still has its record, so it keeps its tab. */ - onUnopened?: (sessionId: string) => void + /** Quit has begun: the restore opens nothing more. */ + isDisposed: () => boolean + /** Whether the chat still has its tab: one closed while the restore ran stays closed. */ + isListed: (sessionId: string) => boolean onReadable: ( sessionId: string, opened: OpenedStructuredAgentSessionConversation @@ -48,6 +52,9 @@ async function restoreOneStructuredAgentSessionRead( sessionId: string, settleLeases: (sessionId: string) => Promise ): Promise { + if (input.isDisposed()) { + return + } await settleLeases(sessionId) await input.serialize(sessionId, () => restoreOneStructuredAgentSessionReadUnderSerialize(input, sessionId) @@ -56,9 +63,16 @@ async function restoreOneStructuredAgentSessionRead( /** The serialized half of the restore. */ async function restoreOneStructuredAgentSessionReadUnderSerialize( - input: Pick, + input: Pick< + StructuredAgentSessionReadRestoreDeps, + 'openDeps' | 'hasSession' | 'isDisposed' | 'isListed' | 'onReadable' + >, sessionId: string ): Promise { + // Checked under the chat's serialize, where its close and quit's teardown also run. + if (input.isDisposed() || !input.isListed(sessionId)) { + return + } if (input.hasSession(sessionId)) { // A read or a send mid-restore already opened this one. return @@ -78,9 +92,9 @@ export async function restoreStructuredAgentSessionsOnRestart( if (!first) { return } - // One check for the pass. Each chat checks again while it holds, since another writer can mark - // leases unreconciled mid-pass; after the first failure, retrying per chat only waits on the - // same store again, and the next attach or send settles those chats instead. + // One check for the pass, and per chat (startup answers both from its lease phase, which checks + // once); after the first failure, retrying per chat only waits on the same store again, and the + // next attach or send settles those chats instead. let settled = await input.reconcile(first.sessionId) const settleLeases = async (sessionId: string): Promise => { // A session latched in recovery exits here at startup, without waiting for a client. @@ -91,19 +105,23 @@ export async function restoreStructuredAgentSessionsOnRestart( settled = false } } - await mapWithConcurrency(input.records, JOURNAL_RESTORE_CONCURRENCY, async ({ sessionId }) => { - // A journal open is synchronous SQLite: without a macrotask per chat the restore is one long task. - await yieldToEventLoop() - // One chat that cannot open must not keep the rest from restoring; a read of it still refuses. - await restoreOneStructuredAgentSessionRead(input, sessionId, settleLeases).catch( - (error: unknown) => { - input.openDeps.logger.warn('restoring a chat at startup failed', { - scope: 'restart-restore', - sessionId, - error - }) - input.onUnopened?.(sessionId) - } - ) + const results = await mapSettledWithConcurrency( + input.records, + JOURNAL_RESTORE_CONCURRENCY, + async ({ sessionId }) => { + // A journal open is synchronous SQLite: without a macrotask per chat the restore is one long task. + await yieldToEventLoop() + await restoreOneStructuredAgentSessionRead(input, sessionId, settleLeases) + } + ) + // One chat's failure costs only that chat, which opens again when it is read. + results.forEach((result, index) => { + if (result.status === 'rejected') { + input.openDeps.logger.warn('restoring a chat for reading failed', { + scope: 'history-restore-chat', + sessionId: input.records[index]?.sessionId, + error: result.reason + }) + } }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts new file mode 100644 index 000000000000..5457f4355b2c --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts @@ -0,0 +1,109 @@ +// The restore pass runs after the tab list is out, so the app is already usable while it works +// through the chats. A chat closed, or a host quit, before the pass reaches it stays that way. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { + createRestTestRig, + restTestChat, + type RestTestRig +} from './structured-agent-session-rest-test-rig' +import { restTestOpens } from './structured-agent-session-rest-test-observations' + +const IDS = ['session-1', 'session-2', 'session-3', 'session-4', 'session-5'] +const LAST = 'session-5' + +let rig: RestTestRig +let releaseHeld = (): void => undefined + +beforeEach(async () => { + rig = await createRestTestRig() +}) + +afterEach(async () => { + releaseHeld() + await rig.dispose() + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +/** Boots over a crashed run and starts the pass with its first open held, so it has not reached + * the last chat. */ +async function bootWithHeldPass() { + for (const sessionId of IDS) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + } + await rig.crash() + const host = await rig.boot() + const held = Promise.withResolvers() + releaseHeld = held.resolve + rig.journalOpens.mockImplementation(async (sessionId) => { + if (sessionId === IDS[0]) { + await held.promise + } + }) + const pass = host.restoreReadableSessions(IDS) + await vi.waitFor(() => expect(rig.journalOpens).toHaveBeenCalledOnce()) + return { host, pass, release: held.resolve } +} + +const publishedFor = (sessionId: string): number => + rig.sink.publish.mock.calls.filter(([summary]) => summary.sessionId === sessionId).length + +describe('the restore pass after the tab list', () => { + it('does not reopen a chat closed before the pass reached it', async () => { + const { host, pass, release } = await bootWithHeldPass() + + // What closing the tab does. + await host.setSessionTabVisibility(LAST, false) + await host.close(LAST, 'user-close') + const publishedAtClose = publishedFor(LAST) + release() + await pass + + expect(host.hasSession(LAST)).toBe(false) + expect(restTestOpens(rig, LAST)).toBe(0) + expect(publishedFor(LAST)).toBe(publishedAtClose) + expect(host.hasSession('session-1')).toBe(true) + }) + + it('does not open a chat it reached before quit began but opens after', async () => { + await restTestChat(rig, LAST, { message: 'asked' }) + await rig.crash() + const host = await rig.boot() + const { runtimeState, lifetime } = host.collaboratorsForTests() + // Holds the chat between the pass reaching it and its open. + const settling = Promise.withResolvers() + releaseHeld = settling.resolve + const resolveRecovery = runtimeState.resolveRecovery.bind(runtimeState) + const recovery = vi.spyOn(runtimeState, 'resolveRecovery').mockImplementation(async (id) => { + await settling.promise + return resolveRecovery(id) + }) + const pass = host.restoreReadableSessions([LAST]) + await vi.waitFor(() => expect(recovery).toHaveBeenCalled()) + + const teardown = host.flushAllStreamedEvents() + await vi.waitFor(() => expect(lifetime.isDisposed()).toBe(true)) + settling.resolve() + await teardown + await pass + + expect(restTestOpens(rig, LAST)).toBe(0) + expect(host.hasSession(LAST)).toBe(false) + }) + + it('opens nothing more, and settles no more leases, once the host is quitting', async () => { + const { host, pass, release } = await bootWithHeldPass() + const recovery = vi.spyOn(host.collaboratorsForTests().runtimeState, 'resolveRecovery') + + const teardown = host.flushAllStreamedEvents() + release() + await teardown + await pass + + expect(restTestOpens(rig, LAST)).toBe(0) + expect(host.hasSession(LAST)).toBe(false) + expect(recovery).not.toHaveBeenCalledWith(LAST) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts index 0cf019e72397..3a1d14c14442 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts @@ -259,6 +259,8 @@ async function restore(sessionIds: readonly string[]) { resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: (sessionId) => sessions.has(sessionId), + isDisposed: () => false, + isListed: () => true, onReadable: (sessionId, opened) => { sessions.set(sessionId, opened.session) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 9a6e3aca5917..051db78066a9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -12,8 +12,19 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import { agentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' +import { sessionTabListed } from './structured-agent-session-host-tabs' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { journalOpenRefusal } from '../agent-session-journal/journal-open-failure' import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' +import { + restoreStructuredAgentSessionsOnRestart, + type StructuredAgentSessionReadRestoreDeps +} from './structured-agent-session-restart-restore' +import { + createStructuredAgentSessionStartupState, + type StructuredAgentSessionStartupState, + type StructuredAgentSessionStartupStateDeps +} from './structured-agent-session-startup-state' import { StructuredAgentSessionRestartRestoreGate } from './structured-agent-session-restart-restore-gate' import { createReaderReconcile, @@ -52,26 +63,31 @@ export async function revealStructuredAgentSession( } } -/** The host's startup readable-restore sweep: reconcile, resolve, then open each chat's journal. - * Its lease bookkeeping is a reader's, which never fails a read or startup; startup shares it. */ +/** The host's startup restore: reconcile, then seed and settle from the state stored beside each + * journal, then open in the background what that state cannot answer. Its lease bookkeeping is a + * reader's, which never fails a read or startup; startup shares it. */ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, wiring: Omit< - ConstructorParameters[0], - 'openDeps' | 'reconcile' | 'resolveRecovery' + StructuredAgentSessionReadRestoreDeps, + 'openDeps' | 'reconcile' | 'resolveRecovery' | 'isListed' > & { reconcileLeases: (sessionId: string) => Promise resolveRecovery: (sessionId: string) => Promise + seedStatus: StructuredAgentSessionStartupStateDeps['seedStatus'] } ): { - reconcileRestartLeases: () => Promise restoreReadableSessions: (sessionIds?: readonly string[]) => Promise -} { - const { reconcileLeases, resolveRecovery, ...rest } = wiring +} & StructuredAgentSessionStartupState { + const { reconcileLeases, resolveRecovery, seedStatus, ...rest } = wiring const failures = reportEachFailureOnce(deps.logger) const reconcile = createReaderReconcile(reconcileLeases, failures) - const restorer = new StructuredAgentSessionReadableRestorer({ + // Reading and settling a chat need no adapter; only starting its agent does. + const canSettle = (record: AgentSessionRecord | null): record is AgentSessionRecord => + record !== null + const readRestore: StructuredAgentSessionReadRestoreDeps = { openDeps: deps, + isListed: (sessionId) => sessionTabListed(deps.store, sessionId), reconcile, // The next attach or send resolves recovery again, strictly, before it acts. resolveRecovery: (sessionId) => @@ -83,12 +99,30 @@ export function createStructuredAgentSessionHostRestore( } ), ...rest + } + const startup = createStructuredAgentSessionStartupState({ + openDeps: deps, + canSettle, + seedStatus, + reconcile, + resolveRecovery: readRestore.resolveRecovery, + restoreListed: (records, leases) => + restoreStructuredAgentSessionsOnRestart({ ...readRestore, ...leases, records }), + recoveryBudgetMs: deps.startupRecoveryBudgetMs, + serialize: rest.serialize, + hasSession: rest.hasSession, + isDisposed: rest.isDisposed + }) + // Startup's lease bookkeeping: no lease is checked again, or recovered twice. + const restorer = new StructuredAgentSessionReadableRestorer({ + ...readRestore, + ...startup.leases }) const gate = new StructuredAgentSessionRestartRestoreGate() return { - reconcileRestartLeases: async () => { - await reconcile('startup') - }, - restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)) + // The listed chats the tab list left to it: no current row, corrupt, unreadable, or still in a + // per-chat file. + restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)), + ...startup } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts index 58003abee526..0300941afc8d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts @@ -23,6 +23,7 @@ import { conversationCommandBlocked } from './structured-conversation-command-ad import { rewindRefusal } from './structured-rewind-refusal' import { persistRewindRecord, recoverStructuredRewind } from './structured-rewind-recovery' import { mergeRetainedHostLifecycleRows } from './structured-rewind-retained-host-rows' +import { serializeStructuredAgentSessionCommand } from './structured-agent-session-command-entry' export async function rewindStructuredAgentSession( context: StructuredAgentSessionMutationContext, @@ -32,7 +33,7 @@ export async function rewindStructuredAgentSession( ): Promise> { const { sessionId, clientOperationId } = params.envelope const store = context.deps.store - return context.serialize(sessionId, async () => { + return serializeStructuredAgentSessionCommand(context, sessionId, async () => { const result = await admitAndRunAgentSessionMutation({ store, adapter: context.deps.adapter, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-during-startup-lease-check.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-during-startup-lease-check.test.ts new file mode 100644 index 000000000000..e293bec65e7c --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-during-startup-lease-check.test.ts @@ -0,0 +1,82 @@ +// A client that is not the desktop window (a phone, the CLI, a remote host's peer) can send before +// startup's lease check has settled the previous run's owner. The send is accepted into the chat +// at once; the agent it needs starts only after the check, through the agent start's own check. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { + createRestTestRig, + restTestChat, + sendRestTestMessage, + type RestTestRig +} from './structured-agent-session-rest-test-rig' + +const CHAT = 'session-a' + +let rig: RestTestRig +// A failed assertion must not leave teardown waiting on a held probe. +let releaseProbe = (): void => undefined + +beforeEach(async () => { + rig = await createRestTestRig() +}) + +afterEach(async () => { + releaseProbe() + await rig.dispose() + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +describe('a send during a slow startup lease check', () => { + it('is accepted at once, and its agent starts once, after the check', async () => { + await restTestChat(rig, CHAT, { message: 'before the restart' }) + await rig.crash() + const probe = Promise.withResolvers() + const probing = Promise.withResolvers() + releaseProbe = probe.resolve + let checked = false + rig.probeOwner.mockImplementation(async () => { + probing.resolve() + await probe.promise + checked = true + return { outcome: 'pid-absent' } + }) + const host = await rig.boot() + const startupCheck = host.reconcileRestartLeases() + await probing.promise + // Each start, with whether the check had adjudicated the chat's lease by then. + const starts: { checked: boolean; unreconciled: boolean | undefined }[] = [] + const acquire = rig.adapter.acquire.getMockImplementation() + rig.adapter.acquire.mockImplementation(async (input) => { + starts.push({ checked, unreconciled: rig.store.getRecord(CHAT)?.lease.unreconciled }) + if (!acquire) { + throw new Error('the rig acquires through its own implementation') + } + return acquire(input) + }) + + const first = await sendRestTestMessage(rig, CHAT, 'first after the restart') + expect(first, JSON.stringify(first)).toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + // Queued behind the first one's delivery, which is waiting on the check. + const second = sendRestTestMessage(rig, CHAT, 'second after the restart') + // Long enough for a delivery that did not wait to have started its agent. + await new Promise((resolve) => setTimeout(resolve, 200)) + expect(starts).toEqual([]) + expect(rig.adapter.dispatch).not.toHaveBeenCalled() + + probe.resolve() + await startupCheck + expect(await second, 'second send').toMatchObject({ ok: true }) + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledTimes(2), { + timeout: 10_000 + }) + + expect(starts).toEqual([{ checked: true, unreconciled: false }]) + const submissions = (await host.journalSnapshot(CHAT)).submissions.slice(-2) + expect(submissions.map((entry) => entry.dispatchState)).toEqual(['accepted', 'accepted']) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-damaged-chat.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-damaged-chat.test.ts new file mode 100644 index 000000000000..b10df671fc43 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-damaged-chat.test.ts @@ -0,0 +1,74 @@ +// A chat whose stored status says it owes work, and whose history is damaged since: startup selects +// it once, its open fails and drops that row, so the next launch selects nothing. Every row stays. + +import { afterEach, expect, it, vi } from 'vitest' +import { + closeTestJournalHostDatabases, + liveTestJournalRows, + openTestJournalHostDatabase, + readTestJournalSessionStatus, + updateTestJournalRowJson +} from '../agent-session-journal/journal-host-database-test-support' +import { readUnsettledJournalSessionIds } from '../agent-session-journal/journal-session-state' +import { createRestTestRig, type RestTestRig } from './structured-agent-session-rest-test-rig' +import { + crashRestTestChatMidTurn, + runRestTestStartup +} from './structured-agent-session-rest-test-startup' + +const SESSION = 'session-damaged' +const rigs: RestTestRig[] = [] + +afterEach(async () => { + for (const rig of rigs.splice(0)) { + await rig.dispose() + } + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +/** Each warning startup logged about the chat. */ +function warningsFor(warn: ReturnType): unknown[] { + return warn.mock.calls.filter( + ([, fields]) => + typeof fields === 'object' && + fields !== null && + 'sessionId' in fields && + fields.sessionId === SESSION + ) +} + +it.each([true, false])( + 'drops the stored mid-turn status of a damaged chat in one launch, so the next selects nothing (listed: %s)', + async (listed) => { + const rig = await createRestTestRig() + rigs.push(rig) + await crashRestTestChatMidTurn(rig, SESSION, { listed }) + await rig.crash() + const { db } = openTestJournalHostDatabase(rig.root) + // Damaged in place: the row still names the chat's tip, so startup selects it. + updateTestJournalRowJson(db, SESSION, 2, '{') + const rows = liveTestJournalRows(db, SESSION).length + expect(readUnsettledJournalSessionIds(db)).toEqual([SESSION]) + + await rig.boot() + const warn = vi.spyOn(rig.host.deps.logger, 'warn') + await runRestTestStartup(rig) + + expect(warningsFor(warn)).toHaveLength(1) + expect(readTestJournalSessionStatus(rig.root, SESSION)).toBeNull() + expect(readUnsettledJournalSessionIds(db)).toEqual([]) + expect(liveTestJournalRows(db, SESSION)).toHaveLength(rows) + + // The obligation died with the row: the next launch selects and opens nothing, and says nothing. + await rig.crash() + await rig.boot() + const again = vi.spyOn(rig.host.deps.logger, 'warn') + await runRestTestStartup(rig) + expect(rig.journalOpens.mock.calls.filter(([id]) => id === SESSION)).toEqual( + listed ? [[SESSION]] : [] + ) + expect(warningsFor(again)).toHaveLength(listed ? 1 : 0) + expect(readUnsettledJournalSessionIds(db)).toEqual([]) + } +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.ts new file mode 100644 index 000000000000..5a07bd6945c6 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.ts @@ -0,0 +1,93 @@ +// Startup's one lease phase: the restart lease check, then the recovery of every lease a crash left +// `recovering` (a provider process that outlived the crash is stopped and its death recorded), so +// each settle verdict reads that evidence. Every startup restore answers its lease bookkeeping from +// this phase and never runs the check itself; each lease is recovered once, within a budget, so a +// restore waits only on the leases of the chats it opens. + +import { forEachWithConcurrency } from '../../../shared/map-with-concurrency' +import { withTimeout } from '../../../shared/promise-timeout-fallback' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' + +// Record-only (a probe and at most a process stop each), so a few at once. +const RECOVERY_CONCURRENCY = 4 +// A recovery is process probes around at most one stop (its SIGTERM grace is 5.5 s). Past this a +// probe has stopped answering: the lease stays `recovering`, unverified and never called dead, for +// the next attach or send to resolve, and startup goes on. +const RECOVERY_BUDGET_MS = 15_000 + +export type StructuredAgentSessionStartupLeasePhase = { + /** Starts the phase's check once; answers, from the records, whether every lease is settled now. + * Never rejects. */ + reconciled: () => Promise + /** After the check, every lease `recovering` at this call has its recovery started; ends once + * each has ended or outlasted its budget. Never rejects. */ + recovered: () => Promise + /** A startup restore's resolver, after the check. A `recovering` lease answers its one recovery, + * or true once that outlasts its budget: a second beside it could hang the same way, so its chat + * opens unverified. Any other lease is resolved as `resolveRecovery` resolves it. */ + resolve: (sessionId: string) => Promise +} + +export function createStructuredAgentSessionStartupLeasePhase(deps: { + store: Pick + logger: StructuredAgentSessionLogger + /** The restart lease check; false when it failed (reported by it). Never rejects. */ + reconcile: (sessionId: string) => Promise + /** Resolves a `recovering` lease; never rejects. */ + resolveRecovery: (sessionId: string) => Promise + budgetMs?: number +}): StructuredAgentSessionStartupLeasePhase { + const budgetMs = deps.budgetMs ?? RECOVERY_BUDGET_MS + const recoveries = new Map>() + let checking: Promise | null = null + // A failed check is usually a store write that failed once, so it runs once more; past that, the + // leases stay unchecked until an attach or send checks them. The answer is read from the records, + // never kept, so a check a command completes later counts here too. + const reconciled = async () => { + await (checking ??= deps + .reconcile('startup') + .then((settled) => settled || deps.reconcile('startup'))) + return !deps.store.listRecords().some((record) => record.lease.unreconciled) + } + const recovering = (sessionId: string) => + deps.store.getRecord(sessionId)?.lease.handoffStage === 'recovering' + const recoverOnce = (sessionId: string): Promise => { + let recovery = recoveries.get(sessionId) + if (!recovery) { + recovery = withTimeout(deps.resolveRecovery(sessionId), budgetMs, null).then( + (resolved) => { + if (resolved !== null) { + return resolved + } + deps.logger.warn('a chat recovery outlasted startup; left unverified', { + scope: 'startup-recovery-timeout', + sessionId + }) + return true + } + ) + recoveries.set(sessionId, recovery) + } + return recovery + } + return { + reconciled, + recovered: async () => { + await reconciled() + await forEachWithConcurrency( + deps.store.listRecords().filter((record) => record.lease.handoffStage === 'recovering'), + RECOVERY_CONCURRENCY, + async ({ sessionId }) => { + await recoverOnce(sessionId) + } + ) + }, + resolve: async (sessionId) => { + await reconciled() + return recoveries.has(sessionId) || recovering(sessionId) + ? recoverOnce(sessionId) + : deps.resolveRecovery(sessionId) + } + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts new file mode 100644 index 000000000000..b55c136979c0 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -0,0 +1,756 @@ +// Host startup from each chat's stored state: settled chats get their status rows without being +// opened, every chat that owes work is settled with no user action (tab or no tab), and a chat is +// opened only when stored state cannot answer for it. + +import { cp, mkdir, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' +import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase, + readTestJournalSessionStatus +} from '../agent-session-journal/journal-host-database-test-support' +import Database from '../../sqlite/sync-database' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { readUnsettledJournalSessionIds } from '../agent-session-journal/journal-session-state' +import { StructuredAgentSessionStartupGate } from '../../runtime/structured-agent-session-startup-gate' +import { writeOlderBuildLease } from '../../runtime/agent-session-older-build-lease.test-fixture' +import { editPersistedTestAgentSessionStore } from '../../runtime/agent-session-record-store-test-harness' +import { hostTestOperationId } from './structured-agent-session-host-test-data' +import { + createRestTestRig, + REST_TEST_CALLER, + restTestChat, + sendRestTestMessage, + type RestTestRig +} from './structured-agent-session-rest-test-rig' +import { + crashRestTestChatMidTurn, + runRestTestStartup +} from './structured-agent-session-rest-test-startup' +import { + latestRestTestStatus, + restTestOpens +} from './structured-agent-session-rest-test-observations' + +const rigs: RestTestRig[] = [] + +afterEach(async () => { + for (const rig of rigs.splice(0)) { + await rig.dispose() + } + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +async function newRig(root?: string): Promise { + const rig = await createRestTestRig({}, root ? { root } : {}) + rigs.push(rig) + return rig +} + +function db(rig: RestTestRig) { + return openTestJournalHostDatabase(rig.root).db +} + +/** A send the provider only admitted, so it stays unanswered: the crash leaves it owed. */ +async function crashMidSend(rig: RestTestRig, sessionId: string, listed = true): Promise { + rig.adapter.dispatch.mockResolvedValueOnce({ state: 'admitted' }) + await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, listed }) +} + +function opened(rig: RestTestRig, ids: readonly string[]): string[] { + return ids.filter((sessionId) => restTestOpens(rig, sessionId) > 0) +} + +function statusRows(rig: RestTestRig, sessionId: string): AgentSessionStatusSummary[] { + return rig.statusEvents.flatMap((event) => + event.type === 'status' && event.session.sessionId === sessionId ? [event.session] : [] + ) +} + +const listedIds = (rig: RestTestRig) => rig.store.getVisibleSessionTabIndex().sessionIds + +describe('seeding statuses from stored state', () => { + it('has every settled chat in the first snapshot a later subscriber gets (T8, T17)', async () => { + const rig = await newRig() + for (const sessionId of ['session-1', 'session-2']) { + await restTestChat(rig, sessionId, { message: sessionId }) + } + await rig.crash() + const host = await rig.boot() + host.startup.seedStoredStatuses(listedIds(rig)) + const snapshots: AgentSessionStatusSummary[][] = [] + host.subscribeStatus({ + id: 'remote-client', + emit: (event) => { + if (event.type === 'snapshot') { + snapshots.push(event.sessions) + } + } + }) + expect(snapshots[0]?.map((row) => [row.sessionId, row.status])).toEqual([ + ['session-1', 'idle'], + ['session-2', 'idle'] + ]) + }) + + it('reaches the status observers once per seeded chat, as a replay (T13)', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-1', { message: 'hi' }) + await rig.crash() + const onSessionStatusChanged = vi.fn() + const host = await rig.boot({ onSessionStatusChanged }) + host.startup.seedStoredStatuses(['session-1']) + expect(onSessionStatusChanged).toHaveBeenCalledOnce() + expect(onSessionStatusChanged).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: 'session-1', status: 'idle' }), + { replay: true } + ) + }) + + it('drops a seeded chat from the status store when its tab closes (T12)', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-1', { message: 'hi' }) + await rig.crash() + const host = await rig.boot() + host.startup.seedStoredStatuses(['session-1']) + expect(rig.sink.publish).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: 'session-1' }), + expect.anything() + ) + + await host.setSessionTabVisibility('session-1', false) + + expect(host.hasSession('session-1')).toBe(false) + expect(rig.sink.forget).toHaveBeenCalledOnce() + }) + + it('seeds nothing and settles nothing from a newer build database (T16, regression guard)', async () => { + const rig = await newRig() + await crashMidSend(rig, 'session-crashed') + await restTestChat(rig, 'session-settled', { message: 'done' }) + await rig.crash() + closeTestJournalHostDatabases() + const raw = new Database(join(rig.root, 'agent-session-journal.db')) + raw.pragma('user_version = 99') + raw.close() + const host = await rig.boot() + const ids = listedIds(rig) + + expect(host.startup.seedStoredStatuses(ids)).toEqual(ids) + await host.startup.settleOwedSessions(ids) + expect(rig.sink.publish).not.toHaveBeenCalled() + expect(opened(rig, ids)).toEqual([]) + }) +}) + +/** A copy of the rig's files for a second host, which reads each chat directly; taken while + * nothing is writing. */ +async function readCopy( + rig: RestTestRig, + probe?: Parameters[0] +): Promise { + closeTestJournalHostDatabases() + const readRoot = `${rig.root}-read` + await cp(rig.root, readRoot, { recursive: true }) + const readRig = await newRig(readRoot) + if (probe) { + readRig.probeOwner.mockImplementation(probe) + } + return readRig +} + +async function readEach(rig: RestTestRig, ids: readonly string[]): Promise { + await rig.host.reconcileRestartLeases() + for (const sessionId of ids) { + await rig.host.history({ sessionId, direction: 'tail' }) + } +} + +describe('startup statuses end where a direct read of each chat lands', () => { + it('seeds each settled chat exactly as its open would publish it, crash-cut verdicts included (T5)', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-answered', { message: 'finished work' }) + await restTestChat(rig, 'session-quiet') + rig.adapter.dispatch.mockResolvedValueOnce({ + state: 'rejected', + ...agentSessionFailureWords(agentSessionFailureFact('providerRejected'), { + surface: 'rejection' + }) + }) + await restTestChat(rig, 'session-refused', { message: 'refused' }) + for (const sessionId of ['session-interrupted', 'session-unconfirmed']) { + await crashRestTestChatMidTurn(rig, sessionId) + } + const ids = [ + 'session-answered', + 'session-quiet', + 'session-refused', + 'session-interrupted', + 'session-unconfirmed' + ] + await rig.crash() + // The unconfirmed chat's owner can never be judged; every other one is proven gone. + const probe = async (record: { sessionId: string }) => + record.sessionId === 'session-unconfirmed' + ? { outcome: 'indeterminate' as const, reason: 'no start time' } + : { outcome: 'pid-absent' as const } + rig.probeOwner.mockImplementation(probe) + // The first launch settles the crash-cut turns; the next seeds every chat from its row. + await rig.boot() + await runRestTestStartup(rig, ids) + await rig.host.flushAllStreamedEvents() + await rig.crash() + const readRig = await readCopy(rig, probe) + + const host = await rig.boot() + await host.reconcileRestartLeases() + expect(host.startup.seedStoredStatuses(ids)).toEqual([]) + expect(opened(rig, ids)).toEqual([]) + await readEach(readRig, ids) + + for (const sessionId of ids) { + // Every field, `updatedAt` included: both come from the journal's own newest activity. + expect(latestRestTestStatus(rig, sessionId)).toEqual(latestRestTestStatus(readRig, sessionId)) + } + expect(latestRestTestStatus(rig, 'session-refused')).toMatchObject({ turnOutcome: 'failure' }) + expect(latestRestTestStatus(rig, 'session-interrupted')).toMatchObject({ + turnOutcome: 'interruption' + }) + expect(latestRestTestStatus(rig, 'session-unconfirmed')).toMatchObject({ + turnOutcome: 'unconfirmed' + }) + // Opening a seeded chat finds its row equal and sends nothing. + const seeded = ids.map((sessionId) => statusRows(rig, sessionId).length) + for (const sessionId of ids) { + await host.history({ sessionId, direction: 'tail' }) + } + expect(ids.map((sessionId) => statusRows(rig, sessionId).length)).toEqual(seeded) + }) + + it('ends every chat the restore after the listing opens where a direct read lands, with no pre-crash work (T8)', async () => { + const rig = await newRig() + const ids = ['session-settled', 'session-quiet', 'session-crashed', 'session-pending'] + // Two a clean quit settled, one cut off mid-turn by a crash, and one whose send the provider + // only admitted, left pending below a fence that has since moved. + await restTestChat(rig, 'session-settled', { message: 'finished work' }) + await restTestChat(rig, 'session-quiet') + await rig.host.flushAllStreamedEvents() + await rig.boot() + await restTestChat(rig, 'session-crashed', { message: 'cut off' }) + rig.adapter.dispatch.mockResolvedValueOnce({ state: 'admitted' }) + await restTestChat(rig, 'session-pending', { message: 'only admitted' }) + await rig.crash() + // A live lease names the link proven at its fence, so the fence moves with that link. + await rig.store.transitionHandoff('session-pending', (record) => ({ + ...record, + lease: { ...record.lease, runtimeFence: record.lease.runtimeFence + 1 }, + providerHandleChain: record.providerHandleChain.map((link, index, chain) => + index === chain.length - 1 ? { ...link, mintedAtFence: link.mintedAtFence + 1 } : link + ) + })) + const readRig = await readCopy(rig) + + const host = await rig.boot() + await host.reconcileRestartLeases() + await host.restoreReadableSessions(ids) + await readEach(readRig, ids) + + // `updatedAt` is when the row was projected, a wall-clock read that differs between two boots. + const rows = (of: RestTestRig): Record> => + Object.fromEntries( + ids.flatMap((sessionId) => { + const summary = latestRestTestStatus(of, sessionId) + if (!summary) { + return [] + } + const { updatedAt: _updatedAt, ...row } = summary + return [[sessionId, row]] + }) + ) + await vi.waitFor(() => expect(rows(rig)).toEqual(rows(readRig))) + expect(Object.keys(rows(rig))).toEqual(ids) + expect(statusRows(rig, 'session-crashed').map((row) => row.status)).not.toContain('working') + }) +}) + +describe('startup opens only what it must (T6, T7, T14)', () => { + it('settles crashed chats with or without a tab, and opens only what stored state cannot answer', async () => { + const rig = await newRig() + const settled = Array.from({ length: 8 }, (_, index) => `session-settled-${index}`) + for (const sessionId of settled) { + await restTestChat(rig, sessionId, { message: sessionId }) + } + await crashMidSend(rig, 'session-crashed') + await crashMidSend(rig, 'session-crashed-closed', false) + for (const sessionId of ['session-rowless', 'session-draft', 'session-uncopied']) { + await restTestChat(rig, sessionId, { message: sessionId }) + } + await rig.crash() + + // Last written before its status table existed: it has a journal and no status. + db(rig).prepare('DELETE FROM journal_session_state WHERE session_id = ?').run('session-rowless') + // A draft an earlier process queued: paused by the restart, so an open would send nothing. + db(rig) + .prepare( + `INSERT INTO queued_messages (session_id, message_id, position, body_json, fingerprint, + created_at, host_instance, state) VALUES (?, 'draft-1', 1, ?, 'fp', 1, 'host-a', 'waiting')` + ) + .run( + 'session-draft', + JSON.stringify({ kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'x' }] }) + ) + // Still in the format a pre-database build wrote: nothing of it is in the host's database. + for (const table of ['journal_rows', 'journal_sessions', 'journal_session_state']) { + db(rig).prepare(`DELETE FROM ${table} WHERE session_id = ?`).run('session-uncopied') + } + const legacy = openTestJournalHostDatabase(rig.root).legacyDirectoryFor({ + workspaceId: 'workspace-1', + sessionId: 'session-uncopied' + }) + await mkdir(legacy, { recursive: true }) + await writeFile(join(legacy, 'log.jsonl'), '{}\n') + + await rig.boot() + const listed = listedIds(rig) + expect(listed).not.toContain('session-crashed-closed') + const background = await runRestTestStartup(rig, listed) + + const unlisted = ['session-crashed-closed'] + expect(opened(rig, [...listed, ...unlisted]).toSorted()).toEqual( + ['session-crashed', ...unlisted, 'session-rowless', 'session-uncopied'].toSorted() + ) + // No stored status to answer from: both are left to the restore after the listing. + expect(background).toEqual(['session-rowless', 'session-uncopied']) + expect(latestRestTestStatus(rig, 'session-draft')).toMatchObject({ status: 'idle' }) + // Settled with no user action; the listed one is open and never showed its pre-crash work. + for (const sessionId of ['session-crashed', ...unlisted]) { + // The unanswered send is now recovered doubt, which projects as no running request. + expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ + lifecycle: 'idle', + handedOverSends: 0, + summary: { status: null } + }) + } + expect(rig.host.hasSession('session-crashed')).toBe(true) + expect(statusRows(rig, 'session-crashed').map((row) => row.status)).not.toContain('working') + // The tabless ones are settled and closed: never indexed, never given a status row (T14). + for (const sessionId of unlisted) { + expect(rig.host.hasSession(sessionId)).toBe(false) + expect(rig.sink.publish.mock.calls.filter(([row]) => row.sessionId === sessionId)).toEqual([]) + } + // Every settled chat has its row without being opened. + for (const sessionId of settled) { + expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ status: 'idle' }) + } + + // The rowless chat's open writes its status back and publishes it. + expect(readTestJournalSessionStatus(rig.root, 'session-rowless')).toMatchObject({ + lifecycle: 'idle' + }) + expect(latestRestTestStatus(rig, 'session-rowless')).toMatchObject({ status: 'idle' }) + + // The next boot finds nothing to settle and nothing without a status (T4b). + await rig.crash() + await rig.boot() + const again = await runRestTestStartup(rig) + expect(again).toEqual([]) + expect(opened(rig, [...listedIds(rig), ...unlisted])).toEqual([]) + }) +}) + +describe('a provider process that outlived the crash (R1T-2)', () => { + it('is stopped at startup for a listed and an unlisted chat, and none is left on the next boot', async () => { + const rig = await newRig() + // Attached, then sent to: its provider process is live when Orca dies. + for (const [id, listed] of [ + ['session-idle-listed', true], + ['session-idle-closed', false] + ] as const) { + await restTestChat(rig, id, { listed }) + await restTestChat(rig, id, { message: 'done', listed }) + } + const ids = ['session-idle-listed', 'session-idle-closed'] + // Each chat's provider process was still up when Orca died: its lease names it, live. + const leases = ids.map((id) => rig.store.getRecord(id)!.lease) + await rig.host.flushAllStreamedEvents() + await rig.crash() + for (const lease of leases) { + await writeOlderBuildLease(rig.root, lease.sessionId, { ...lease }) + } + // The rig's chats share one recorded process; stopping it ends both. + let alive = true + const stopOwnerProcess = vi.fn(() => { + alive = false + }) + rig.probeOwner.mockImplementation(async () => + alive + ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } + : { outcome: 'pid-absent' } + ) + + await rig.boot({ stopOwnerProcess }) + await runRestTestStartup(rig) + + expect(stopOwnerProcess).toHaveBeenCalled() + expect(alive).toBe(false) + for (const id of ids) { + expect(rig.store.getRecord(id)!.lease).toMatchObject({ + handoffStage: null, + ownerProcess: null, + deathEvidence: { kind: 'pid-absent' } + }) + } + // The listed chat was seeded, not opened, and still had its process stopped. + expect(opened(rig, ids)).toEqual([]) + + await rig.crash() + stopOwnerProcess.mockClear() + await rig.boot({ stopOwnerProcess }) + await runRestTestStartup(rig) + expect(stopOwnerProcess).not.toHaveBeenCalled() + }) + + const LEASE_CHECK_FAILURES = [ + { check: "'s first try fails", fails: 1, command: null, checks: 2, stopped: 1 }, + { check: ' fails twice', fails: 2, command: null, checks: 2, stopped: 0 }, + { + check: ' fails twice and a command checks the leases before the settle', + fails: 2, + command: 'before the settle', + checks: 3, + stopped: 1 + }, + { + check: ' fails twice and a command checks the leases once the settle is running', + fails: 2, + command: 'once the settle is running', + checks: 3, + stopped: 1 + } + ] as const + // A loop, not it.each: it.each shortens a long interpolated title, which `-t` then cannot match. + for (const { check, fails, command, checks, stopped } of LEASE_CHECK_FAILURES) { + it(`settles each chat from what its recovery found when the lease check${check}`, async () => { + const rig = await newRig() + const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-listed', 'session-closed']) + const listed = listedIds(rig) + expect(listed).toEqual(['session-listed']) + const reconcile = vi.spyOn(rig.store, 'reconcileOnRestart') + for (let failure = 0; failure < fails; failure += 1) { + reconcile.mockRejectedValueOnce(new Error('disk I/O error')) + } + // An attach the gate let through checks the leases first; what it answers does not matter. + const attach = () => restTestChat(rig, 'session-listed').catch(() => undefined) + + await rig.host.reconcileRestartLeases() + if (command === 'before the settle') { + expect(rig.store.getRecord('session-closed')!.lease.unreconciled).toBe(true) + await attach() + expect(rig.store.getRecord('session-closed')!.lease.handoffStage).toBe('recovering') + } + const background = rig.host.startup.seedStoredStatuses(listed) + // Owed: the crash cut its turn, so the settle opens it. + expect(readUnsettledJournalSessionIds(db(rig))).toContain('session-closed') + const settling = rig.host.startup.settleOwedSessions(listed) + if (command === 'once the settle is running') { + // The settle has waited on its recoveries and read its records. + await new Promise((resolve) => setImmediate(resolve)) + expect(rig.store.getRecord('session-closed')!.lease.unreconciled).toBe(true) + await attach() + } + await settling + await rig.host.restoreReadableSessions(background) + + // No startup restore checks the leases again; with no check that held, nothing is stopped + // and the chat settles unverified. + expect(reconcile).toHaveBeenCalledTimes(checks) + expect(stopOwnerProcess).toHaveBeenCalledTimes(stopped) + expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ + lifecycle: 'idle', + summary: { turnOutcome: stopped ? 'interruption' : 'unconfirmed' } + }) + expect(rig.store.getRecord('session-closed')!.lease).toMatchObject( + stopped ? { deathEvidence: { kind: 'pid-absent' } } : { unreconciled: true } + ) + expect(rig.host.hasSession('session-closed')).toBe(false) + // The listed chat is settled by the listed worker from the same lease answers. + expect(readTestJournalSessionStatus(rig.root, 'session-listed')).toMatchObject({ + lifecycle: 'idle', + summary: { turnOutcome: stopped ? 'interruption' : 'unconfirmed' } + }) + }) + } +}) + +/** Chats whose turn was running when Orca died; the first is listed. The last one's provider + * process is still up until a stop ends it. */ +async function crashWithLiveOwner( + rig: RestTestRig, + ids: readonly string[] +): Promise> { + const leases: AgentSessionRecord['lease'][] = [] + for (const [index, sessionId] of ids.entries()) { + await crashRestTestChatMidTurn(rig, sessionId, { listed: index === 0 }) + leases.push(rig.store.getRecord(sessionId)!.lease) + } + await rig.crash() + for (const lease of leases) { + await writeOlderBuildLease(rig.root, lease.sessionId, { ...lease }) + } + let alive = true + const stopOwnerProcess = vi.fn(() => { + alive = false + }) + rig.probeOwner.mockImplementation(async (record) => + alive && record.sessionId === ids.at(-1) + ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } + : { outcome: 'pid-absent' } + ) + await rig.boot({ stopOwnerProcess }) + return stopOwnerProcess +} + +describe('one awaited settle covers a chat whose tab closes meanwhile (R1T-4)', () => { + it('settles a listed chat the listed worker skips because its tab closed', async () => { + const rig = await newRig() + await crashMidSend(rig, 'session-closing') + await rig.crash() + await rig.boot() + const listed = listedIds(rig) + expect(listed).toContain('session-closing') + await rig.host.reconcileRestartLeases() + rig.host.startup.seedStoredStatuses(listed) + // The tab closes after the listing named it, before its settle runs. + await rig.store.setSessionTabVisibility('session-closing', false) + + await rig.host.startup.settleOwedSessions(listed) + + expect(readTestJournalSessionStatus(rig.root, 'session-closing')).toMatchObject({ + lifecycle: 'idle', + handedOverSends: 0 + }) + expect(rig.host.hasSession('session-closing')).toBe(false) + }) +}) + +describe('commands held for the real startup settle never deadlock it (R2T-1)', () => { + const CEILING_MS = 3_000 + + it('holds a command issued before the settle, lets it, one during it and a healthy read through as the settle ends', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const rig = await newRig() + for (const sessionId of ['session-a', 'session-b']) { + await crashRestTestChatMidTurn(rig, sessionId) + } + await restTestChat(rig, 'session-healthy', { message: 'fine' }) + await rig.crash() + const gate = new StructuredAgentSessionStartupGate(CEILING_MS) + gate.hold() + await rig.boot({ commandsReady: gate.ready }) + const listed = listedIds(rig) + await rig.host.reconcileRestartLeases() + rig.host.startup.seedStoredStatuses(listed) + expect(readUnsettledJournalSessionIds(db(rig)).toSorted()).toEqual(['session-a', 'session-b']) + const started = Date.now() + const elapsed = () => Date.now() - started + const dispatched = rig.adapter.dispatch.mock.calls.length + + // Before the settle: a send to a crashed chat, and the option read a chat pane fires on mount. + let sendAnswered = false + const sendBefore = sendRestTestMessage(rig, 'session-a', 'during startup').then((result) => { + sendAnswered = true + expect(result).toMatchObject({ ok: true }) + return elapsed() + }) + const optionsBefore = rig.host.readOptions('session-a').then(elapsed) + await new Promise((resolve) => setTimeout(resolve, 20)) + // Held, neither answered nor dispatched; the tab list and the seeded status answer meanwhile. + expect(sendAnswered).toBe(false) + expect(rig.adapter.dispatch.mock.calls.length).toBe(dispatched) + expect(listedIds(rig)).toContain('session-healthy') + expect(latestRestTestStatus(rig, 'session-healthy')).toMatchObject({ status: 'idle' }) + const settled = rig.host.startup.settleOwedSessions(listed) + gate.openWhen(settled) + // During the settle: the second crashed chat, which the settle has not reached yet. + const optionsDuring = rig.host.readOptions('session-b').then(elapsed) + const healthyRead = rig.host.journalSnapshot('session-healthy').then(elapsed) + const settleEnded = await settled.then(elapsed) + + const answered = await Promise.all([sendBefore, optionsBefore, optionsDuring, healthyRead]) + expect(settleEnded).toBeLessThan(CEILING_MS / 2) + for (const at of answered) { + expect(at).toBeLessThan(CEILING_MS / 2) + } + expect(gate.ready()).toBeNull() + await vi.waitFor(() => + expect(rig.adapter.dispatch.mock.calls.length).toBeGreaterThan(dispatched) + ) + }, 20_000) + + it('holds a rewind and a /clear sent straight to the host, and a read, then answers them as the settle ends', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const rig = await newRig() + const crashed = ['session-a', 'session-b', 'session-c'] + for (const sessionId of crashed) { + await crashRestTestChatMidTurn(rig, sessionId) + } + await rig.crash() + const gate = new StructuredAgentSessionStartupGate(CEILING_MS) + gate.hold() + await rig.boot({ commandsReady: gate.ready }) + const listed = listedIds(rig) + await rig.host.reconcileRestartLeases() + rig.host.startup.seedStoredStatuses(listed) + const opensAtBoot = crashed.map((id) => restTestOpens(rig, id)) + const started = Date.now() + const elapsed = () => Date.now() - started + + // Neither takes the RPC's reveal first; /clear is on the chat the settle reaches last. + const rewind = rig.host + .rewind(REST_TEST_CALLER, rewindParams('session-a', 'missing-item')) + .then(elapsed, elapsed) + const clear = rig.host + .conversationCommand(REST_TEST_CALLER, conversationCommandParams('session-b', 'clear')) + .then(elapsed, elapsed) + // A read opens a closed chat too, which would settle it ahead of the lease resolution. + const read = rig.host.journalSnapshot('session-c').then(elapsed) + await new Promise((resolve) => setTimeout(resolve, 20)) + // Held: none has opened its crashed chat ahead of the settle. + expect(crashed.map((id) => restTestOpens(rig, id))).toEqual(opensAtBoot) + const settled = rig.host.startup.settleOwedSessions(listed) + gate.openWhen(settled) + const settleEnded = await settled.then(elapsed) + + expect(settleEnded).toBeLessThan(CEILING_MS / 2) + for (const at of await Promise.all([rewind, clear, read])) { + expect(at).toBeGreaterThanOrEqual(20) + expect(at).toBeLessThan(CEILING_MS / 2) + } + }, 20_000) +}) + +function rewindParams(sessionId: string, itemId: string) { + return { + envelope: { + sessionId, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.rewind', + sessionId, + fields: { itemId, expectedEpoch: 'epoch-unknown' } + }) + }, + itemId, + expectedEpoch: 'epoch-unknown' + } +} + +function conversationCommandParams(sessionId: string, command: 'clear') { + return { + envelope: { + sessionId, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.conversationCommand', + sessionId, + fields: { command } + }) + }, + command + } +} + +describe('a listed chat whose recovery never answers', () => { + it.each([ + ['its stored status, through the settle', true], + ['no stored status, through the restore after the listing', false] + ])( + 'is opened unverified with %s, with no second recovery beside the first, and the rest still settle', + async (_path, rowed) => { + const rig = await newRig() + await crashMidSend(rig, 'session-stuck') + await crashMidSend(rig, 'session-closed', false) + const lease = rig.store.getRecord('session-stuck')!.lease + await rig.crash() + if (!rowed) { + db(rig) + .prepare('DELETE FROM journal_session_state WHERE session_id = ?') + .run('session-stuck') + } + // Its provider process was up when Orca died; the check of it answers once, then never again. + await writeOlderBuildLease(rig.root, 'session-stuck', { ...lease }) + let answered = false + let hung = 0 + rig.probeOwner.mockImplementation(async (record) => { + if (record.sessionId !== 'session-stuck') { + return { outcome: 'pid-absent' } + } + if (!answered) { + answered = true + return { outcome: 'identity-matched', matchedOn: ['spawn-token'] } + } + hung += 1 + return new Promise(() => {}) + }) + await rig.boot({ startupRecoveryBudgetMs: 50 }) + expect(listedIds(rig)).toEqual(['session-stuck']) + const warn = vi.spyOn(rig.host.deps.logger, 'warn') + + let background: string[] = [] + const done = await Promise.race([ + runRestTestStartup(rig).then((left) => { + background = left + return 'done' + }), + new Promise((resolve) => setTimeout(() => resolve('still waiting'), 5_000)) + ]) + + expect(done).toBe('done') + expect(background).toEqual(rowed ? [] : ['session-stuck']) + expect(rig.host.hasSession('session-stuck')).toBe(true) + expect(hung).toBe(1) + expect(rig.store.getRecord('session-stuck')!.lease.handoffStage).toBe('recovering') + expect(warn).toHaveBeenCalledWith('a chat recovery outlasted startup; left unverified', { + scope: 'startup-recovery-timeout', + sessionId: 'session-stuck' + }) + expect(opened(rig, ['session-closed'])).toEqual(['session-closed']) + expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ + lifecycle: 'idle' + }) + }, + 20_000 + ) +}) + +describe('a stored status no settle here can clear (R2A-4)', () => { + it('drops the row of a chat whose record is gone, so the next boot selects it no more', async () => { + const rig = await newRig() + await crashMidSend(rig, 'session-gone', false) + await rig.crash() + await editPersistedTestAgentSessionStore(rig.root, (persisted) => { + delete persisted.records['session-gone'] + }) + + await rig.boot() + await runRestTestStartup(rig) + + expect(readTestJournalSessionStatus(rig.root, 'session-gone')).toBeNull() + expect(opened(rig, ['session-gone'])).toEqual([]) + + // The obligation died: the next boot finds no row to select and opens nothing. + await rig.crash() + await rig.boot() + await runRestTestStartup(rig) + expect(readTestJournalSessionStatus(rig.root, 'session-gone')).toBeNull() + expect(opened(rig, ['session-gone'])).toEqual([]) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts new file mode 100644 index 000000000000..901db056bf24 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -0,0 +1,277 @@ +// What host startup does with each chat's stored status, before any client lists a tab. +// +// Leases: one phase checks them, then starts recovering every lease a crash left `recovering` (a +// surviving provider process is stopped and its death recorded), so each settle verdict below reads +// that evidence; every startup restore answers its lease bookkeeping from that phase. +// Seed: every settled chat still listed and not open has its status row published from its stored +// status, so session lists have it at paint, without opening the chat. A listed chat with no current +// row (last written before stored status, by other rules, or by an older build), or whose history is +// still in a per-chat file, is left to the background restore after the listing, which opens it. +// Settle: once every recovery has ended, every chat whose stored status shows work a gone process +// left, listed or not, is opened once, one at a time, and its open appends the settlement plan. A +// listed one goes through the restart restore's own per-chat worker and stays open; any other is +// settled and closed, never indexed or published. Chat commands wait for the settle (see +// `StructuredAgentSessionHostDeps.commandsReady`); listing, paint and status reads do not. A listed +// chat that is corrupt is left to the background restore after the listing, which is the same worker. + +import { setImmediate as yieldToEventLoop } from 'node:timers/promises' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { StructuredAgentSessionStatusProjection } from '../../../shared/structured-agent-session-projection' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + deleteJournalSessionStatus, + isUnsettledJournalSessionStatus, + readJournalSessionStatuses, + readUnsettledJournalSessionIds, + type StoredJournalSessionStatus +} from '../agent-session-journal/journal-session-state' +import { + openStructuredAgentSessionConversationJournal, + type StructuredAgentSessionConversationOpenDeps +} from './structured-agent-session-conversation-open' +import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-read-restore' +import { + createStructuredAgentSessionStartupLeasePhase, + type StructuredAgentSessionStartupLeasePhase +} from './structured-agent-session-startup-lease-phase' + +export type StructuredAgentSessionStartupStateDeps = { + openDeps: StructuredAgentSessionConversationOpenDeps & { + store: Pick + } + /** Whether a chat can be settled here: its record exists. Reading and settling need no adapter. */ + canSettle: (record: AgentSessionRecord | null) => record is AgentSessionRecord + seedStatus: ( + record: AgentSessionRecord, + stored: { projected: StructuredAgentSessionStatusProjection; lastActivityAt: number } + ) => void + /** The restart lease check; false when it failed (reported by it). Never rejects. */ + reconcile: (sessionId: string) => Promise + /** Resolves a `recovering` lease; never throws (a failure is reported and left to the next + * attach or send). */ + resolveRecovery: (sessionId: string) => Promise + /** The restart restore's per-chat worker (lease bookkeeping, serialize, open, publish), with its + * lease bookkeeping answered by `leases`. */ + restoreListed: ( + records: AgentSessionRecord[], + leases: StructuredAgentSessionStartupLeases + ) => Promise + /** Tests shorten it; production takes the default. */ + recoveryBudgetMs?: number + serialize: (sessionId: string, task: () => Promise) => Promise + hasSession: (sessionId: string) => boolean + isDisposed: () => boolean +} + +/** A startup restore's lease bookkeeping, answered from the startup lease phase. */ +export type StructuredAgentSessionStartupLeases = { + reconcile: (sessionId: string) => Promise + resolveRecovery: (sessionId: string) => Promise +} + +export type StructuredAgentSessionStartupState = { + /** The startup lease phase's check, which then starts every recovery. Never rejects. */ + reconcileRestartLeases: () => Promise + /** Seeds settled listed chats; answers the listed ids the background restore still opens. */ + seedStoredStatuses: (listedIds: readonly string[]) => string[] + /** Settles every chat a gone process left with work, once per host. Never rejects. */ + settleOwedSessions: (listedIds: readonly string[]) => Promise + /** Every startup restore's lease bookkeeping, so none checks leases or recovers one twice. */ + leases: StructuredAgentSessionStartupLeases +} + +export function createStructuredAgentSessionStartupState( + deps: StructuredAgentSessionStartupStateDeps +): StructuredAgentSessionStartupState { + const phase = createStructuredAgentSessionStartupLeasePhase({ + ...deps, + store: deps.openDeps.store, + logger: deps.openDeps.logger, + budgetMs: deps.recoveryBudgetMs + }) + const leases: StructuredAgentSessionStartupLeases = { + reconcile: () => phase.reconciled(), + resolveRecovery: phase.resolve + } + let settling: Promise | null = null + return { + reconcileRestartLeases: async () => { + await phase.reconciled() + // Started now, awaited by the settle; the listing never waits on a recovery. + if (!deps.openDeps.journalDatabase.readOnly) { + void phase.recovered() + } + }, + seedStoredStatuses: (listedIds) => seedStoredStatuses(deps, listedIds), + settleOwedSessions: (listedIds) => { + settling ??= settleOwedSessions(deps, listedIds, phase, leases) + return settling + }, + leases + } +} + +function seedStoredStatuses( + deps: StructuredAgentSessionStartupStateDeps, + listedIds: readonly string[] +): string[] { + const database = deps.openDeps.journalDatabase + // A newer build's database: nothing is stored this build can read, and every chat reads as it does. + if (database.readOnly) { + return [...listedIds] + } + let stored: readonly StoredJournalSessionStatus[] + try { + stored = readJournalSessionStatuses(database.db, listedIds) + } catch (error) { + // Fails open: every listed chat is restored in the background, as before stored status existed. + deps.openDeps.logger.warn('reading stored chat status failed', { + scope: 'startup-status-read', + error + }) + return [...listedIds] + } + const byId = new Map(stored.map((row) => [row.sessionId, row.status])) + const background: string[] = [] + for (const sessionId of listedIds) { + // Its open (a restore, read or send) publishes its own status. + if (deps.hasSession(sessionId)) { + continue + } + const record = deps.openDeps.store.getRecord(sessionId) + if (!record) { + // Its record may still be owed by the records import: the restore reads records when it runs. + background.push(sessionId) + continue + } + if (!deps.canSettle(record)) { + continue + } + if (!byId.has(sessionId)) { + // Never sent opens nothing; a chat still in a per-chat file is opened from it in the background. + if (hasHistoryOutsideJournalDatabase(database, record)) { + background.push(sessionId) + } + continue + } + const status = byId.get(sessionId) + if (status && isUnsettledJournalSessionStatus(status)) { + // The settle's: its open settles and publishes. + continue + } + if (!status || (status.summary.status !== null && status.summary.status !== 'idle')) { + // No row this build can read: its open writes and publishes one. + background.push(sessionId) + continue + } + deps.seedStatus(record, { projected: status.summary, lastActivityAt: status.lastActivityAt }) + } + return background +} + +async function settleOwedSessions( + deps: StructuredAgentSessionStartupStateDeps, + listedIds: readonly string[], + phase: StructuredAgentSessionStartupLeasePhase, + leases: StructuredAgentSessionStartupLeases +): Promise { + try { + const database = deps.openDeps.journalDatabase + if (database.readOnly) { + return + } + await phase.recovered() + const listedOrder = new Map(listedIds.map((sessionId, index) => [sessionId, index])) + const listed: AgentSessionRecord[] = [] + const others: AgentSessionRecord[] = [] + for (const sessionId of readUnsettledJournalSessionIds(database.db)) { + const record = deps.openDeps.store.getRecord(sessionId) + if (!deps.canSettle(record)) { + dropUnreachableStatus(deps, sessionId, record) + continue + } + if (listedOrder.has(sessionId)) { + listed.push(record) + } else { + others.push(record) + } + } + listed.sort( + (left, right) => + (listedOrder.get(left.sessionId) ?? 0) - (listedOrder.get(right.sessionId) ?? 0) + ) + await deps.restoreListed(listed, leases) + // A listed chat the worker left closed (its tab closed meanwhile) is settled like any other, + // unless its row is gone: an open that failed on its rows dropped it. + const owed = new Set(readUnsettledJournalSessionIds(database.db)) + others.push( + ...listed.filter((record) => !deps.hasSession(record.sessionId) && owed.has(record.sessionId)) + ) + for (const record of others) { + // A journal open is synchronous SQLite: one chat per macrotask. + await yieldToEventLoop() + if (deps.isDisposed()) { + return + } + // A check a command completed since the records were read may have moved this lease to + // `recovering`: recover it, and settle from the record it left. + await phase.resolve(record.sessionId) + await deps + .serialize(record.sessionId, () => + settleClosed(deps, deps.openDeps.store.getRecord(record.sessionId) ?? record) + ) + .catch((error: unknown) => { + deps.openDeps.logger.warn('settling a chat at startup failed', { + scope: 'startup-settle-chat', + sessionId: record.sessionId, + error + }) + }) + } + } catch (error) { + deps.openDeps.logger.warn('settling chats at startup failed', { + scope: 'startup-settle', + error + }) + } +} + +/** + * A row no settle here can clear: its chat's record is gone. Dropped, so it is not selected every + * boot; an open writes it again if the chat is ever opened here. Kept while the records import is + * owed, which may still bring the record. + */ +function dropUnreachableStatus( + deps: StructuredAgentSessionStartupStateDeps, + sessionId: string, + record: AgentSessionRecord | null +): void { + const database = deps.openDeps.journalDatabase + if (!record && database.legacyRecordImportOwed) { + return + } + try { + deleteJournalSessionStatus(database.db, sessionId) + } catch (error) { + deps.openDeps.logger.warn('dropping an unreachable chat status failed', { + scope: 'startup-drop-status', + sessionId, + error + }) + } +} + +/** A chat nothing holds open: settled and closed, never indexed, so it gets no status row. */ +async function settleClosed( + deps: StructuredAgentSessionStartupStateDeps, + record: AgentSessionRecord +): Promise { + if (deps.isDisposed() || deps.hasSession(record.sessionId)) { + return false + } + const opened = await openStructuredAgentSessionConversationJournal(deps.openDeps, record, { + deferPerSessionImport: true + }) + await opened.session.journal.close() + return true +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.test.ts new file mode 100644 index 000000000000..54a8a625c857 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.test.ts @@ -0,0 +1,53 @@ +// The user's newest accepted send, read from the journal's submissions without rendering it. + +import { describe, expect, it } from 'vitest' +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import { newestAcceptedSendKey } from './structured-agent-session-status-child-work' + +function submission( + index: number, + submittedAt: number, + dispatchState: AgentJournalSubmission['dispatchState'] +): AgentJournalSubmission { + return { + clientMessageId: `send-${index}`, + fence: 1, + payloadFingerprint: `fingerprint-${index}`, + dispatchState, + providerItemId: null, + reason: null, + submittedAt, + resolvedAt: null + } +} + +/** The key as read from the journal's render: its submissions by `submittedAt` (a stable sort), + * then the last one accepted. */ +function renderedKey(submissions: readonly AgentJournalSubmission[]): string { + const rendered = [...submissions].sort((a, b) => a.submittedAt - b.submittedAt) + return `epoch:${rendered.findLast((entry) => entry.dispatchState === 'accepted')?.clientMessageId ?? ''}` +} + +describe('newestAcceptedSendKey', () => { + it('reads from the submissions in fold order what it reads from the rendered list', () => { + let seed = 7 + const next = (bound: number) => { + seed = (seed * 1_103_515_245 + 12_345) % 2_147_483_648 + return seed % bound + } + const states: AgentJournalSubmission['dispatchState'][] = ['accepted', 'pending', 'rejected'] + for (let round = 0; round < 300; round += 1) { + // Few distinct clocks, so ties are common; fold order is insertion order, not clock order. + const submissions = Array.from({ length: next(8) }, (_, index) => + submission(index, next(4), states[next(states.length)] ?? 'accepted') + ) + expect(newestAcceptedSendKey('epoch', submissions)).toBe(renderedKey(submissions)) + } + }) + + it('names the later of two accepted sends with the same clock, as the render orders them', () => { + const submissions = [submission(0, 5, 'accepted'), submission(1, 5, 'accepted')] + + expect(newestAcceptedSendKey('epoch', submissions)).toBe('epoch:send-1') + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.ts index 52b7df745469..bce4097ebcab 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.ts @@ -58,8 +58,18 @@ export function structuredStatusChildrenEqual( * never one; a rewind replaces the epoch. */ export function newestAcceptedSendKey( epoch: string, - submissions: readonly AgentJournalSubmission[] + submissions: Iterable ): string { - const accepted = submissions.findLast((submission) => submission.dispatchState === 'accepted') + // Newest by `submittedAt`, a tie to the later one: what the last accepted send of the journal's + // render (sorted by `submittedAt`, stable) is, without rendering the journal. + let accepted: AgentJournalSubmission | undefined + for (const submission of submissions) { + if ( + submission.dispatchState === 'accepted' && + (!accepted || submission.submittedAt >= accepted.submittedAt) + ) { + accepted = submission + } + } return `${epoch}:${accepted?.clientMessageId ?? ''}` } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts index 9fa62fd170e3..0578e95f6b6e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts @@ -128,6 +128,8 @@ describe('StructuredAgentSessionStatusFeed', () => { if (firstStatus?.type !== 'status') { throw new Error('status publication missing') } + // Which provider child is starting is not on the wire: nothing reads it. + expect(firstStatus.session).not.toHaveProperty('hostExecutionChild') const journalTime = firstStatus.session.updatedAt sessions.get(SESSION)!.child = null feed.publish(SESSION, journal) @@ -652,19 +654,25 @@ describe('StructuredAgentSessionStatusFeed', () => { const { feed, events } = feedFor(new Map([[SESSION, { journal }]]), null, undefined, () => [ statusFeedChildView({ state: taskState }) ]) + // The journal's own projection, which the status it stores beside each write reads too. + const projected = feed.journalProjection(SESSION)?.state + expect(projected).toBe(journal.sessionStatus.at(undefined)) for (let tick = 1; tick <= 100; tick++) { taskState = tick % 2 === 1 ? 'waiting' : 'working' feed.publish(SESSION) + expect(feed.journalProjection(SESSION)?.state).toBe(projected) } expect(events).toHaveLength(101) - expect(snapshot).toHaveBeenCalledTimes(1) + // The row never renders the journal: the projection and the accepted send come without it. + expect(snapshot).not.toHaveBeenCalled() expect(events.at(-1)).toMatchObject({ type: 'status', session: { status: 'working', backgroundTasks: [{ state: 'working' }] } }) await journal.appendTombstone(TURN_IDENTITY, { fence: 1 }) feed.publish(SESSION) - expect(snapshot).toHaveBeenCalledTimes(2) + expect(snapshot).not.toHaveBeenCalled() + expect(feed.journalProjection(SESSION)?.state).not.toBe(projected) expect(events.at(-1)).toMatchObject({ type: 'status', session: { status: 'idle' } }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts index e053d401200f..c64ee2a785fb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts @@ -20,7 +20,11 @@ import type { AgentChildWorkEvidence } from '../../../shared/agent-status-child- import type { AgentChildWorkView } from '../../../shared/agent-status-child-work-view' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' -import { structuredAgentSessionStatusSummary } from './structured-agent-session-status-summary' +import type { StructuredAgentSessionStatusProjection } from '../../../shared/structured-agent-session-projection' +import { + statusSummaryChildWorkFields, + structuredAgentSessionStatusSummary +} from './structured-agent-session-status-summary' import { structuredStatusChildWork } from './structured-agent-session-status-child-work' import { StructuredAgentSessionJournalProjections, @@ -28,6 +32,11 @@ import { } from './structured-agent-session-status-journal-projection' import { structuredStatusSummariesEqual } from './structured-agent-session-status-summary-equality' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' +import { StructuredAgentSessionStatusPublications } from './structured-agent-session-status-publications' +import { + StructuredAgentSessionStatusSubscribers, + type StructuredAgentSessionStatusSubscriber +} from './structured-agent-session-status-subscribers' import { StructuredAgentSessionStatusOwnership, type StructuredAgentSessionStatusSink @@ -37,10 +46,7 @@ export type { StructuredAgentSessionStatusSink } from './structured-agent-sessio export type { StructuredAgentSessionStatusState } from './structured-agent-session-status-journal-projection' -export type StructuredAgentSessionStatusSubscriber = { - id: string - emit: (event: AgentSessionStatusEvent) => void -} +export type { StructuredAgentSessionStatusSubscriber } from './structured-agent-session-status-subscribers' type StatusFeedSession = { journal: AgentSessionJournal @@ -73,15 +79,8 @@ export class StructuredAgentSessionStatusFeed { private readonly ownership = new StructuredAgentSessionStatusOwnership(() => this.deps.statusSink?.() ) - private readonly subscribers = new Map() - // Never evicted: chats are named only while in here, and AI Vault reads closed ones' names here. - private readonly published = new Map< - string, - { - summary: AgentSessionStatusSummary - firstInputSubmissionKey: string | null - } - >() + private readonly subscribers = new StructuredAgentSessionStatusSubscribers() + private readonly published = new StructuredAgentSessionStatusPublications() /** The user's newest accepted send each session was last projected with; a new one retires * settled children. */ private readonly acceptedSends = new Map() @@ -100,11 +99,7 @@ export class StructuredAgentSessionStatusFeed { for (const [sessionId] of this.deps.sessions) { this.publish(sessionId, undefined, { replay: true }) } - this.subscribers.set(subscriber.id, subscriber) - this.emit(subscriber, { - type: 'snapshot', - sessions: [...this.published.values()].map(({ summary }) => summary) - }) + this.subscribers.add(subscriber, { type: 'snapshot', sessions: this.published.summaries() }) return () => this.unsubscribe(subscriber.id) } @@ -125,38 +120,25 @@ export class StructuredAgentSessionStatusFeed { } catch (error) { this.logFailure('status-sink-forget', 'status sink forget failed', sessionId, error) } - const publication = this.published.get(sessionId) - const previous = publication?.summary + const previous = this.published.get(sessionId)?.summary if (!previous || (!previous.children && !previous.backgroundTasks)) { return } const { children: _children, backgroundTasks: _backgroundTasks, ...rest } = previous - const retained = { ...rest, ...this.childWorkFields(sessionId, previous.agent) } - this.published.set(sessionId, { - summary: retained, - firstInputSubmissionKey: publication?.firstInputSubmissionKey ?? null - }) + const childWork = structuredStatusChildWork(this.readChildWork(sessionId), previous.agent) + const retained = { ...rest, ...statusSummaryChildWorkFields(childWork) } + this.published.replace(sessionId, retained) this.broadcast({ type: 'status', session: retained }) } unsubscribe(id: string): void { - const subscriber = this.subscribers.get(id) - if (!subscriber) { - return - } - this.subscribers.delete(id) - try { - subscriber.emit({ type: 'end' }) - } catch { - // The transport is already gone; teardown must remain idempotent. - } + this.subscribers.remove(id) } /** Revoke live execution authority while retaining the last projection for reload history. A * Stop still ending work is live state too: with the host gone, nothing here is ending it. */ revokeLive(sessionId: string): void { - const publication = this.published.get(sessionId) - const previous = publication?.summary + const previous = this.published.get(sessionId)?.summary if (!previous) { return } @@ -166,10 +148,7 @@ export class StructuredAgentSessionStatusFeed { stopping: _stopping, ...retained } = previous - this.published.set(sessionId, { - summary: retained, - firstInputSubmissionKey: publication?.firstInputSubmissionKey ?? null - }) + this.published.replace(sessionId, retained) this.sink(retained) this.broadcast({ type: 'status', @@ -184,14 +163,14 @@ export class StructuredAgentSessionStatusFeed { this.publish(sessionId) return } - const publication = this.published.get(sessionId) + const previous = this.published.get(sessionId)?.summary const name = this.deps.getRecord(sessionId)?.conversationName - if (!publication || publication.summary.conversationName === name) { + if (!previous || previous.conversationName === name) { return } - const { conversationName: _previous, ...rest } = publication.summary + const { conversationName: _previous, ...rest } = previous const summary = name ? { ...rest, conversationName: name } : rest - this.published.set(sessionId, { ...publication, summary }) + this.published.replace(sessionId, summary) this.broadcast({ type: 'status', session: summary }) } @@ -220,32 +199,68 @@ export class StructuredAgentSessionStatusFeed { const record = this.deps.getRecord(sessionId) const projection = this.projections.read(source, record) this.retireSettledChildrenOnNewTurn(sessionId, session, projection.acceptedSendKey) - const summary = structuredAgentSessionStatusSummary({ - sessionId, - session, - journal: source, - record, - state: projection.state, - stopping: projection.stopping, - childWork: this.childWorkFields(sessionId, session.params.provider), - now: this.deps.now - }) + this.publishSummary( + this.summaryFor(sessionId, session, source, record, projection), + session.params.location, + options?.replay === true, + projection.firstInputSubmissionKey + ) + } + + /** + * A chat's row from the status stored beside its journal, for one this host has not opened: the + * same builder an open's publish uses, so the open later finds it equal and sends nothing. A + * replay, as a restore's publish is. No Stop of this run is ending its work. + */ + seed( + record: AgentSessionRecord, + stored: { projected: StructuredAgentSessionStatusProjection; lastActivityAt: number } + ): void { + const { sessionId } = record + if (this.deps.sessions.has(sessionId)) { + return + } + const params = { location: record.location, provider: record.provider } + const childWork = structuredStatusChildWork(this.readChildWork(sessionId), record.provider) + const { projected, lastActivityAt } = stored + const now = this.deps.now + this.publishSummary( + structuredAgentSessionStatusSummary({ + sessionId, + params, + record, + projected, + stopping: false, + childWork, + lastActivityAt, + now + }), + params.location, + true + ) + } + + private publishSummary( + summary: AgentSessionStatusSummary, + location: AgentSessionRecord['location'], + replay: boolean, + /** Absent for a seed, which read no journal. */ + firstInputSubmissionKey?: string | null + ): void { + const { sessionId } = summary const publication = this.published.get(sessionId) const previous = publication?.summary const summaryChanged = !previous || !structuredStatusSummariesEqual(previous, summary) - const inputChanged = publication?.firstInputSubmissionKey !== projection.firstInputSubmissionKey + const inputChanged = publication?.firstInputSubmissionKey !== (firstInputSubmissionKey ?? null) if (!summaryChanged && !inputChanged) { - if (!this.ownership.matchesLocation(sessionId, session.params.location)) { - this.sink(summary, session.params.location) + if (!this.ownership.matchesLocation(sessionId, location)) { + this.sink(summary, location) } return } - this.published.set(sessionId, { - summary, - firstInputSubmissionKey: projection.firstInputSubmissionKey - }) + this.published.set(sessionId, summary, firstInputSubmissionKey ?? null) if (summaryChanged) { - this.sink(summary, session.params.location) + this.sink(summary, location) this.broadcast({ type: 'status', session: summary }) } if (summary.hostExecutionPhase === 'ready' && previous?.hostExecutionPhase !== 'ready') { @@ -253,8 +268,8 @@ export class StructuredAgentSessionStatusFeed { } try { this.deps.onStatusChanged?.(summary, { - replay: options?.replay === true, - firstInputSubmissionKey: projection.firstInputSubmissionKey + replay, + ...(firstInputSubmissionKey !== undefined ? { firstInputSubmissionKey } : {}) }) } catch (error) { // An observer must never cost the subscribers their status event. @@ -284,24 +299,24 @@ export class StructuredAgentSessionStatusFeed { this.admitChildWork(sessionId, session, [{ type: 'turn-started', observedAt: this.deps.now() }]) } - /** The summary's child fields, from the records the store holds for the session. Usage is - * dropped on purpose: a `task_progress` tick would otherwise fail the equality check and - * re-broadcast a full summary to every remote subscriber for a number no session list renders. - * Tokens stay live on the background-task channel. */ - private childWorkFields( + private summaryFor( sessionId: string, - provider: StatusFeedSession['params']['provider'] - ): Pick { - const { children, backgroundTasks } = structuredStatusChildWork( - this.readChildWork(sessionId), - provider - ) - return { - ...(backgroundTasks && backgroundTasks.length > 0 - ? { backgroundTasks: backgroundTasks.map(({ totalTokens: _tokens, ...task }) => task) } - : {}), - ...(children ? { children } : {}) - } + session: StatusFeedSession, + journal: AgentSessionJournal, + record: AgentSessionRecord | null, + projection: StructuredAgentSessionJournalProjection + ): AgentSessionStatusSummary { + return structuredAgentSessionStatusSummary({ + sessionId, + params: session.params, + record, + child: session.child, + projected: projection.state.summary, + stopping: projection.stopping, + childWork: structuredStatusChildWork(this.readChildWork(sessionId), session.params.provider), + lastActivityAt: journal.lastActivityAt(), + now: this.deps.now + }) } /** Child-work evidence for a session this feed publishes; a failing sink costs nothing else. @@ -357,18 +372,6 @@ export class StructuredAgentSessionStatusFeed { } private broadcast(event: AgentSessionStatusEvent): void { - // A Map skips entries deleted mid-iteration, so a failing subscriber can drop itself here. - for (const subscriber of this.subscribers.values()) { - this.emit(subscriber, event) - } - } - - /** A dead transport must not poison every later publication. */ - private emit(subscriber: StructuredAgentSessionStatusSubscriber, event: AgentSessionStatusEvent) { - try { - subscriber.emit(event) - } catch { - this.subscribers.delete(subscriber.id) - } + this.subscribers.broadcast(event) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts index f28a49c81aea..0f4ec3aaa9d2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts @@ -5,7 +5,7 @@ import { isStructuredAgentSessionCommandEntry } from '../../../shared/structured // row is, and the user's newest send the provider accepted. import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' +import type { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { newestAcceptedSendKey } from './structured-agent-session-status-child-work' import { structuredAgentSessionStopping } from './structured-agent-session-stopping' @@ -53,14 +53,10 @@ export class StructuredAgentSessionJournalProjections { ) { // A journalled submission bumps `lastSequence`, so the send-time working // signal reaches the cache; the lease fence does not, hence the extra key. - const snapshot = journal.snapshot() + const submissions = journal.submissions() let firstInputSubmissionKey = projection?.epoch === cursor.epoch ? projection.firstInputSubmissionKey : null const start = projection?.epoch === cursor.epoch ? projection.submissionCount : 0 - const submissions = - !firstInputSubmissionKey && snapshot.submissions.length !== start - ? journal.submissions() - : snapshot.submissions // New submissions are visited once; streamed output cannot rescan command history. for (let index = start; !firstInputSubmissionKey && index < submissions.length; index++) { const submission = submissions[index] @@ -79,16 +75,17 @@ export class StructuredAgentSessionJournalProjections { projection = { ...cursor, firstInputSubmissionKey, - submissionCount: snapshot.submissions.length, + submissionCount: submissions.length, fence, stopRevision, - state: projectStructuredAgentSessionStatusState( - snapshot.items, - snapshot.submissions, - fence - ), - acceptedSendKey: newestAcceptedSendKey(cursor.epoch, snapshot.submissions), - stopping: structuredAgentSessionStopping(journal, snapshot.items, snapshot.submissions) + // The journal's own projection, shared with the status it stores beside each write. + state: journal.sessionStatus.at(fence), + // From the submissions alone: rendering the whole journal for one key costs every commit. + acceptedSendKey: newestAcceptedSendKey(cursor.epoch, submissions), + // Only a chat with a Stop on record renders its journal for this. + stopping: + journal.stopMarks.latest() !== null && + structuredAgentSessionStopping(journal, journal.snapshot().items, submissions) } this.byJournal.set(journal, projection) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-publications.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-publications.ts new file mode 100644 index 000000000000..2fae0859b4dc --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-publications.ts @@ -0,0 +1,34 @@ +// The summary the status feed last published for each chat, with the first input it was projected +// with. Never evicted: chats are named only while in here, and AI Vault reads closed ones' names here. + +import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' + +type StructuredAgentSessionStatusPublication = { + summary: AgentSessionStatusSummary + firstInputSubmissionKey: string | null +} + +export class StructuredAgentSessionStatusPublications { + private readonly bySession = new Map() + + get(sessionId: string): StructuredAgentSessionStatusPublication | undefined { + return this.bySession.get(sessionId) + } + + summaries(): AgentSessionStatusSummary[] { + return [...this.bySession.values()].map(({ summary }) => summary) + } + + set( + sessionId: string, + summary: AgentSessionStatusSummary, + firstInputSubmissionKey: string | null + ): void { + this.bySession.set(sessionId, { summary, firstInputSubmissionKey }) + } + + /** The row's summary changed outside a projection; the input it was projected with stays. */ + replace(sessionId: string, summary: AgentSessionStatusSummary): void { + this.set(sessionId, summary, this.bySession.get(sessionId)?.firstInputSubmissionKey ?? null) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-subscribers.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-subscribers.ts new file mode 100644 index 000000000000..dce7a6d23e58 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-subscribers.ts @@ -0,0 +1,46 @@ +// The status feed's subscribers: each gets every event, and one whose transport throws is dropped. + +import type { AgentSessionStatusEvent } from '../../../shared/agent-session-wire' + +export type StructuredAgentSessionStatusSubscriber = { + id: string + emit: (event: AgentSessionStatusEvent) => void +} + +export class StructuredAgentSessionStatusSubscribers { + private readonly subscribers = new Map() + + add(subscriber: StructuredAgentSessionStatusSubscriber, first: AgentSessionStatusEvent): void { + this.subscribers.set(subscriber.id, subscriber) + this.emit(subscriber, first) + } + + remove(id: string): void { + const subscriber = this.subscribers.get(id) + if (!subscriber) { + return + } + this.subscribers.delete(id) + try { + subscriber.emit({ type: 'end' }) + } catch { + // The transport is already gone; teardown must remain idempotent. + } + } + + broadcast(event: AgentSessionStatusEvent): void { + // A Map skips entries deleted mid-iteration, so a failing subscriber can drop itself here. + for (const subscriber of this.subscribers.values()) { + this.emit(subscriber, event) + } + } + + /** A dead transport must not poison every later publication. */ + private emit(subscriber: StructuredAgentSessionStatusSubscriber, event: AgentSessionStatusEvent) { + try { + subscriber.emit(event) + } catch { + this.subscribers.delete(subscriber.id) + } + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-summary.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-summary.ts index 1b5c8d2b0924..465dbcc990c4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-summary.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-summary.ts @@ -1,63 +1,72 @@ +// One structured session's status row, built the same way whether its projected fields come from +// an open journal or from the status stored beside it (journal-session-state.ts), so a row seeded +// at startup and the row the chat's open later publishes are equal field for field. + import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' import { normalizeOptionalField } from '../../../shared/agent-status-field-normalization' import { AGENT_MODEL_MAX_LENGTH } from '../../../shared/agent-status-types' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import type { StructuredAgentSessionStatusState } from './structured-agent-session-status-journal-projection' +import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' +import type { StructuredAgentSessionStatusProjection } from '../../../shared/structured-agent-session-projection' import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' import { structuredAgentSessionProviderSessionMetadata } from './structured-agent-session-history-result' +import type { StructuredStatusChildWork } from './structured-agent-session-status-child-work' import { agentSessionPinnedLaunchDirectory } from '../../runtime/agent-session-record-launch-directory' -export function structuredAgentSessionStatusSummary({ - sessionId, - session, - journal, - record, - state, - stopping, - childWork, - now -}: { +export function structuredAgentSessionStatusSummary(input: { sessionId: string - session: { - params: { location: AgentSessionRecord['location']; provider: AgentSessionRecord['provider'] } - child?: Pick | null - } - journal: AgentSessionJournal + params: { location: AgentSessionRecord['location']; provider: AgentSessionRecord['provider'] } record: AgentSessionRecord | null - state: StructuredAgentSessionStatusState + child?: Pick | null + projected: StructuredAgentSessionStatusProjection /** A person's Stop is still ending the work it stopped. */ stopping: boolean - childWork: Pick + childWork: StructuredStatusChildWork + /** The journal's newest activity; 0 when it can date none. */ + lastActivityAt: number now: () => number }): AgentSessionStatusSummary { - const projected = state.summary + const { record, child, projected } = input const providerSession = structuredAgentSessionProviderSessionMetadata(record) // The journal has no model: the record's acknowledged options are where a mid-session // switch lands, so the row follows whichever is in force. const model = normalizeOptionalField(record?.options?.model, AGENT_MODEL_MAX_LENGTH) const launchDirectory = record ? agentSessionPinnedLaunchDirectory(record) : undefined return { - sessionId, - workspaceId: session.params.location.workspaceId, - agent: session.params.provider, - ...(session.child + sessionId: input.sessionId, + workspaceId: input.params.location.workspaceId, + agent: input.params.provider, + ...(child ? { hostExecutionOwned: true as const, - hostExecutionPhase: session.child.phase + hostExecutionPhase: child.phase } : {}), ...projected, // Only a working session is still being stopped; any other status already ended that work. - ...(stopping && projected.status === 'working' ? { stopping: true as const } : {}), + ...(input.stopping && projected.status === 'working' ? { stopping: true as const } : {}), ...(record?.rewind?.phase === 'prepared' || record?.rewind?.phase === 'provider-succeeded' ? { rewindBlockedReason: 'outcome-unknown' as const } : {}), ...(model ? { model } : {}), - ...childWork, + ...statusSummaryChildWorkFields(input.childWork), ...(providerSession ? { providerSession } : {}), ...(launchDirectory ? { launchDirectory } : {}), ...(record?.conversationName ? { conversationName: record.conversationName } : {}), - updatedAt: journal.lastActivityAt() || now() + updatedAt: input.lastActivityAt || input.now() + } +} + +/** The summary's child fields. Usage is dropped on purpose: a `task_progress` tick would otherwise + * fail the equality check and re-broadcast a full summary to every remote subscriber for a number + * no session list renders. Tokens stay live on the background-task channel. */ +export function statusSummaryChildWorkFields({ + children, + backgroundTasks +}: StructuredStatusChildWork): Pick { + return { + ...(backgroundTasks && backgroundTasks.length > 0 + ? { backgroundTasks: backgroundTasks.map(({ totalTokens: _tokens, ...task }) => task) } + : {}), + ...(children ? { children } : {}) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts index d0b286b24b91..c35d3b17e4fd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts @@ -431,7 +431,7 @@ describe('startup', () => { // The record is readable — the tab comes back, history answers — and nothing is running. expect(acquire).not.toHaveBeenCalled() - expect(host.listSessionTabs()).toEqual([ + expect(host.listSessionTabs([SESSION])).toEqual([ { sessionId: SESSION, workspaceId: 'workspace-1', agent: 'codex' } ]) expect((await host.history({ sessionId: SESSION, direction: 'tail' })).ok).toBe(true) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unopened-tab.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unopened-tab.test.ts index c95edf516029..6c636078ade8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unopened-tab.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unopened-tab.test.ts @@ -86,15 +86,17 @@ it.each([ 'lists a restored chat that could not be opened (%s), and its read and send are refused with why', async (_why, edit, reason) => { const host = await relaunchWith(edit) - const workspaceId = hostTestState().store.getRecord(SESSION)?.location.workspaceId + const { store } = hostTestState() + const workspaceId = store.getRecord(SESSION)?.location.workspaceId + const listed = () => host.listSessionTabs(store.getVisibleSessionTabIndex().sessionIds) + // A restore runs for the tabs the listing names. + await store.setSessionTabVisibility(SESSION, true) const refusal = { code: 'agent_session_journal_unreadable', details: { reason } } await host.restoreReadableSessions([SESSION]) expect(host.hasSession(SESSION)).toBe(false) - expect(host.listSessionTabs()).toEqual([ - { sessionId: SESSION, workspaceId, agent: expect.any(String) } - ]) + expect(listed()).toEqual([{ sessionId: SESSION, workspaceId, agent: expect.any(String) }]) // Every read is refused with why, and the host logs it once for the chat. for (let read = 0; read < 3; read += 1) { await expect(host.history({ sessionId: SESSION, direction: 'tail' })).rejects.toMatchObject({ @@ -111,7 +113,7 @@ it.each([ ).resolves.toMatchObject({ ok: false, refusal }) await host.setSessionTabVisibility(SESSION, false) - expect(host.listSessionTabs()).toEqual([]) + expect(listed()).toEqual([]) await host.flushAllStreamedEvents() } ) diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts index 764b569d8ff5..dff24fdb9cbc 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts @@ -22,6 +22,7 @@ import { type AgentSessionFailureWordsContext } from '../../../shared/agent-session-failure-words' import { carryQueuedMessagesToClearReplacement } from './structured-agent-session-queued-mutations' +import { serializeStructuredAgentSessionCommand } from './structured-agent-session-command-entry' import type { StructuredAgentId } from '../../../shared/agent-session-provider-handle' /** A command's `error` is the sentence its row shows. */ @@ -98,7 +99,7 @@ export function runStructuredConversationCommand( ? record : null } - return context.serialize(sessionId, async () => { + return serializeStructuredAgentSessionCommand(context, sessionId, async () => { const committed = await answerFromCommittedClear(context, caller, params) if (committed) { return committed diff --git a/src/main/orcad/orcad-entry.ts b/src/main/orcad/orcad-entry.ts index 522ef82d742b..fd5b364e80ab 100644 --- a/src/main/orcad/orcad-entry.ts +++ b/src/main/orcad/orcad-entry.ts @@ -321,6 +321,12 @@ async function startOrcadRuntime( // Recovery binds terminal and dispatch identities; only now can startup observations be fenced. observedStatusCapture.attach(runtime) + // Why here and not on a client's first listing: a headless host seeds its chats' statuses and + // settles crashed ones itself, before any client connects. Chat commands a client sends meanwhile + // wait for that settle. + runtime.holdStructuredAgentSessionCommandsForStartup() + runtime.prepareStructuredAgentSessionStartupRestorationAfter(Promise.resolve()) + const bindHost = resolveOrcadBindHost(options.bind) rpc = new OrcaRuntimeRpcServer({ runtime, diff --git a/src/main/orcad/orcad-push-startup.test.ts b/src/main/orcad/orcad-push-startup.test.ts index 41bba43d4cf4..1a0bbea809ae 100644 --- a/src/main/orcad/orcad-push-startup.test.ts +++ b/src/main/orcad/orcad-push-startup.test.ts @@ -83,6 +83,8 @@ vi.mock('../runtime/orca-runtime', () => ({ rehydrateClientHostedBrowserPages() {} async refreshRestoredOrchestrationAuthority() {} async reconcileLegacyWorkerTerminals() {} + prepareStructuredAgentSessionStartupRestorationAfter() {} + holdStructuredAgentSessionCommandsForStartup() {} setMobilePushRegistrar( registrar: Parameters[0] ) { diff --git a/src/main/runtime/agent-session-legacy-record-import.test.ts b/src/main/runtime/agent-session-legacy-record-import.test.ts index d9a1d0268ab2..badf7b819e01 100644 --- a/src/main/runtime/agent-session-legacy-record-import.test.ts +++ b/src/main/runtime/agent-session-legacy-record-import.test.ts @@ -20,6 +20,10 @@ import { agentSessionRecordFixture } from '../../shared/agent-session-record.test-fixture' import { journalPragmaNumber } from '../native-chat/agent-session-journal/journal-database' +import { + JOURNAL_DB_OLDEST_RELEASED_VERSION, + JOURNAL_DB_SCHEMA_VERSION +} from '../native-chat/agent-session-journal/journal-database-schema' import type { JournalHostDatabase } from '../native-chat/agent-session-journal/journal-host-database' import { AgentSessionRecordStore } from './agent-session-record-store' import { legacyAgentSessionStorePath } from './agent-session-record-store-file' @@ -163,7 +167,7 @@ describe('copying the records file into the chat database', () => { const { database, store, reports } = await install() expect(reports).toEqual([]) - expect(journalPragmaNumber(database.db, 'user_version')).toBe(4) + expect(journalPragmaNumber(database.db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) expect(store.getRecord(ALPHA)).toEqual(unreconciled(alpha)) expect(store.getRecord(BETA)).toEqual(unreconciled(beta)) expect(store.listOperationRows()).toEqual([OPERATION]) @@ -200,7 +204,7 @@ describe('copying the records file into the chat database', () => { error: expect.objectContaining({ message: 'agent_session_store_corrupt' }) } ]) - expect(journalPragmaNumber(database.db, 'user_version')).toBe(4) + expect(journalPragmaNumber(database.db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) expect(store.listRecords()).toEqual([]) expect(await readFile(legacyPath(), 'utf-8')).toBe(adHoc) }) @@ -337,7 +341,9 @@ describe('a primary the backup stands in for', () => { const { database, store, reports } = await install() expect(reports).toMatchObject([{ kind: 'unavailable' }]) - expect(journalPragmaNumber(database.db, 'user_version')).toBe(3) + expect(journalPragmaNumber(database.db, 'user_version')).toBe( + JOURNAL_DB_OLDEST_RELEASED_VERSION + ) expect(store.listRecords()).toEqual([]) }) @@ -349,7 +355,9 @@ describe('a primary the backup stands in for', () => { const { database, store, reports } = await install() expect(reports).toMatchObject([{ kind: 'unavailable' }]) - expect(journalPragmaNumber(database.db, 'user_version')).toBe(3) + expect(journalPragmaNumber(database.db, 'user_version')).toBe( + JOURNAL_DB_OLDEST_RELEASED_VERSION + ) expect(store.listRecords()).toEqual([]) }) }) @@ -421,7 +429,7 @@ describe('the tab index from before the table', () => { const { database, store } = await install() - expect(journalPragmaNumber(database.db, 'user_version')).toBe(4) + expect(journalPragmaNumber(database.db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) expect(store.listRecords().map(({ sessionId }) => sessionId)).toEqual([BETA, ALPHA]) // Restore then takes the profile's tabs, beside the tab the owed-era chat left. expect(store.getVisibleSessionTabIndex()).toEqual({ present: false, sessionIds: [BETA] }) diff --git a/src/main/runtime/orca-runtime-collect-mobile-visible-graph-changed-worktrees.ts b/src/main/runtime/orca-runtime-collect-mobile-visible-graph-changed-worktrees.ts index 391a33f0e247..3ffa30b8a3a7 100644 --- a/src/main/runtime/orca-runtime-collect-mobile-visible-graph-changed-worktrees.ts +++ b/src/main/runtime/orca-runtime-collect-mobile-visible-graph-changed-worktrees.ts @@ -4,6 +4,7 @@ import type { RuntimeMobileSessionTabsResult, RuntimeSyncedTab } from '../../sha import type { RuntimeLeafRecord } from './runtime-terminal-state-records' import type { PtyControllerInventory } from './runtime-pty-controller-contract' import { parseExecutionHostId } from '../../shared/execution-host' +import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' export class OrcaRuntimeWithCollectMobileVisibleGraphChangedWorktrees extends OrcaRuntimeWithSyncWindowGraph { // Why: toMobileSessionTabsResult resolves handles/titles from this.tabs and @@ -100,9 +101,11 @@ export class OrcaRuntimeWithCollectMobileVisibleGraphChangedWorktrees extends Or this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession() const ptyInventory = await this.refreshMobileSessionPtyInventory() this.restoreLivePairedRendererSessionOwnedMobileTerminals(null) + // Once for every worktree's answer, after the census's await, so a change made during it counts. + const replacements = getStructuredAgentSessionHost()?.conversationReplacements?.() ?? [] const snapshots = [...this.mobileSessionTabsByWorktree.values()].map((snapshot) => this.projectMobileSessionTabsForClient( - this.toMobileSessionTabsResult(snapshot), + this.toMobileSessionTabsResult(snapshot, replacements), clientNavigationId ) ) diff --git a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts index 0fcdaae95323..2b9bbf4ba977 100644 --- a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts +++ b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts @@ -14,7 +14,6 @@ import { resolveTuiAgentLaunchEnv } from '../../shared/tui-agent-launch-defaults import { structuredAgentRuntimeRegistration } from './structured-agent-runtime-registrations' import { resolveStructuredLaunchSeedOptions } from '../../shared/native-chat-session-option-defaults' import { hasPersistedStructuredAgentSessionStore as hasPersistedStructuredAgentSessionStoreOnDisk } from './structured-agent-session-runtime' -import { ensureStructuredAgentSessionHostUnlessRefused } from './structured-agent-session-host-refusal' import { getProfileUserDataPath } from '../orca-profiles/profile-storage-paths' import { parseWslUncPath } from '../../shared/wsl-paths' import { @@ -305,28 +304,6 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR return this.structuredAgentSessionTabRestorePromise } - prepareStructuredAgentSessionStartupRestoration(): Promise { - this.structuredAgentSessionStartupRestorePromise ??= - this.prepareStructuredAgentSessionStartupRestorationOnce().catch((error) => { - this.structuredAgentSessionStartupRestorePromise = null - throw error - }) - return this.structuredAgentSessionStartupRestorePromise - } - - protected async prepareStructuredAgentSessionStartupRestorationOnce(): Promise { - if (!this.hasPersistedStructuredAgentSessionStore()) { - return - } - // Durable agent records must exist before daemon inventory can be reconciled against them. - // A refused host is no host: startup goes on, and only structured requests are refused. - await ensureStructuredAgentSessionHostUnlessRefused(() => - this.ensureStructuredAgentSessionHost() - ) - await this.refreshMobileSessionPtyRecords() - await getStructuredAgentSessionHost()?.reconcileRestartLeases() - } - protected hasPersistedStructuredAgentSessionStore(): boolean { return hasPersistedStructuredAgentSessionStoreOnDisk(getProfileUserDataPath()) } diff --git a/src/main/runtime/orca-runtime-get-worktree-ps.ts b/src/main/runtime/orca-runtime-get-worktree-ps.ts index 8b7d19af50b0..632eb18dc512 100644 --- a/src/main/runtime/orca-runtime-get-worktree-ps.ts +++ b/src/main/runtime/orca-runtime-get-worktree-ps.ts @@ -32,6 +32,7 @@ import { getProfileUserDataPath } from '../orca-profiles/profile-storage-paths' import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' import { buildWorktreeListingPage } from './worktree-listing-host-scope' import { structuredWorkerOwesWork } from './structured-worker-custody' +import { StructuredAgentSessionStartupGate } from './structured-agent-session-startup-gate' import { resolvedTuiAgentArgsBypassPermissions, resolveTuiAgentLaunchEnv @@ -46,6 +47,14 @@ import { structuredAgentConfiguredArgs } from '../native-chat/structured-agent-c import { claudeThinkingDisplaySupport } from '../claude/claude-thinking-display-support' export class OrcaRuntimeWithGetWorktreePs extends OrcaRuntimeWithStartTuiIdleVisibleReadProbe { + // Chat commands wait on it while startup settles chats a gone process left with work. + protected structuredAgentSessionStartupGate = new StructuredAgentSessionStartupGate() + + /** Before any request is served: chat commands wait until the startup settle ends. */ + holdStructuredAgentSessionCommandsForStartup(): void { + this.structuredAgentSessionStartupGate.hold() + } + async getWorktreePs( limit = DEFAULT_WORKTREE_PS_LIMIT, sourceDefaultsSupported = true @@ -259,7 +268,8 @@ export class OrcaRuntimeWithGetWorktreePs extends OrcaRuntimeWithStartTuiIdleVis // Read per sweep tick from the orchestration database: a worker whose dispatch is open keeps // its agent running. No database answers no. hasOpenDispatch: (record) => - structuredWorkerOwesWork(this.getOrchestrationDbIfAvailable?.() ?? null, record) + structuredWorkerOwesWork(this.getOrchestrationDbIfAvailable?.() ?? null, record), + commandsReady: () => this.structuredAgentSessionStartupGate.ready() }) } } diff --git a/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts b/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts index a1a4f820a486..bf34671eb4b9 100644 --- a/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts +++ b/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts @@ -23,8 +23,9 @@ import type { import { buildRuntimeMobileAgentStatus } from './runtime-mobile-agent-status-builder' import { FIRST_PANE_ID } from '../../shared/pane-key' import { isTerminalLeafId, makePaneKey, parsePaneKey } from '../../shared/stable-pane-id' -import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { replaceConversationInSnapshot } from './structured-conversation-tab-replacement' +import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' import { structuredWorkerHandleAgentStatus } from './orchestration/structured-worker-group-addressing' import { retitleStructuredConversationTab, @@ -79,9 +80,12 @@ export class OrcaRuntimeWithPruneMobileSessionTabGroupLayout extends OrcaRuntime /** Transforms an internal mobile session tab snapshot into a sanitized client payload, resolving launch-agent ownership and normalizing titles. */ protected toMobileSessionTabsResult( - snapshot: RuntimeMobileSessionTabsSnapshot + snapshot: RuntimeMobileSessionTabsSnapshot, + // A caller answering many worktrees in one synchronous loop derives these once for it. + replacements: readonly ConversationReplacement[] = getStructuredAgentSessionHost()?.conversationReplacements?.() ?? + [] ): RuntimeMobileSessionTabsResult { - for (const replacement of getStructuredAgentSessionHost()?.conversationReplacements?.() ?? []) { + for (const replacement of replacements) { snapshot = replaceConversationInSnapshot(snapshot, replacement) } const host = getStructuredAgentSessionHost() diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index a4a2e6e71a67..0ac2fcb55649 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -1,11 +1,11 @@ // @ts-nocheck -- mechanically split from OrcaRuntimeService; behavior is covered by AST equivalence and characterization tests. import { defaultAgentChatLabel } from '../../shared/agent-session-chat-label' -import { OrcaRuntimeWithGetStructuredAgentSessionCreateSupport } from './orca-runtime-get-structured-agent-session-create-support' +import { OrcaRuntimeWithStructuredAgentSessionStartupStep } from './orca-runtime-structured-agent-session-startup-step' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { replaceConversationInSnapshot } from './structured-conversation-tab-replacement' import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' -import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' import { seedStructuredAgentSessionTabIndex } from './structured-agent-session-tab-index-seed' +import { listedStructuredAgentSessionIds } from './structured-agent-session-startup-step' import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' import type { RuntimeMobileSessionAgentTab, @@ -29,7 +29,7 @@ import { parseAppSshPtyId } from '../../shared/ssh-pty-id' import type { PtyProcessInspection } from '../providers/pty-process-inspection' import type { StructuredAgentId } from '../../shared/agent-session-provider-handle' -export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRuntimeWithGetStructuredAgentSessionCreateSupport { +export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRuntimeWithStructuredAgentSessionStartupStep { /** Projects only: a replacement's chat already has its tab in the store, which the /clear commit * moved in the same write. */ replaceStructuredAgentSessionTab(replacement: ConversationReplacement): void { @@ -54,19 +54,10 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu protected async restoreStructuredAgentSessionTabsOnce(): Promise { await this.prepareStructuredAgentSessionStartupRestoration() const host = getStructuredAgentSessionHost() - const persistedVisibleIndex = - typeof host?.getPersistedVisibleSessionTabIndex === 'function' - ? host.getPersistedVisibleSessionTabIndex() - : { present: false, sessionIds: [] } - const profileIds = collectSavedStructuredAgentSessionIds( + const listedIds = listedStructuredAgentSessionIds( + host, this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null ) - // Unrecorded, the profile's chats join the tabs chats opened while the import was owed left. - // First: after a /clear the profile's chat would take their tab id, so seeds hit tabIdTaken. - const targets = persistedVisibleIndex.present - ? persistedVisibleIndex.sessionIds - : [...new Set([...persistedVisibleIndex.sessionIds, ...profileIds])] - await host?.restoreReadableSessions(targets) for (const worktreeId of this.getKnownWorkspaceSessionWorktreeIds()) { this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId, { allowAttachedWindow: true, @@ -75,7 +66,7 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu } this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession() // Every session here is of an agent this host registered: its store holds no other agent's records. - const restored = (host?.listSessionTabs() ?? []).flatMap((session) => { + const restored = (host?.listSessionTabs(listedIds) ?? []).flatMap((session) => { let sessionId = session.sessionId while (sessionId.startsWith('agent-session:')) { sessionId = sessionId.slice('agent-session:'.length) @@ -84,16 +75,33 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu }) await seedStructuredAgentSessionTabIndex( host, - targets, + listedIds, restored.map((session) => session.sessionId) ) // Past the seed, projecting records nothing. - for (const replacement of host?.conversationReplacements?.() ?? []) { + // Derived once for the loop below, which stores a snapshot per tab; nothing awaits in between. + const replacements = host?.conversationReplacements?.() ?? [] + for (const replacement of replacements) { this.replaceStructuredAgentSessionTab(replacement) } + const quiet = { activate: false, notify: false, replacements } for (const session of restored) { - this.projectStructuredAgentSessionTab({ ...session, activate: false, notify: false }) + this.projectStructuredAgentSessionTab({ ...session, ...quiet }) } + // Startup already seeded each settled listed chat not yet open, and is settling the ones a gone + // process left with work; this opens only what stored status could not answer (no current row, + // a corrupt history, a chat still in a per-chat file). + // Whoever answers with this list starts it, once that answer is out. + const background = this.structuredAgentSessionBackgroundRestoreIds ?? listedIds + this.owedStructuredAgentSessionHistoryRestore = host + ? () => + void host.restoreReadableSessions(background).catch((error: unknown) => { + host.deps.logger.warn('restoring chat history after listing failed', { + scope: 'history-restore-after-listing', + error + }) + }) + : null const wasUnverifiable = this.structuredAgentSessionInventoryUnverifiable // No host, or one still owed the records file's chats, means no one can say which chats exist; // with none on disk, empty is the answer. @@ -142,6 +150,8 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu activate: boolean notify?: boolean replacesSessionId?: string + /** The /clear replacements, derived once by a caller projecting many tabs in one loop. */ + replacements?: readonly ConversationReplacement[] }): void { const existing = this.mobileSessionTabsByWorktree.get(input.workspaceId) const id = `agent-session:${input.sessionId}` @@ -218,7 +228,7 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu ...(existing?.tabGroupLayout ? { tabGroupLayout: existing.tabGroupLayout } : {}), tabs } - const stored = this.storeMobileSessionSnapshot(input.workspaceId, snapshot) + const stored = this.storeMobileSessionSnapshot(input.workspaceId, snapshot, input.replacements) if (input.notify !== false) { this.emitMobileSessionTabsSnapshot(stored) } diff --git a/src/main/runtime/orca-runtime-runtime-id.ts b/src/main/runtime/orca-runtime-runtime-id.ts index c872401f04ba..904aaa8c1b84 100644 --- a/src/main/runtime/orca-runtime-runtime-id.ts +++ b/src/main/runtime/orca-runtime-runtime-id.ts @@ -102,12 +102,13 @@ export class OrcaRuntimeWithRuntimeId { protected mobileSessionTabsByWorktree = new Map() - /** Single host writer for mobile session snapshots; versions are total-order stamps. */ + /** Single host writer for mobile snapshots; total-order versions; a loop passes `replacements`. */ protected storeMobileSessionSnapshot( worktreeId: string, - snapshot: RuntimeMobileSessionTabsSnapshot + snapshot: RuntimeMobileSessionTabsSnapshot, + replacements = getStructuredAgentSessionHost()?.conversationReplacements?.() ?? [] ): RuntimeMobileSessionTabsSnapshot { - for (const replacement of getStructuredAgentSessionHost()?.conversationReplacements?.() ?? []) { + for (const replacement of replacements) { snapshot = replaceConversationInSnapshot(snapshot, replacement) } const existing = this.mobileSessionTabsByWorktree.get(worktreeId) diff --git a/src/main/runtime/orca-runtime-structured-agent-session-startup-step.ts b/src/main/runtime/orca-runtime-structured-agent-session-startup-step.ts new file mode 100644 index 000000000000..3ebb91c74be5 --- /dev/null +++ b/src/main/runtime/orca-runtime-structured-agent-session-startup-step.ts @@ -0,0 +1,92 @@ +// The structured-chat startup step on the runtime: the host build, then the lease check, seed and +// settle, once per launch, and the history restore the tab restore owes once a listing has answered. + +import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' +import { createStructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' +import { OrcaRuntimeWithGetStructuredAgentSessionCreateSupport } from './orca-runtime-get-structured-agent-session-create-support' +import { runStructuredAgentSessionStartup } from './structured-agent-session-startup-step' + +export class OrcaRuntimeWithStructuredAgentSessionStartupStep extends OrcaRuntimeWithGetStructuredAgentSessionCreateSupport { + // The history restore a tab restore owes, until a caller that answered with its list starts it. + protected owedStructuredAgentSessionHistoryRestore: (() => void) | null = null + // Listed chats startup could not answer from stored state; null until it has run. + protected structuredAgentSessionBackgroundRestoreIds: string[] | null = null + protected structuredAgentSessionStartupStepPromise: Promise | null = null + private readonly structuredAgentSessionStartupLogger = createStructuredAgentSessionLogger() + + /** Starts the history restore the tab restore owes, once. On the next macrotask, so a caller that + * starts it as it answers has sent that answer first. */ + startStructuredAgentSessionHistoryRestore(): void { + const owed = this.owedStructuredAgentSessionHistoryRestore + this.owedStructuredAgentSessionHistoryRestore = null + if (owed) { + setImmediate(owed) + } + } + + /** The tab restore's preparation: the startup step, then the terminal records refresh, which + * lists the daemon's terminals against the records the host build brought in. */ + prepareStructuredAgentSessionStartupRestoration(): Promise { + this.structuredAgentSessionStartupRestorePromise ??= this.startStructuredAgentSessionStartup() + .then(async () => { + if (this.hasPersistedStructuredAgentSessionStore()) { + await this.refreshMobileSessionPtyRecords() + } + }) + .catch((error) => { + this.structuredAgentSessionStartupRestorePromise = null + throw error + }) + return this.structuredAgentSessionStartupRestorePromise + } + + /** + * Desktop launch. The step needs neither the terminal daemon nor the hook server (structured + * chats never run in WSL, the lease check probes processes, and the seed publishes in process), + * so only the shell PATH is awaited, as before: the host build sets up the agents' launch + * environments from it. Off Windows it is already resolved. + */ + startStructuredAgentSessionStartupAfter(shellPathReady: Promise): void { + void shellPathReady + .then(() => this.startStructuredAgentSessionStartup()) + .catch(this.reportStructuredAgentSessionStartupFailure) + } + + /** `prepare` once `after` resolves, its failure reported rather than thrown: desktop runs it once + * the first window's services are up or timed out, orcad at once. */ + prepareStructuredAgentSessionStartupRestorationAfter(after: Promise): void { + void after + .then(() => this.prepareStructuredAgentSessionStartupRestoration()) + .catch(this.reportStructuredAgentSessionStartupFailure) + } + + /** A failed host build or seed at launch: in the diagnostics trace, not only the console. */ + private reportStructuredAgentSessionStartupFailure = (error: unknown): void => { + this.structuredAgentSessionStartupLogger.warn('the chat startup step failed', { + scope: 'startup-step-failed', + error + }) + } + + /** The host's startup step, once: the host build, then the lease check, seed and settle. */ + startStructuredAgentSessionStartup(): Promise { + this.structuredAgentSessionStartupStepPromise ??= + this.startStructuredAgentSessionStartupOnce().catch((error) => { + this.structuredAgentSessionStartupStepPromise = null + throw error + }) + return this.structuredAgentSessionStartupStepPromise + } + + protected async startStructuredAgentSessionStartupOnce(): Promise { + const background = await runStructuredAgentSessionStartup({ + gate: this.structuredAgentSessionStartupGate, + hasChatsOnDisk: () => this.hasPersistedStructuredAgentSessionStore(), + buildHost: () => this.ensureStructuredAgentSessionHost(), + savedSession: () => this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null + }) + if (background) { + this.structuredAgentSessionBackgroundRestoreIds = background + } + } +} diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index 9852c83d17bd..3d15317ebcd1 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -1,9 +1,36 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' import { OrcaRuntimeService } from './orca-runtime' afterEach(() => setStructuredAgentSessionHost(null)) +/** The host members startup restoration reaches, beyond the seed and settle every test gets. */ +type StartupHostMembers = Partial< + Pick< + StructuredAgentSessionHost, + | 'reconcileRestartLeases' + | 'restoreReadableSessions' + | 'listSessionTabs' + | 'getPersistedVisibleSessionTabIndex' + | 'setSessionTabVisibility' + | 'close' + > +> + +/** A host offering the startup step's seed and settle, plus the members a test drives. */ +function installStartupHost(members: StartupHostMembers): void { + const host = { + startup: { + seedStoredStatuses: (ids: readonly string[]) => [...ids], + settleOwedSessions: async () => undefined + }, + ...members + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: startup restoration reaches only the members each test supplies. + setStructuredAgentSessionHost(host as unknown as StructuredAgentSessionHost) +} + describe('structured session cold restoration', () => { it('publishes a versioned title change and derives listAll titles from the record', async () => { const runtime = new OrcaRuntimeService() @@ -53,7 +80,9 @@ describe('structured session cold restoration', () => { internal.hasPersistedStructuredAgentSessionStore = () => false internal.refreshMobileSessionPtyRecords = refresh internal.ensureStructuredAgentSessionHost = ensureHost - setStructuredAgentSessionHost({ reconcileRestartLeases } as never) + installStartupHost({ + reconcileRestartLeases + }) await runtime.prepareStructuredAgentSessionStartupRestoration() @@ -76,7 +105,10 @@ describe('structured session cold restoration', () => { internal.hasPersistedStructuredAgentSessionStore = () => true internal.refreshMobileSessionPtyRecords = refresh internal.ensureStructuredAgentSessionHost = ensureHost - setStructuredAgentSessionHost({ reconcileRestartLeases, restoreReadableSessions } as never) + installStartupHost({ + reconcileRestartLeases, + restoreReadableSessions + }) await runtime.prepareStructuredAgentSessionStartupRestoration() @@ -86,7 +118,7 @@ describe('structured session cold restoration', () => { expect(restoreReadableSessions).not.toHaveBeenCalled() }) - it('loads records, inventories PTYs, restores ownership, then projects tabs exactly once', async () => { + it('loads records, then restores ownership and inventories PTYs, projects tabs, then restores history, once', async () => { const runtime = new OrcaRuntimeService() const hydrate = vi.fn() const refresh = vi.fn(async () => new Set()) @@ -108,16 +140,22 @@ describe('structured session cold restoration', () => { internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = hydrate internal.refreshMobileSessionPtyRecords = refresh internal.ensureStructuredAgentSessionHost = ensureHost - setStructuredAgentSessionHost({ + installStartupHost({ reconcileRestartLeases, restoreReadableSessions, listSessionTabs: () => [] - } as never) + }) const first = runtime.restoreStructuredAgentSessionTabs() const second = runtime.restoreStructuredAgentSessionTabs() expect(second).toBe(first) await Promise.all([first, second]) + // The history pass is owed, not started: the caller that answers with the list starts it. + expect(restoreReadableSessions).not.toHaveBeenCalled() + runtime.startStructuredAgentSessionHistoryRestore() + runtime.startStructuredAgentSessionHistoryRestore() + expect(restoreReadableSessions).not.toHaveBeenCalled() + await new Promise((resolve) => setImmediate(resolve)) expect(hydrate).toHaveBeenCalledWith('workspace-1', { allowAttachedWindow: true, @@ -128,17 +166,19 @@ describe('structured session cold restoration', () => { expect(reconcileRestartLeases).toHaveBeenCalledOnce() expect(restoreReadableSessions).toHaveBeenCalledOnce() expect(ensureHost).toHaveBeenCalledOnce() + // The lease check probes processes, not terminals, so it does not wait for the PTY inventory. expect(ensureHost.mock.invocationCallOrder[0]).toBeLessThan( refresh.mock.invocationCallOrder[0] ?? Infinity ) - expect(refresh.mock.invocationCallOrder[0]).toBeLessThan( + expect(ensureHost.mock.invocationCallOrder[0]).toBeLessThan( reconcileRestartLeases.mock.invocationCallOrder[0] ?? Infinity ) expect(reconcileRestartLeases.mock.invocationCallOrder[0]).toBeLessThan( restoreReadableSessions.mock.invocationCallOrder[0] ?? Infinity ) - expect(restoreReadableSessions.mock.invocationCallOrder[0]).toBeLessThan( - hydrate.mock.invocationCallOrder[0] ?? Infinity + // Tabs are projected first; history opens after, so no chat's history holds the list. + expect(hydrate.mock.invocationCallOrder[0]).toBeLessThan( + restoreReadableSessions.mock.invocationCallOrder[0] ?? Infinity ) }) @@ -168,7 +208,7 @@ describe('structured session cold restoration', () => { internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() internal.refreshMobileSessionPtyRecords = async () => new Set() internal.ensureStructuredAgentSessionHost = async () => undefined - setStructuredAgentSessionHost({ + installStartupHost({ reconcileRestartLeases: async () => undefined, getPersistedVisibleSessionTabIndex: () => ({ present: true, @@ -176,11 +216,14 @@ describe('structured session cold restoration', () => { }), restoreReadableSessions, listSessionTabs: () => [] - } as never) + }) await runtime.restoreStructuredAgentSessionTabs() + runtime.startStructuredAgentSessionHistoryRestore() - expect(restoreReadableSessions).toHaveBeenCalledWith(['session-survives-rollback']) + await vi.waitFor(() => + expect(restoreReadableSessions).toHaveBeenCalledWith(['session-survives-rollback']) + ) }) it('treats an empty durable visible-session index as authoritative', async () => { @@ -225,16 +268,17 @@ describe('structured session cold restoration', () => { internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() internal.refreshMobileSessionPtyRecords = async () => new Set() internal.ensureStructuredAgentSessionHost = async () => undefined - setStructuredAgentSessionHost({ + installStartupHost({ reconcileRestartLeases: async () => undefined, getPersistedVisibleSessionTabIndex: () => ({ present: true, sessionIds: [] }), restoreReadableSessions, listSessionTabs: () => [] - } as never) + }) await runtime.restoreStructuredAgentSessionTabs() + runtime.startStructuredAgentSessionHistoryRestore() - expect(restoreReadableSessions).toHaveBeenCalledWith([]) + await vi.waitFor(() => expect(restoreReadableSessions).toHaveBeenCalledWith([])) }) it('normalizes a restored tab id and removes it when closed', async () => { @@ -258,7 +302,7 @@ describe('structured session cold restoration', () => { internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() internal.refreshMobileSessionPtyRecords = async () => new Set() internal.ensureStructuredAgentSessionHost = async () => undefined - setStructuredAgentSessionHost({ + installStartupHost({ reconcileRestartLeases: async () => undefined, restoreReadableSessions: async () => undefined, close: closeStructuredSession, @@ -270,7 +314,7 @@ describe('structured session cold restoration', () => { agent: 'codex' } ] - } as never) + }) runtime.syncWindowGraph(1, { tabs: [], leaves: [], @@ -373,7 +417,7 @@ describe('structured session cold restoration', () => { internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() internal.refreshMobileSessionPtyRecords = async () => new Set() internal.ensureStructuredAgentSessionHost = async () => undefined - setStructuredAgentSessionHost({ + installStartupHost({ reconcileRestartLeases: async () => undefined, restoreReadableSessions: async () => undefined, listSessionTabs: () => [ @@ -383,7 +427,7 @@ describe('structured session cold restoration', () => { agent: 'claude' } ] - } as never) + }) await runtime.restoreStructuredAgentSessionTabs() @@ -392,7 +436,9 @@ describe('structured session cold restoration', () => { sessionId: 'restored-claude', agent: 'claude', activate: false, - notify: false + notify: false, + // Derived once by the restore for every tab it projects. + replacements: [] }) const restored = await runtime.listMobileSessionTabs('id:workspace-1') diff --git a/src/main/runtime/rpc/methods/session-tabs-inventory.ts b/src/main/runtime/rpc/methods/session-tabs-inventory.ts index be3a1d2de42c..354e23770912 100644 --- a/src/main/runtime/rpc/methods/session-tabs-inventory.ts +++ b/src/main/runtime/rpc/methods/session-tabs-inventory.ts @@ -5,7 +5,7 @@ import type { RpcContext } from '../core' import { projectSessionTabAgentStatus } from './session-tab-agent-status-projection' import { projectSessionTabBrowserPlacements } from './session-tab-browser-placement-projection' import { createSessionTabsRetirementProofDelta } from './session-tabs-retirement-proof-delta' -import { restoreStructuredTabsIfSupported } from './structured-session-tab-restore' +import { answerAfterStructuredTabRestore } from './structured-session-tab-restore' type SessionTabsInventory = { snapshots: RuntimeMobileSessionTabsResult[] @@ -230,37 +230,30 @@ export async function subscribeSessionTabsInventory( } let collected: Awaited> | undefined try { - // Why: restore after registering, so an unsubscribe or socket close while it runs still finds the stream. - const restoring = restoreStructuredTabsIfSupported(context) - if (restoring) { - await restoring - if (closed) { - return - } - } - for (let attempt = 1; !collected; attempt += 1) { - censusInvalidated = false - const candidate = await collectSessionTabsInventory( - { ...context, signal: inventoryController.signal }, - true - ) - if (closed) { - return - } - if (censusInvalidated) { + // Why: restore after registering, so an unsubscribe or socket close while it runs still finds the + // stream. The snapshots emit right after this settles, before the history restore it starts. + collected = await answerAfterStructuredTabRestore(context, async () => { + for (let attempt = 1; !closed; attempt += 1) { + censusInvalidated = false + const candidate = await collectSessionTabsInventory( + { ...context, signal: inventoryController.signal }, + true + ) + if (!censusInvalidated || closed) { + return candidate + } clearBufferedChanges() if (attempt === MAX_CENSUS_COLLECTION_ATTEMPTS) { throw new Error('session_tabs_inventory_unstable') } - } else { - collected = candidate } - } + return undefined + }) } catch (error) { runtime.cleanupSubscription(subscriptionId) throw error } - if (closed) { + if (closed || !collected) { return } const { inventory, changeSequence } = collected diff --git a/src/main/runtime/rpc/methods/session-tabs.ts b/src/main/runtime/rpc/methods/session-tabs.ts index d549daffbaf8..d51252eaa439 100644 --- a/src/main/runtime/rpc/methods/session-tabs.ts +++ b/src/main/runtime/rpc/methods/session-tabs.ts @@ -15,7 +15,7 @@ import { import { SESSION_TAB_MARKDOWN_METHODS } from './session-tab-markdown-methods' import { SESSION_TAB_MUTATION_METHODS } from './session-tab-mutation-methods' import { createSessionTabsRetirementProofDelta } from './session-tabs-retirement-proof-delta' -import { restoreStructuredTabsIfSupported } from './structured-session-tab-restore' +import { answerAfterStructuredTabRestore } from './structured-session-tab-restore' import { assertLegacyAiVaultResumeCommandAllowed } from '../../../ai-vault/structured-session-ownership' import { SessionTabsUnsubscribeAllParams } from '../../../../shared/rpc-contract/session-tabs-params' import { SESSION_TABS_SPLIT_GROUP_PLACEMENT_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' @@ -24,22 +24,20 @@ export const SESSION_TAB_METHODS = [ defineMethod({ name: 'session.tabs.list', params: WorktreeTabSelector, - handler: async (params, { runtime, pairedDeviceId, clientKind, clientCapabilities }) => { - await restoreStructuredTabsIfSupported({ runtime, clientKind, clientCapabilities }) - return projectSessionTabsForClient( - await runtime.listMobileSessionTabs(params.worktree, pairedDeviceId), - clientKind, - clientCapabilities + handler: (params, { runtime, pairedDeviceId, clientKind, clientCapabilities }) => + answerAfterStructuredTabRestore({ runtime, clientKind, clientCapabilities }, async () => + projectSessionTabsForClient( + await runtime.listMobileSessionTabs(params.worktree, pairedDeviceId), + clientKind, + clientCapabilities + ) ) - } }), defineMethod({ name: 'session.tabs.listAll', params: null, - handler: async (_params, context) => { - await restoreStructuredTabsIfSupported(context) - return listSessionTabsInventory(context) - } + handler: (_params, context) => + answerAfterStructuredTabRestore(context, () => listSessionTabsInventory(context)) }), ...SESSION_TAB_MUTATION_METHODS, ...SESSION_TAB_CLOSE_METHODS, @@ -142,39 +140,46 @@ export const SESSION_TAB_METHODS = [ register(explicitWorktreeId) } try { - await restoreStructuredTabsIfSupported({ runtime, clientKind, clientCapabilities }) - if (released) { - return - } - const initial = await runtime.listMobileSessionTabs(params.worktree, pairedDeviceId) - if (released) { - return - } - if (!subscriptionId) { - register(initial.worktree) - if (released) { - return - } - } - const subscribedWorktree = initial.worktree - const withProofDelta = createSessionTabsRetirementProofDelta(clientCapabilities) - emit({ - type: 'snapshot', - ...withProofDelta(projectSessionTabsForClient(initial, clientKind, clientCapabilities)) - }) - if (released) { - return - } - stopListening = runtime.onMobileSessionTabsChanged((snapshot) => { - if (snapshot.worktree === subscribedWorktree) { + // The first snapshot emits inside this, before the history restore it starts. + await answerAfterStructuredTabRestore( + { runtime, clientKind, clientCapabilities }, + async () => { + if (released) { + return + } + const initial = await runtime.listMobileSessionTabs(params.worktree, pairedDeviceId) + if (released) { + return + } + if (!subscriptionId) { + register(initial.worktree) + if (released) { + return + } + } + const subscribedWorktree = initial.worktree + const withProofDelta = createSessionTabsRetirementProofDelta(clientCapabilities) emit({ - type: 'updated', + type: 'snapshot', ...withProofDelta( - projectSessionTabsForClient(snapshot, clientKind, clientCapabilities) + projectSessionTabsForClient(initial, clientKind, clientCapabilities) ) }) + if (released) { + return + } + stopListening = runtime.onMobileSessionTabsChanged((snapshot) => { + if (snapshot.worktree === subscribedWorktree) { + emit({ + type: 'updated', + ...withProofDelta( + projectSessionTabsForClient(snapshot, clientKind, clientCapabilities) + ) + }) + } + }, pairedDeviceId) } - }, pairedDeviceId) + ) } catch (error) { // A stream already ended by its release must not also report an error. if (released) { diff --git a/src/main/runtime/rpc/methods/structured-agent-session-queued-methods.ts b/src/main/runtime/rpc/methods/structured-agent-session-queued-methods.ts index 259ee2dbe148..3b8d31a81be8 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-queued-methods.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-queued-methods.ts @@ -4,7 +4,7 @@ import { defineMethod } from '../core' import { - requireStructuredHost as requireHost, + requireInstalledStructuredHost as commandHost, structuredCallerFor as callerFor } from './structured-agent-session-gate' import { @@ -16,16 +16,19 @@ export const STRUCTURED_AGENT_SESSION_QUEUED_METHODS = [ defineMethod({ name: 'agentSession.queuedMessageSend', params: QueuedMessageActionParams, - handler: async (params, ctx) => requireHost(ctx).queuedMessageSend(callerFor(ctx), params) + handler: async (params, ctx) => + (await commandHost(ctx)).queuedMessageSend(callerFor(ctx), params) }), defineMethod({ name: 'agentSession.queuedMessageDelete', params: QueuedMessageActionParams, - handler: async (params, ctx) => requireHost(ctx).queuedMessageDelete(callerFor(ctx), params) + handler: async (params, ctx) => + (await commandHost(ctx)).queuedMessageDelete(callerFor(ctx), params) }), defineMethod({ name: 'agentSession.queuedMessagesResume', params: QueuedMessagesResumeParams, - handler: async (params, ctx) => requireHost(ctx).queuedMessagesResume(callerFor(ctx), params) + handler: async (params, ctx) => + (await commandHost(ctx)).queuedMessagesResume(callerFor(ctx), params) }) ] diff --git a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts index 43f813b35ee6..5440420b9b2a 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts @@ -5,6 +5,7 @@ import { vi } from 'vitest' import type { AgentJournalRenderItem } from '../../../../shared/agent-session-journal-types' import type { AgentSessionJournal } from '../../../native-chat/agent-session-journal/journal-store' +import { projectStructuredAgentSessionStatusState } from '../../../../shared/structured-agent-session-projection' import type { StructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-host' import { setStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' import { @@ -101,12 +102,16 @@ function statusFeed(): StructuredAgentSessionStatusFeed { [ STATUS_SESSION, { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks and snapshot() of a journal. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a stand-in journal offering only what the status feed reads (submissions, cursor, Stop marks, its projection, activity). journal: { cursor: () => ({ epoch: 'epoch-status', sequence: 2 }), lastActivityAt: () => 2, + submissions: () => [], stopMarks: { latest: () => null, revision: () => 0 }, - snapshot: () => ({ items: STATUS_ITEMS, submissions: [] }) + sessionStatus: { + at: (fence?: number) => + projectStructuredAgentSessionStatusState(STATUS_ITEMS, [], fence) + } } as unknown as AgentSessionJournal, params: { location: { diff --git a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts index ed989fb83e83..733f491021c8 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts @@ -1,7 +1,7 @@ // Which clients get a send answered at acceptance, and which have their reply held until the // message is handed over: a client that cannot show a rejection after `pending` must not see one. -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' import { ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES } from '../../../../shared/electron-remote-runtime-client-capabilities' import { DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES } from '../../../ipc/desktop-renderer-runtime-capabilities' @@ -9,6 +9,7 @@ import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/age import { call, clearStructuredHostStub, + envelope, hostCalls, installStructuredHostStub, SESSION, @@ -58,6 +59,44 @@ describe('agentSession.send reply timing', () => { }) }) +describe('a send that arrives before the host is built', () => { + it('builds the host and is answered by it, rather than refused', async () => { + clearStructuredHostStub() + hostCalls.send?.mockReset() + const ensureStructuredAgentSessionHost = vi.fn(async () => installStructuredHostStub()) + + const response = await call('agentSession.send', sendParams(), STRUCTURED_CLIENT, { + ensureStructuredAgentSessionHost + }) + + expect(ensureStructuredAgentSessionHost).toHaveBeenCalledOnce() + expect(response).not.toMatchObject({ ok: false }) + expect(hostCalls.send).toHaveBeenCalledOnce() + }) + + it.each([ + ['agentSession.queuedMessageSend', 'queuedMessageSend', { messageId: 'message-1' }], + ['agentSession.queuedMessageDelete', 'queuedMessageDelete', { messageId: 'message-1' }], + ['agentSession.queuedMessagesResume', 'queuedMessagesResume', {}], + ['agentSession.threadGoal', 'changeThreadGoal', { change: { kind: 'clear' } }] + ])('builds the host for %s too, and is answered by it', async (method, hostMethod, fields) => { + clearStructuredHostStub() + const answered = vi.fn(async () => ({ ok: true, replayed: false })) + const ensureStructuredAgentSessionHost = vi.fn(async () => { + installStructuredHostStub() + hostCalls[hostMethod] = answered + }) + + const response = await call(method, { envelope: envelope(), ...fields }, STRUCTURED_CLIENT, { + ensureStructuredAgentSessionHost + }) + + expect(ensureStructuredAgentSessionHost).toHaveBeenCalledOnce() + expect(response).not.toMatchObject({ ok: false }) + expect(answered).toHaveBeenCalledOnce() + }) +}) + function pendingSendResult() { return { ok: true, diff --git a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts index 019c0d63453f..094dbb6b34cc 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts @@ -6,7 +6,10 @@ import { agentSessionSendSubmission } from '../../../../shared/agent-session-wir import type { StructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-host' import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement' import type { RpcContext } from '../core' -import { requireStructuredHost, structuredCallerFor } from './structured-agent-session-gate' +import { + requireInstalledStructuredHost, + structuredCallerFor +} from './structured-agent-session-gate' /** * A send answers once the host accepts it. A client that predates that answer cannot show a @@ -19,7 +22,9 @@ export async function sendStructuredAgentSessionForClient( params: Parameters[1], context: RpcContext ) { - const host = requireStructuredHost(context) + // Built first when this process has none yet, so a send that arrives during startup is held + // for the startup settle instead of refused. + const host = await requireInstalledStructuredHost(context) // A client's own send: a restart or a close keeps it as a card if it never reached the agent. const result = await host.send(structuredCallerFor(context), { ...params, userSend: true }) const capabilities = context.clientCapabilities ?? [] diff --git a/src/main/runtime/rpc/methods/structured-agent-session-thread-goal.ts b/src/main/runtime/rpc/methods/structured-agent-session-thread-goal.ts index b28a51500d21..bde2f9ddd769 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-thread-goal.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-thread-goal.ts @@ -5,7 +5,7 @@ import { defineMethod } from '../core' import { - requireStructuredHost as requireHost, + requireInstalledStructuredHost as commandHost, structuredCallerFor as callerFor } from './structured-agent-session-gate' import { ThreadGoalParams } from './structured-agent-session-schemas' @@ -14,6 +14,7 @@ export const STRUCTURED_AGENT_SESSION_THREAD_GOAL_METHODS = [ defineMethod({ name: 'agentSession.threadGoal', params: ThreadGoalParams, - handler: async (params, ctx) => requireHost(ctx).changeThreadGoal(callerFor(ctx), params) + handler: async (params, ctx) => + (await commandHost(ctx)).changeThreadGoal(callerFor(ctx), params) }) ] diff --git a/src/main/runtime/rpc/methods/structured-agent-session.ts b/src/main/runtime/rpc/methods/structured-agent-session.ts index d2da7b0f92c2..71ebf1f75bcd 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session.ts @@ -20,6 +20,8 @@ import { defineMethod, defineStreamingMethod, type RpcContext } from '../core' import { ensureStructuredHostInstalled as ensureHostInstalled, requireInstalledStructuredHost as requireInstalledHost, + // A chat command builds the host when it is this process's first: held for startup, not refused. + requireInstalledStructuredHost as commandHost, requireStructuredCapability, requireStructuredCleanupHost, requireStructuredCreateSupportAdmission, @@ -231,18 +233,18 @@ export const STRUCTURED_AGENT_SESSION_METHODS = [ name: 'agentSession.respondToApproval', params: RespondParams, handler: async (params, ctx) => - requireHost(ctx).respondToPrompt(callerFor(ctx), { ...params, kind: 'approval' }) + (await commandHost(ctx)).respondToPrompt(callerFor(ctx), { ...params, kind: 'approval' }) }), defineMethod({ name: 'agentSession.respondToQuestion', params: RespondToQuestionParams, handler: async (params, ctx) => - requireHost(ctx).respondToPrompt(callerFor(ctx), { ...params, kind: 'question' }) + (await commandHost(ctx)).respondToPrompt(callerFor(ctx), { ...params, kind: 'question' }) }), defineMethod({ name: 'agentSession.setOption', params: SetOptionParams, - handler: async (params, ctx) => requireHost(ctx).setOption(callerFor(ctx), params) + handler: async (params, ctx) => (await commandHost(ctx)).setOption(callerFor(ctx), params) }), defineMethod({ name: 'agentSession.handoffStatus', diff --git a/src/main/runtime/rpc/methods/structured-session-tab-restore.ts b/src/main/runtime/rpc/methods/structured-session-tab-restore.ts index 81ecde983603..d5c20fe85c8c 100644 --- a/src/main/runtime/rpc/methods/structured-session-tab-restore.ts +++ b/src/main/runtime/rpc/methods/structured-session-tab-restore.ts @@ -7,7 +7,7 @@ import { supportsStructuredAgentSessions } from './structured-agent-session-poli * of each chat, and gating on capability left it with nothing to project after a desktop restart — * no chat and no prompt. Restoring spawns no provider child for a cleanly closed session, and a * host with no saved sessions returns before building anything. */ -export function restoreStructuredTabsIfSupported( +function restoreStructuredTabsIfSupported( context: Pick ): Promise | undefined { const shouldRestore = context.clientKind === 'mobile' || supportsStructuredAgentSessions(context) @@ -17,3 +17,22 @@ export function restoreStructuredTabsIfSupported( // Nothing to restore: callers skip the await, so a stream's setup keeps its timing. return undefined } + +/** The tab restore, then the caller's answer. The chat history restore the tab restore owes starts + * only once that answer has settled (sent, failed or abandoned), so the answer's own work, such as + * the PTY census, never waits behind it. The first caller to settle starts it. */ +export async function answerAfterStructuredTabRestore( + context: Pick, + answer: () => Promise +): Promise { + const restoring = restoreStructuredTabsIfSupported(context) + if (!restoring) { + return answer() + } + try { + await restoring + return await answer() + } finally { + context.runtime.startStructuredAgentSessionHistoryRestore?.() + } +} diff --git a/src/main/runtime/structured-agent-session-codex-stopped-send-order.test.ts b/src/main/runtime/structured-agent-session-codex-stopped-send-order.test.ts index 3824f6edb6de..4372f0ed1657 100644 --- a/src/main/runtime/structured-agent-session-codex-stopped-send-order.test.ts +++ b/src/main/runtime/structured-agent-session-codex-stopped-send-order.test.ts @@ -62,6 +62,9 @@ let resumeTurns: unknown[] | null let turns: ReturnType /** Codex takes the interrupt and answers it; the test streams more and sends the end later. */ let interruptEndHeld: boolean +/** Resolved once the test opens the turn: Codex takes no interrupt before it has a turn to take. */ +let turnOpened: Promise +let openTurn: () => void let operations = 0 /** The durable ledger stamps its own clock and refuses an id far from it. */ @@ -150,6 +153,7 @@ const openConnection: typeof openCodexAppServerConnection = async ( return turns.routes['turn/steer'](params) } if (method === 'turn/interrupt' && interruptEndHeld) { + await turnOpened turns.takeInterrupt() return {} } @@ -204,6 +208,9 @@ beforeEach(async () => { resumes = 0 resumeTurns = null interruptEndHeld = false + turnOpened = new Promise((resolve) => { + openTurn = resolve + }) turns = codexTurnLifecycleFake( THREAD, () => (method, params) => handlers?.onNotification?.(method, params) @@ -230,6 +237,7 @@ describe("a send made while Codex held the first send's answer, then a Stop", () const secondId = await second await vi.waitFor(async () => expect(verdictOf(await submissions(), secondId)).toBe('withdrawn')) turns.start() + openTurn() turns.echo(first) // The turn opened after the second send was accepted, and streams after the Stop was pressed. handlers?.onNotification?.('item/completed', { diff --git a/src/main/runtime/structured-agent-session-record-import-install.test.ts b/src/main/runtime/structured-agent-session-record-import-install.test.ts index 8495fb49f5de..8520c823debe 100644 --- a/src/main/runtime/structured-agent-session-record-import-install.test.ts +++ b/src/main/runtime/structured-agent-session-record-import-install.test.ts @@ -9,6 +9,7 @@ import { afterEach, beforeEach, expect, it, vi } from 'vitest' import { agentSessionRecordFixture } from '../../shared/agent-session-record.test-fixture' import Database from '../sqlite/sync-database' import { journalPragmaNumber } from '../native-chat/agent-session-journal/journal-database' +import { JOURNAL_DB_SCHEMA_VERSION } from '../native-chat/agent-session-journal/journal-database-schema' import { journalDatabasePath } from '../native-chat/agent-session-journal/journal-host-database' import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' import { legacyAgentSessionStorePath } from './agent-session-record-store-file' @@ -151,7 +152,7 @@ it.skipIf(process.platform === 'win32' || process.getuid?.() === 0)( expect(relaunched.legacyRecordImportOwed()).toBe(false) expect(second.unverifiable()).toBe(false) - expect(databaseVersion()).toBe(4) + expect(databaseVersion()).toBe(JOURNAL_DB_SCHEMA_VERSION) expect(relaunched.deps.store.getRecord(IMPORTED)).not.toBeNull() expect(relaunched.deps.store.getRecord(CREATED_WHILE_OWED)).not.toBeNull() } @@ -168,7 +169,7 @@ it('installs over a file no read will make usable, reports it once, and leaves i { kind: 'unusable', error: expect.objectContaining({ message: 'agent_session_store_corrupt' }) } ]) expect((await host()).legacyRecordImportOwed()).toBe(false) - expect(databaseVersion()).toBe(4) + expect(databaseVersion()).toBe(JOURNAL_DB_SCHEMA_VERSION) await stopStructuredAgentSessionRuntime() expect(await readFile(legacyAgentSessionStorePath(root), 'utf-8')).toBe('{ truncated') }) diff --git a/src/main/runtime/structured-agent-session-runtime.ts b/src/main/runtime/structured-agent-session-runtime.ts index 7a833a74d530..40a3cee80f51 100644 --- a/src/main/runtime/structured-agent-session-runtime.ts +++ b/src/main/runtime/structured-agent-session-runtime.ts @@ -135,6 +135,8 @@ export type StructuredAgentSessionRuntimeDeps = { statusSink?: StructuredAgentSessionHostDeps['statusSink'] /** See `StructuredAgentSessionHostDeps.hasOpenDispatch`. */ hasOpenDispatch?: StructuredAgentSessionHostDeps['hasOpenDispatch'] + /** See `StructuredAgentSessionHostDeps.commandsReady`. */ + commandsReady?: StructuredAgentSessionHostDeps['commandsReady'] /** See `StructuredAgentSessionHostDeps.onSessionTabHidden`. */ onSessionTabHidden?: StructuredAgentSessionHostDeps['onSessionTabHidden'] /** Host-owned phone delivery and reconciliation from the current journal projection. */ @@ -306,6 +308,7 @@ async function installOnJournal( ...(deps.onSessionStatusChanged ? { onSessionStatusChanged: deps.onSessionStatusChanged } : {}), ...(deps.statusSink ? { statusSink: deps.statusSink } : {}), ...(deps.hasOpenDispatch ? { hasOpenDispatch: deps.hasOpenDispatch } : {}), + ...(deps.commandsReady ? { commandsReady: deps.commandsReady } : {}), ...(deps.onSessionTabHidden ? { onSessionTabHidden: deps.onSessionTabHidden } : {}), ...(await modelCatalogHostDeps({ store, agents, deps, envResolvers })) }) diff --git a/src/main/runtime/structured-agent-session-startup-gate.ts b/src/main/runtime/structured-agent-session-startup-gate.ts new file mode 100644 index 000000000000..256e7e8592be --- /dev/null +++ b/src/main/runtime/structured-agent-session-startup-gate.ts @@ -0,0 +1,123 @@ +// Holds every structured chat command until host startup has settled the chats a gone process left +// with work. Every open settles its own chat too, so what the hold adds is order: each crashed turn +// gets its verdict from the death evidence startup records first, never "Couldn't confirm" from an +// open that ran before it. Commands wait at their entry, before any chat's lock (the settle takes +// those locks); none is refused. It always opens: when the settle ends, however it ends, or at a +// ceiling, so a startup fault never strands a command. + +import { + createStructuredAgentSessionLogger, + type StructuredAgentSessionLogger +} from '../native-chat/agent-session-wire/structured-agent-session-logger' + +// Under the shortest client timeout on a held call, with margin: the AI vault's chat restore gives +// its reveal 5 s (activate-ai-vault-structured-session.ts); remote desktop and phone give 15 s. +// Counted from launch (`hold`). A held command lets go before its caller gives up, and an open +// settles its own chat, so letting go early is safe. Past it, the order is what is lost: a crashed +// turn may get its verdict from an open. +const STARTUP_GATE_CEILING_MS = 4_000 + +/** How the startup step ended. */ +export type StartupStepOutcome = + | 'settle ended' + | 'settle failed' + | 'step failed' + | 'no chats on disk' + | 'no host' + +type StartupGateOpener = StartupStepOutcome | 'ceiling' + +export class StructuredAgentSessionStartupGate { + private pending: Promise | null = null + private release: (() => void) | null = null + private ceiling: ReturnType | null = null + private held = false + // One timing line per launch, for the startup measurements. The step's times are recorded + // whenever they happen, whoever opened the gate; the line is written once the gate is open and + // the step is over. + private heldAt: number | null = null + private step: { + startedAt: number + ended: { at: number; outcome: StartupStepOutcome } | null + } | null = null + private opened: { at: number; by: StartupGateOpener } | null = null + + constructor( + private readonly ceilingMs = STARTUP_GATE_CEILING_MS, + private readonly now: () => number = Date.now, + private readonly logger: StructuredAgentSessionLogger = createStructuredAgentSessionLogger() + ) {} + + /** Closes the gate until it is opened, once per launch, before any request is served. */ + hold(): void { + // One-shot: re-closing after it opened could hold /clear's replacement attach under a chat lock. + if (this.held) { + return + } + this.held = true + this.heldAt = this.now() + this.pending = new Promise((resolve) => { + this.release = resolve + }) + this.ceiling = setTimeout(() => { + this.logger.warn('startup settle still running; chat commands go ahead', { + scope: 'startup-gate-ceiling' + }) + this.open('ceiling') + }, this.ceilingMs) + this.ceiling.unref?.() + } + + /** The startup step began; only timed. */ + stepStarted(): void { + this.step ??= { startedAt: this.now(), ended: null } + } + + /** The startup step is over: timed, and the gate opens now unless the ceiling opened it. */ + stepEnded(outcome: StartupStepOutcome): void { + const at = this.now() + this.step ??= { startedAt: at, ended: null } + this.step.ended ??= { at, outcome } + this.open(outcome) + } + + /** The step ends when the settle does, whether it settled or failed. */ + openWhen(settled: Promise): void { + void settled.then( + () => this.stepEnded('settle ended'), + (error: unknown) => { + this.logger.warn('startup settle failed', { scope: 'startup-settle', error }) + this.stepEnded('settle failed') + } + ) + } + + /** What a chat command waits for, or null when the gate is open. */ + ready = (): Promise | null => this.pending + + private open(by: StartupGateOpener): void { + if (this.ceiling) { + clearTimeout(this.ceiling) + this.ceiling = null + } + if (this.pending) { + this.opened = { at: this.now(), by } + } + this.release?.() + this.release = null + this.pending = null + this.report() + } + + private report(): void { + const { heldAt, step, opened } = this + if (heldAt === null || !opened || !step?.ended) { + return + } + console.info( + `[structured-agent-session] startup gate: held at launch; step started +${step.startedAt - heldAt} ms, ` + + `ended +${step.ended.at - heldAt} ms (${step.ended.outcome}); opened +${opened.at - heldAt} ms by ${opened.by}` + ) + this.heldAt = null + } +} diff --git a/src/main/runtime/structured-agent-session-startup-restoration.test.ts b/src/main/runtime/structured-agent-session-startup-restoration.test.ts new file mode 100644 index 000000000000..c289eb651662 --- /dev/null +++ b/src/main/runtime/structured-agent-session-startup-restoration.test.ts @@ -0,0 +1,279 @@ +// The host's own startup step, run from `prepare` whether or not any client ever lists a tab: a +// headless host has its seeded statuses and its settled crashed chats, and no failure in the step +// (one chat's open, one chat's settlement) costs startup or the other chats. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + closeTestJournalHostDatabases, + readTestJournalSessionStatus +} from '../native-chat/agent-session-journal/journal-host-database-test-support' +import { isUnsettledJournalSessionStatus } from '../native-chat/agent-session-journal/journal-session-state' +import { AgentSessionJournal } from '../native-chat/agent-session-journal/journal-store' +import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import { + createRestTestRig, + restTestChat, + type RestTestRig +} from '../native-chat/agent-session-wire/structured-agent-session-rest-test-rig' +import { + latestRestTestStatus, + restTestOpens +} from '../native-chat/agent-session-wire/structured-agent-session-rest-test-observations' +import { OrcaRuntimeService } from './orca-runtime' +import { StructuredAgentSessionStartupGate } from './structured-agent-session-startup-gate' +import type { StructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' + +type PrepareInternals = { + structuredAgentSessionStartupGate: StructuredAgentSessionStartupGate + structuredAgentSessionStartupLogger: StructuredAgentSessionLogger + store: { getWorkspaceSession: () => unknown } + hasPersistedStructuredAgentSessionStore(): boolean + refreshMobileSessionPtyRecords(): Promise | null> + ensureStructuredAgentSessionHost(): Promise +} + +let rig: RestTestRig + +beforeEach(async () => { + rig = await createRestTestRig() +}) + +afterEach(async () => { + setStructuredAgentSessionHost(null) + await rig.dispose() + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +function internals(runtime: OrcaRuntimeService): PrepareInternals { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these members exist on the runtime; they are protected, not absent. + return runtime as unknown as PrepareInternals +} + +/** A restarted runtime over the rig's host, with nothing but `prepare` ever called on it. */ +function restartedRuntime(): OrcaRuntimeService { + const runtime = new OrcaRuntimeService() + const internal = internals(runtime) + internal.store = { getWorkspaceSession: () => null } + internal.hasPersistedStructuredAgentSessionStore = () => true + internal.refreshMobileSessionPtyRecords = async () => new Set() + internal.ensureStructuredAgentSessionHost = async () => { + setStructuredAgentSessionHost(rig.host) + } + return runtime +} + +async function crashMidSend(sessionId: string, listed = true): Promise { + rig.adapter.dispatch.mockResolvedValueOnce({ state: 'admitted' }) + await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, listed }) +} + +function owes(sessionId: string): boolean | undefined { + const stored = readTestJournalSessionStatus(rig.root, sessionId) + return stored ? isUnsettledJournalSessionStatus(stored) : undefined +} + +it('holds chat commands until the startup settle ends, and then lets them through', async () => { + await crashMidSend('session-crashed') + await rig.crash() + await rig.boot() + const runtime = restartedRuntime() + const gate = internals(runtime).structuredAgentSessionStartupGate + const timing = vi.spyOn(console, 'info').mockImplementation(() => undefined) + runtime.holdStructuredAgentSessionCommandsForStartup() + expect(gate.ready()).not.toBeNull() + + await runtime.prepareStructuredAgentSessionStartupRestoration() + await vi.waitFor(() => expect(owes('session-crashed')).toBe(false)) + + await vi.waitFor(() => expect(gate.ready()).toBeNull()) + expect(timing).toHaveBeenCalledWith( + expect.stringMatching( + /step started \+\d+ ms, ended \+\d+ ms \(settle ended\); opened \+\d+ ms by settle ended$/ + ) + ) +}) + +it.each([ + ['throws', 'step failed'], + ['builds no host', 'no host'] +] as const)( + 'opens the gate when the host build %s, never stranding a command', + async (how, outcome) => { + await rig.crash() + await rig.boot() + const runtime = restartedRuntime() + const gate = internals(runtime).structuredAgentSessionStartupGate + internals(runtime).ensureStructuredAgentSessionHost = async () => { + if (how === 'throws') { + throw new Error('host refused') + } + } + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + runtime.holdStructuredAgentSessionCommandsForStartup() + + const timing = vi.spyOn(console, 'info').mockImplementation(() => undefined) + await runtime.prepareStructuredAgentSessionStartupRestoration().catch(() => undefined) + + expect(gate.ready()).toBeNull() + expect(timing).toHaveBeenCalledOnce() + expect(timing).toHaveBeenCalledWith( + expect.stringMatching(new RegExp(`\\(${outcome}\\); opened \\+\\d+ ms by ${outcome}$`)) + ) + } +) + +it('lets held commands go before the shortest client timeout on a held call (the AI vault restore, 5 s)', () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.useFakeTimers() + try { + const gate = new StructuredAgentSessionStartupGate() + gate.hold() + vi.advanceTimersByTime(4_500) + expect(gate.ready()).toBeNull() + } finally { + vi.useRealTimers() + } +}) + +it('reports a failed startup step to the diagnostics logger, not only the console', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.spyOn(console, 'info').mockImplementation(() => undefined) + const runtime = restartedRuntime() + const internal = internals(runtime) + const failure = new Error('host build failed') + internal.ensureStructuredAgentSessionHost = async () => { + throw failure + } + const warn = vi.spyOn(internal.structuredAgentSessionStartupLogger, 'warn') + + runtime.startStructuredAgentSessionStartupAfter(Promise.resolve()) + + await vi.waitFor(() => + expect(warn).toHaveBeenCalledWith('the chat startup step failed', { + scope: 'startup-step-failed', + error: failure + }) + ) +}) + +it('closes the gate once per launch: a hold after it opened leaves it open', async () => { + const gate = new StructuredAgentSessionStartupGate() + gate.hold() + gate.hold() + vi.spyOn(console, 'info').mockImplementation(() => undefined) + gate.stepStarted() + gate.stepEnded('no chats on disk') + expect(gate.ready()).toBeNull() + + gate.hold() + + expect(gate.ready()).toBeNull() +}) + +describe('one timing line per launch, with the real step times whoever opened the gate', () => { + const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + const CEILING_MS = 5 + let timing: ReturnType + + beforeEach(() => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + timing = vi.spyOn(console, 'info').mockImplementation(() => undefined) + }) + + /** Exactly one line, for a gate the ceiling opened before the step ended `outcome`. */ + async function expectOneCeilingLine(outcome: string, started = /\+\d+ ms/): Promise { + await vi.waitFor(() => expect(timing).toHaveBeenCalledOnce()) + await sleep(20) + expect(timing).toHaveBeenCalledOnce() + const line = String(timing.mock.calls[0]?.[0]) + expect(line).toMatch( + new RegExp(`ended \\+\\d+ ms \\(${outcome}\\); opened \\+\\d+ ms by ceiling$`) + ) + expect(line).toMatch(new RegExp(`step started ${started.source},`)) + } + + it.each(['settle ended', 'no host', 'step failed'] as const)( + 'writes it when the step ends after the ceiling: %s', + async (outcome) => { + const gate = new StructuredAgentSessionStartupGate(CEILING_MS) + gate.hold() + gate.stepStarted() + await vi.waitFor(() => expect(gate.ready()).toBeNull()) + expect(timing).not.toHaveBeenCalled() + + gate.stepEnded(outcome) + + await expectOneCeilingLine(outcome) + } + ) + + it('times a step that starts only after the ceiling', async () => { + const gate = new StructuredAgentSessionStartupGate(CEILING_MS) + gate.hold() + await vi.waitFor(() => expect(gate.ready()).toBeNull()) + await sleep(10) + expect(timing).not.toHaveBeenCalled() + + gate.stepStarted() + gate.openWhen(Promise.resolve()) + + await expectOneCeilingLine('settle ended', /\+([5-9]|\d{2,}) ms/) + }) +}) + +it('seeds and settles on a host no client ever lists (T8)', async () => { + await restTestChat(rig, 'session-settled', { message: 'done' }) + await crashMidSend('session-crashed') + await crashMidSend('session-crashed-closed', false) + await rig.crash() + await rig.boot() + const runtime = restartedRuntime() + + await runtime.prepareStructuredAgentSessionStartupRestoration() + + expect(latestRestTestStatus(rig, 'session-settled')).toMatchObject({ status: 'idle' }) + await vi.waitFor(() => { + expect(owes('session-crashed')).toBe(false) + expect(owes('session-crashed-closed')).toBe(false) + }) + expect(rig.host.hasSession('session-crashed')).toBe(true) + expect(rig.host.hasSession('session-crashed-closed')).toBe(false) + expect(restTestOpens(rig, 'session-settled')).toBe(0) +}) + +it('never fails startup: each failure is logged by chat and the rest still settle (T18)', async () => { + await restTestChat(rig, 'session-settled', { message: 'done' }) + for (const sessionId of ['session-open-fails', 'session-settle-fails', 'session-fine']) { + await crashMidSend(sessionId) + } + await rig.crash() + await rig.boot() + const warn = vi.spyOn(rig.host.deps.logger, 'warn') + rig.journalOpens.mockImplementation(async (sessionId) => { + if (sessionId === 'session-open-fails') { + throw new Error('EACCES: permission denied') + } + }) + const markUnknown = AgentSessionJournal.prototype.markPendingSubmissionsUnknown + vi.spyOn(AgentSessionJournal.prototype, 'markPendingSubmissionsUnknown').mockImplementation( + function (this: AgentSessionJournal, ...args) { + if (this.snapshot().sessionId === 'session-settle-fails') { + return Promise.reject(new Error('settlement append failed')) + } + return markUnknown.apply(this, args) + } + ) + const runtime = restartedRuntime() + + await runtime.prepareStructuredAgentSessionStartupRestoration() + + expect(latestRestTestStatus(rig, 'session-settled')).toMatchObject({ status: 'idle' }) + await vi.waitFor(() => expect(owes('session-fine')).toBe(false)) + expect(owes('session-open-fails')).toBe(true) + expect(owes('session-settle-fails')).toBe(true) + expect(warn).toHaveBeenCalledWith( + 'restoring a chat for reading failed', + expect.objectContaining({ sessionId: 'session-open-fails' }) + ) +}) diff --git a/src/main/runtime/structured-agent-session-startup-step.ts b/src/main/runtime/structured-agent-session-startup-step.ts new file mode 100644 index 000000000000..3a778e64df11 --- /dev/null +++ b/src/main/runtime/structured-agent-session-startup-step.ts @@ -0,0 +1,106 @@ +// The structured-chat step of host startup, before any client lists a tab: the restart lease check, +// each listed chat's status seeded from its stored status, and every chat a gone process left with +// work settled. The tab list waits for all but the settle; chat commands wait for the settle too. + +import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' +import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' +import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' +import { ensureStructuredAgentSessionHostUnlessRefused } from './structured-agent-session-host-refusal' +import type { + StartupStepOutcome, + StructuredAgentSessionStartupGate +} from './structured-agent-session-startup-gate' + +/** Named from the host so a rename fails to compile. */ +export type StructuredAgentSessionStartupHost = Pick< + StructuredAgentSessionHost, + 'reconcileRestartLeases' | 'startup' +> & + StructuredAgentSessionListingHost + +type StructuredAgentSessionListingHost = Partial< + Pick +> + +/** + * Answers the listed chats stored status could not answer, which the post-listing restore opens. + * The lease check runs first: it marks crashed leases `recovering` and starts their recoveries, + * which record the death evidence the settle's verdicts read. It reports its own failures and + * never throws. + */ +export async function runStructuredAgentSessionStartupStep( + host: StructuredAgentSessionStartupHost, + savedSession: WorkspaceSessionState | null, + /** The settle, which chat commands wait for; it never rejects. */ + onSettling: (settled: Promise) => void +): Promise { + await host.reconcileRestartLeases() + const listedIds = listedStructuredAgentSessionIds(host, savedSession) + const background = host.startup.seedStoredStatuses(listedIds) + // Not awaited here: the tab list and paint never wait on it; chat commands do. + onSettling(host.startup.settleOwedSessions(listedIds)) + return background +} + +/** The chats with a tab, for the startup step and the tab restore alike: the host's persisted tab + * index, or before it is recorded, the saved workspace session's. */ +export function listedStructuredAgentSessionIds( + host: StructuredAgentSessionListingHost | null, + savedSession: WorkspaceSessionState | null +): string[] { + const persistedVisibleIndex = host?.getPersistedVisibleSessionTabIndex?.() ?? { + present: false, + sessionIds: [] + } + if (persistedVisibleIndex.present) { + return persistedVisibleIndex.sessionIds + } + // Unrecorded, the profile's chats join the tabs chats opened while the import was owed left. + // First: after a /clear the profile's chat would take their tab id, so seeds hit tabIdTaken. + return [ + ...new Set([ + ...persistedVisibleIndex.sessionIds, + ...collectSavedStructuredAgentSessionIds(savedSession) + ]) + ] +} + +/** + * The whole startup step, timed on the gate: the host build, then the step above. Chat commands + * held for startup go ahead however this ends: when the settle does, or as it returns or throws. + * Answers the listed chats the background restore still opens, or null when there is no host. + */ +export async function runStructuredAgentSessionStartup(input: { + gate: StructuredAgentSessionStartupGate + hasChatsOnDisk: () => boolean + buildHost: () => Promise + savedSession: () => WorkspaceSessionState | null +}): Promise { + const { gate } = input + gate.stepStarted() + let ended: StartupStepOutcome | null = 'no chats on disk' + try { + if (!input.hasChatsOnDisk()) { + return null + } + ended = 'no host' + // A refused host is no host: startup goes on, and only structured requests are refused. + await ensureStructuredAgentSessionHostUnlessRefused(input.buildHost) + const host = getStructuredAgentSessionHost() + if (!host) { + return null + } + return await runStructuredAgentSessionStartupStep(host, input.savedSession(), (settled) => { + ended = null + gate.openWhen(settled) + }) + } catch (error) { + ended &&= 'step failed' + throw error + } finally { + if (ended) { + gate.stepEnded(ended) + } + } +} diff --git a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts index bed708ae29a8..82ae3252db46 100644 --- a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts +++ b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts @@ -1,7 +1,7 @@ -// With native chat on, the renderer's startup also awaits the chat tab restore (`session.tabs.listAll`), -// which reads every chat whose tab was open at quit. That read must not wait on record-store -// bookkeeping: with a saved tab index it writes nothing, and a store that cannot be written costs -// a bounded number of failed writes, not one per chat. +// With native chat on, the renderer's startup also awaits the chat tab list (`session.tabs.listAll`). +// The list opens no chat; a restore it does not wait for then reads every chat whose tab was open at +// quit. Neither may wait on record-store bookkeeping: with a saved tab index they write nothing, and +// a store that cannot be written costs a bounded number of failed writes, not one per chat. import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -256,6 +256,14 @@ function startupRuntime(options: { afterInstall?: () => void; profileChats?: str } } +/** Starts the history restore the listing owes, as the caller that answers with it does, and joins + * it. */ +async function historyRestored(runtime: OrcaRuntimeService): Promise { + runtime.startStructuredAgentSessionHistoryRestore() + await new Promise((resolve) => setImmediate(resolve)) + await getStructuredAgentSessionHost()!.restoreReadableSessions() +} + async function expectHistory(sessionId: string): Promise { const host = getStructuredAgentSessionHost() expect(JSON.stringify((await host!.journalSnapshot(sessionId)).items)).toContain(PROMPT) @@ -300,6 +308,7 @@ describe('restoring the chat tabs open at quit', () => { }) await expect(runtime.restoreStructuredAgentSessionTabs()).resolves.toBeUndefined() + await historyRestored(runtime) expect(published().map((tab) => tab.id)).toEqual([ `agent-session:${CHAT_A}`, @@ -335,26 +344,32 @@ describe('restoring the chat tabs open at quit', () => { } ) - // Each failed write stands for one refused commit. - it.each([1, 4, 8])('fails one write per startup step with %i chats open', async (count) => { - const chats = Array.from({ length: count }, (_, index) => `chat-${index}-000${index}`) - const records = chats.map((sessionId) => chatRecord(sessionId)) - await seedProfile(records, { visible: chats }) - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const { runtime, published } = startupRuntime({ - afterInstall: () => { - writes.failing = true - } - }) + // Each failed write stands for one refused commit. The lease check is tried twice, then left to + // the next attach or send. + it.each([1, 4, 8])( + "fails only the lease check's two writes with %i chats open", + async (count) => { + const chats = Array.from({ length: count }, (_, index) => `chat-${index}-000${index}`) + const records = chats.map((sessionId) => chatRecord(sessionId)) + await seedProfile(records, { visible: chats }) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const { runtime, published } = startupRuntime({ + afterInstall: () => { + writes.failing = true + } + }) - await runtime.prepareStructuredAgentSessionStartupRestoration() - const prepared = writes.refused - await runtime.restoreStructuredAgentSessionTabs() + await runtime.prepareStructuredAgentSessionStartupRestoration() + const prepared = writes.refused + await runtime.restoreStructuredAgentSessionTabs() + await historyRestored(runtime) - expect(published()).toHaveLength(count) - expect(prepared).toBe(1) - expect(writes.refused - prepared).toBe(1) - }) + expect(published()).toHaveLength(count) + expect(prepared).toBe(2) + // Every chat's stored state answers it, so the background restore opens none and writes nothing. + expect(writes.refused - prepared).toBe(0) + } + ) describe('on a legacy profile, with no tab index yet', () => { const legacyChats = () => [ @@ -396,11 +411,12 @@ describe('restoring the chat tabs open at quit', () => { await runtime.prepareStructuredAgentSessionStartupRestoration() const prepared = writes.refused await runtime.restoreStructuredAgentSessionTabs() + await historyRestored(runtime) expect(published()).toHaveLength(2) - expect(prepared).toBe(1) - // The restore's lease check and the seed. - expect(writes.refused - prepared).toBe(2) + expect(prepared).toBe(2) + // The seed alone: stored state answers both chats, so the background restore writes nothing. + expect(writes.refused - prepared).toBe(1) }) // The profile never lists a Claude chat, so only the tab row that chat left brings it back. @@ -431,6 +447,7 @@ describe('restoring the chat tabs open at quit', () => { writes.failing = true await expect(runtime.restoreStructuredAgentSessionTabs()).resolves.toBeUndefined() + await historyRestored(runtime) expect(published()).toHaveLength(2) await expectHistory(CHAT_A) diff --git a/src/main/runtime/structured-agent-session-support-probe.test.ts b/src/main/runtime/structured-agent-session-support-probe.test.ts index 6bfdcddba0f0..710fa025eebd 100644 --- a/src/main/runtime/structured-agent-session-support-probe.test.ts +++ b/src/main/runtime/structured-agent-session-support-probe.test.ts @@ -35,6 +35,10 @@ function stubStructuredHostInstall(runtime: OrcaRuntimeService): { // still returns the right answer and fails on the install effects alone. const host = { reconcileRestartLeases: vi.fn(async () => {}), + startup: { + seedStoredStatuses: vi.fn((ids: readonly string[]) => [...ids]), + settleOwedSessions: vi.fn(async () => undefined) + }, supportsCreate: (location: { executionHostId: string; wslDistro: string | null }) => location.executionHostId === 'local' && location.wslDistro === null } diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts new file mode 100644 index 000000000000..4a01c9c156d1 --- /dev/null +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -0,0 +1,299 @@ +// The chat tab list a client asks for at startup answers from records and the tab table. It waits +// for no chat's history; each chat's history is read after the answer. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../shared/protocol-version' +import type { RuntimeMobileSessionTabsResult } from '../../shared/runtime-types' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../native-chat/agent-session-journal/journal-host-database-test-support' +import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import { + createRestTestRig, + restTestChat, + type RestTestRig +} from '../native-chat/agent-session-wire/structured-agent-session-rest-test-rig' +import { AgentSessionStoreTransactions } from './agent-session-store-transactions' +import { OrcaRuntimeService } from './orca-runtime' +import { RpcDispatcher } from './rpc/dispatcher' +import { SESSION_TAB_METHODS } from './rpc/methods/session-tabs' + +type ListingInternals = { + store: { getWorkspaceSession: () => unknown } + getClientSettings(): { experimentalStructuredNativeChat: boolean } + supportsAuthoritativeSessionTabsInventory(): boolean + hasPersistedStructuredAgentSessionStore(): boolean + getKnownWorkspaceSessionWorktreeIds(): Set + hydrateHeadlessMobileSessionTabsFromWorkspaceSession(): Set + refreshMobileSessionPtyRecords(): Promise | null> + refreshMobileSessionPtyInventory(): Promise + ensureStructuredAgentSessionHost(): Promise +} + +const CONTEXT = { + clientKind: 'runtime' as const, + clientCapabilities: [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY] +} + +let rig: RestTestRig +// Anything a test holds is let go before teardown, so a failed assertion cannot hang it. +let releaseHeld = (): void => undefined + +beforeEach(async () => { + rig = await createRestTestRig() +}) + +afterEach(async () => { + releaseHeld() + // The listing starts the history restore after its answer, without awaiting it; it finishes + // before the rig removes its files. + await new Promise((resolve) => setImmediate(resolve)) + await rig.host.restoreReadableSessions() + setStructuredAgentSessionHost(null) + await rig.dispose() + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +/** A restarted runtime whose structured host is the rig's current one. */ +function restartedRuntime(workspaceSession: unknown = null) { + const runtime = new OrcaRuntimeService() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these members exist on the runtime; they are protected, not absent. + const internal = runtime as unknown as ListingInternals + internal.store = { getWorkspaceSession: () => workspaceSession } + internal.getClientSettings = () => ({ experimentalStructuredNativeChat: true }) + // The PTY census behind an authoritative inventory is not what this pins. + internal.supportsAuthoritativeSessionTabsInventory = () => false + internal.hasPersistedStructuredAgentSessionStore = () => true + internal.getKnownWorkspaceSessionWorktreeIds = () => new Set() + internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() + internal.refreshMobileSessionPtyRecords = async () => new Set() + // The census an answer takes after the tab restore: daemon, git and SSH round trips in the app. + internal.refreshMobileSessionPtyInventory = async () => { + await new Promise((resolve) => setTimeout(resolve, 20)) + return null + } + internal.ensureStructuredAgentSessionHost = async () => { + setStructuredAgentSessionHost(rig.host) + } + const dispatcher = new RpcDispatcher({ runtime, methods: SESSION_TAB_METHODS }) + const listAll = async (): Promise => { + const response = await dispatcher.dispatch( + { id: 'list-all', authToken: 'tok', method: 'session.tabs.listAll' }, + CONTEXT + ) + if (!response.ok) { + throw new Error(`listAll refused: ${JSON.stringify(response.error)}`) + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: listAll answers with the inventory shape its handler returns. + const { snapshots } = response.result as { snapshots: RuntimeMobileSessionTabsResult[] } + return chatIds(snapshots) + } + /** Mobile's stream: resolves with its first snapshots, and what had run when they were sent. */ + const subscribeAll = (seen: () => unknown) => + new Promise<{ ids: string[]; atAnswer: unknown }>((resolve, reject) => { + void dispatcher.dispatchStreaming( + { id: 'subscribe-all', authToken: 'tok', method: 'session.tabs.subscribeAll' }, + (message) => { + const frame = JSON.parse(message) + if (!frame.ok) { + reject(new Error(JSON.stringify(frame.error))) + } else if (frame.result?.type === 'snapshots') { + resolve({ ids: chatIds(frame.result.snapshots), atAnswer: seen() }) + } + }, + { ...CONTEXT, connectionId: 'connection-1' } + ) + }) + return { runtime, listAll, subscribeAll } +} + +function chatIds(snapshots: RuntimeMobileSessionTabsResult[]): string[] { + return snapshots.flatMap((snapshot) => + snapshot.tabs.flatMap((tab) => (tab.type === 'agent-session' ? [tab.sessionId] : [])) + ) +} + +/** Resolves with the answer, or with 'still waiting' once `ms` pass without one. */ +function within(promise: Promise, ms = 2_000): Promise { + return Promise.race([ + promise, + new Promise<'still waiting'>((resolve) => setTimeout(() => resolve('still waiting'), ms)) + ]) +} + +describe('listing chat tabs at startup', () => { + it.each(['listAll', 'subscribeAll'] as const)( + 'starts the history restore only after %s has answered, census and all', + async (method) => { + const ids = ['session-1', 'session-2', 'session-3', 'session-4', 'session-5'] + for (const sessionId of ids) { + await restTestChat(rig, sessionId, { message: sessionId }) + } + await rig.crash() + await rig.boot() + // No stored status: every chat is left to the restore, which opens it. + openTestJournalHostDatabase(rig.root).db.exec('DELETE FROM journal_session_state') + const restore = vi.spyOn(rig.host, 'restoreReadableSessions') + const { runtime, listAll, subscribeAll } = restartedRuntime() + const seen = () => ({ + started: restore.mock.calls.length, + opens: rig.journalOpens.mock.calls.length + }) + + const answered = + method === 'listAll' ? { ids: await listAll(), atAnswer: seen() } : await subscribeAll(seen) + + expect(answered).toEqual({ ids, atAnswer: { started: 0, opens: 0 } }) + await vi.waitFor(() => expect(restore).toHaveBeenCalledWith(ids)) + runtime.cleanupSubscriptionsForConnection('connection-1') + } + ) + + it.each(['listAll', 'subscribeAll'] as const)( + 'derives the /clear replacements once per loop of %s, whatever the tab and worktree count', + async (method) => { + const ids = Array.from({ length: 9 }, (_, index) => `session-${index + 1}`) + for (const [index, sessionId] of ids.entries()) { + await restTestChat(rig, sessionId, { workspaceId: `workspace-${(index % 3) + 1}` }) + } + await rig.crash() + await rig.boot() + const derived = vi.spyOn(rig.host, 'conversationReplacements') + const { runtime, listAll, subscribeAll } = restartedRuntime() + const answer = async () => + method === 'listAll' ? listAll() : (await subscribeAll(() => undefined)).ids + + // The first answer also restores the tabs (one loop over every tab, one over every worktree); + // the second only lists them. + for (const loops of [2, 1]) { + derived.mockClear() + expect((await answer()).toSorted()).toEqual(ids.toSorted()) + expect(derived).toHaveBeenCalledTimes(loops) + runtime.cleanupSubscriptionsForConnection('connection-1') + } + } + ) + + it('answers with a cleared chat revealed while the listing waits on its census', async () => { + await restTestChat(rig, 'session-cleared') + await restTestChat(rig, 'session-fresh', { listed: false }) + await rig.crash() + await rig.boot() + const { runtime, listAll } = restartedRuntime() + expect(await listAll()).toEqual(['session-cleared']) + const record = rig.store.getRecord('session-cleared')! + await rig.store.setConversationCommand('session-cleared', record.lease.runtimeFence, { + command: 'clear', + state: 'completed', + phase: 'committed', + operationId: 'clear-1', + callerKey: 'client-1', + replacementSessionId: 'session-fresh' + }) + expect(await listAll()).toEqual(['session-fresh']) + // An authoritative census that could not reach every host collects a second time; hold that one. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these runtime members exist; they are protected. + const internal = runtime as unknown as { + supportsAuthoritativeSessionTabsInventory(): boolean + getAuthoritativeSessionTabsInventoryEpoch(): string + refreshMobileSessionPtyInventory(): Promise + } + internal.supportsAuthoritativeSessionTabsInventory = () => true + internal.getAuthoritativeSessionTabsInventoryEpoch = () => 'epoch-1' + let collections = 0 + const inSecond = Promise.withResolvers() + const held = Promise.withResolvers() + releaseHeld = held.resolve + internal.refreshMobileSessionPtyInventory = async () => { + collections += 1 + if (collections === 2) { + inSecond.resolve() + await held.promise + } + return null + } + const listing = listAll() + await inSecond.promise + + // agentSession.reveal from another client: the cleared chat opened from its history. + const revealed = await rig.host.revealSession('session-cleared') + await runtime.publishStructuredAgentSessionTab({ ...revealed, activate: true }) + held.resolve() + + expect(await listing).toEqual(['session-cleared']) + expect(await listAll()).toEqual(['session-cleared']) + }) + + it('writes nothing to the record store while it lists', async () => { + const ids = ['session-1', 'session-2', 'session-3', 'session-4', 'session-5'] + for (const sessionId of ids) { + await restTestChat(rig, sessionId, { message: sessionId }) + } + await rig.crash() + await rig.boot() + // Settled first, so a write here could only be the listing's own. + await rig.host.reconcileRestartLeases() + const writes = vi.spyOn(AgentSessionStoreTransactions.prototype, 'transact') + const { listAll } = restartedRuntime() + + expect(await listAll()).toEqual(ids) + // Every write, a tab's visibility included, is one store transaction. + expect(writes).not.toHaveBeenCalled() + }) + + it('lists in tab-table order, not the order the chats were created (T7)', async () => { + for (const sessionId of ['session-c', 'session-a', 'session-b']) { + await restTestChat(rig, sessionId, { listed: false }) + } + for (const sessionId of ['session-b', 'session-c', 'session-a']) { + await rig.store.setSessionTabVisibility(sessionId, true) + } + await rig.crash() + await rig.boot() + const { listAll } = restartedRuntime() + + expect(await within(listAll())).toEqual(['session-b', 'session-c', 'session-a']) + }) + + it('lists in the saved window order when the profile has no tab table (T7)', async () => { + for (const sessionId of ['session-c', 'session-a', 'session-b']) { + await restTestChat(rig, sessionId, { listed: false }) + } + await rig.crash() + const host = await rig.boot() + vi.spyOn(host, 'getPersistedVisibleSessionTabIndex').mockReturnValue({ + present: false, + sessionIds: [] + }) + const savedTab = (sessionId: string, sortOrder: number) => ({ + id: `agent-session:${sessionId}`, + entityId: sessionId, + groupId: 'group-1', + worktreeId: 'workspace-1', + contentType: 'agent-session', + label: 'Codex Chat', + customLabel: null, + color: null, + sortOrder, + createdAt: 1 + }) + const { listAll } = restartedRuntime({ + activeRepoId: null, + activeWorktreeId: 'workspace-1', + activeTabId: null, + tabsByWorktree: {}, + terminalLayoutsByTabId: {}, + unifiedTabs: { + 'workspace-1': [ + savedTab('session-a', 0), + savedTab('session-b', 1), + savedTab('session-c', 2) + ] + } + }) + + expect(await within(listAll())).toEqual(['session-a', 'session-b', 'session-c']) + }) +}) diff --git a/src/main/startup/main-process-runtime-launch-activation.test.ts b/src/main/startup/main-process-runtime-launch-activation.test.ts index 6fd2ec6ec571..551dcad8b65c 100644 --- a/src/main/startup/main-process-runtime-launch-activation.test.ts +++ b/src/main/startup/main-process-runtime-launch-activation.test.ts @@ -168,8 +168,11 @@ describe('desktop startup activation', () => { launchHooks.failBeforeWindow = false state.mainWindow = null state.isServeMode = false - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch only null-checks the runtime and marks its launch-record warm-up before the mocked RPC server takes it. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch only null-checks the runtime, holds chat commands, starts the chat startup step and its tab-restore preparation, and marks its launch-record warm-up before the mocked RPC server takes it. state.runtime = { + holdStructuredAgentSessionCommandsForStartup: () => undefined, + startStructuredAgentSessionStartupAfter: () => undefined, + prepareStructuredAgentSessionStartupRestorationAfter: () => undefined, noteAgentLaunchStartupSettled: startupSettled } as unknown as NonNullable // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch only calls whenReady(). diff --git a/src/main/startup/main-process-runtime-launch.ts b/src/main/startup/main-process-runtime-launch.ts index 054adf9f71b8..c65e4ca577c0 100644 --- a/src/main/startup/main-process-runtime-launch.ts +++ b/src/main/startup/main-process-runtime-launch.ts @@ -1,21 +1,15 @@ import { app, powerMonitor, type BrowserWindow } from 'electron' -import { is } from '@electron-toolkit/utils' import { getOrcaCloudAuthConfig } from '../orca-profiles/profile-cloud-auth-config' import { getProfileUserDataPath } from '../orca-profiles/profile-storage-paths' -import { - getCanonicalUserDataPath, - migrateMobilePairingDataToCanonicalUserDataPath -} from '../persistence' -import { OrcaRuntimeRpcServer } from '../runtime/runtime-rpc' -import { registerMobileHandlers } from '../ipc/mobile' +import type { OrcaRuntimeRpcServer } from '../runtime/runtime-rpc' import { getLocalPtyProvider, registerHeadlessPtyRuntime } from '../ipc/pty' import { LocalPtyProvider } from '../providers/local-pty-provider' import { HEADLESS_RUNTIME_WINDOW_ID } from '../../shared/runtime-types' import { OffscreenBrowserBackend } from '../browser/offscreen-browser-backend' import { browserManager } from '../browser/browser-manager' -import { getDesktopRelayStatus, publishDesktopRelayStatus } from './main-process-relay-status' +import { publishDesktopRelayStatus } from './main-process-relay-status' import { DesktopRelayService } from '../runtime/relay/desktop-relay-service' -import { getServeOptions, getBundledWebClientRoot, printServeReady } from './main-process-serve' +import { getServeOptions, printServeReady } from './main-process-serve' import { bindTerminalRuntimeStartupServices, handleCodexHomePtySpawned, @@ -41,6 +35,7 @@ import { mainProcessState as state } from './main-process-state' import { logStartupMilestone } from './startup-diagnostics' import { scheduleAgentLaunchRecordWarmup } from './agent-launch-record-warmup' import { emitServeBrowserIdentityActionLine } from '../server/serve-stdout-boundary' +import { installRuntimeRpc } from './main-process-runtime-rpc-install' import { getBrowserIdentityModeStatus } from '../browser/browser-identity-mode-store' type RuntimeService = NonNullable @@ -60,67 +55,6 @@ function settleDesktopActivation(): void { }) } -function installRuntimeRpc( - runtime: RuntimeService, - serveOptions: ReturnType | null -): OrcaRuntimeRpcServer { - // Why: existing installs may have pairing creds under the late app.getPath('userData'); copy them forward before switching to the canonical path. - migrateMobilePairingDataToCanonicalUserDataPath(app.getPath('userData')) - // Why: parallel E2E Electron instances would race the fixed port (EADDRINUSE); port 0 gives each a random OS-assigned port. - const isE2E = Boolean(process.env.ORCA_E2E_USER_DATA_DIR) - const requestedE2EWsPort = process.env.ORCA_E2E_RUNTIME_WS_PORT - const e2eWsPort = requestedE2EWsPort === undefined ? 0 : Number(requestedE2EWsPort) - if (isE2E && (!Number.isInteger(e2eWsPort) || e2eWsPort < 0 || e2eWsPort > 65_535)) { - throw new Error(`Invalid ORCA_E2E_RUNTIME_WS_PORT value: ${requestedE2EWsPort}`) - } - // Why: pin dev to 6769 so `pnpm dev` doesn't race packaged Orca on 6768 and fall back to a random port, breaking deterministic mobile pairing/repro (STA-1511). - const devWsPort = is.dev && !isE2E ? 6769 : undefined - const runtimeRpc = new OrcaRuntimeRpcServer({ - runtime, - // Why: mobile pairing needs the stable pre-setName() path (getCanonicalUserDataPath), not a late app.getPath('userData') that drops paired devices across restarts. - userDataPath: getCanonicalUserDataPath(), - enableWebSocket: true, - // Why: STA-2370 — the desktop app binds the WS listener to loopback until the user pairs a device; - // `orca serve` is an explicit remote opt-in, and E2E keeps the wide bind its harness connects over. - exposeNetworkByDefault: Boolean(serveOptions) || isE2E, - ...(isE2E ? { wsPort: e2eWsPort } : {}), - ...(devWsPort !== undefined ? { wsPort: devWsPort } : {}), - ...(serveOptions?.wsPort !== undefined - ? { - wsPort: serveOptions.wsPort, - // Why: only explicit `orca serve --port` overrides a stale STA-1511 fallback (issue #8535); default/dev stay fallback-first for pairing stability. - preferPinnedWsPort: true - } - : {}), - webClientRoot: getBundledWebClientRoot() - }) - state.runtimeRpc = runtimeRpc - registerMobileHandlers(runtimeRpc, { - getRelayStatus: getDesktopRelayStatus, - consumePendingUnpairedDeviceAuthFailure: (webContentsId) => { - if ( - !state.mainWindow || - state.mainWindow.isDestroyed() || - state.mainWindow.webContents.id !== webContentsId || - !state.pendingUnpairedDeviceAuthFailure - ) { - return false - } - state.pendingUnpairedDeviceAuthFailure = false - return true - } - }) - // Why: repeated direct auth failures otherwise look like a client that never connects; point users to re-pairing. - runtimeRpc.setOnUnpairedDeviceAuthFailure(() => { - // Why: runtime startup races renderer mount; retain the one-shot until the listener consumes it. - state.pendingUnpairedDeviceAuthFailure = true - if (state.mainWindow && !state.mainWindow.isDestroyed()) { - state.mainWindow.webContents.send('mobile:unpairedDeviceAuthFailure') - } - }) - return runtimeRpc -} - async function launchServeMode( runtime: RuntimeService, runtimeRpc: OrcaRuntimeRpcServer, @@ -311,11 +245,17 @@ export async function initializeMainProcessRuntimeLaunch( return } state.serveOptions = serveOptions + // Before anything is served: chat commands wait for the startup settle started below. + runtime.holdStructuredAgentSessionCommandsForStartup() const runtimeRpc = installRuntimeRpc(runtime, serveOptions) const shellPathReady = shellPathHydration.whenReady() // Why published: the renderer's git-environment barrier must fence on the same // generation the terminal startup services wait for, not a later re-read. state.shellPathReady = shellPathReady + // The host seeds its chats' statuses and settles crashed ones itself, so a window, a remote + // client or neither sees the same state. It needs neither the terminal daemon nor the hook + // server, so it does not wait for the first window's services. + runtime.startStructuredAgentSessionStartupAfter(shellPathReady) // Why before any window: the poisoned install DACL kills the renderer at init, and // the probe that detects it cannot finish before createMainWindow. Bounded, and a // no-op (one absent-file read) unless a previous launch already recorded the verdict. @@ -340,6 +280,11 @@ export async function initializeMainProcessRuntimeLaunch( await shellPathReady bindTerminalRuntimeStartupServices(Promise.resolve(startTerminalRuntimeStartupServices())) } + // The tab restore's terminal records refresh lists the daemon's terminals. The first-window + // services resolve when the daemon and hook server are up, or at their 12 s timeout regardless. + runtime.prepareStructuredAgentSessionStartupRestorationAfter( + state.firstWindowStartupServicesReady + ) app.on('activate', options.handleMacAppActivation) if (serveOptions) { await launchServeMode(runtime, runtimeRpc, serveOptions) diff --git a/src/main/startup/main-process-runtime-rpc-install.ts b/src/main/startup/main-process-runtime-rpc-install.ts new file mode 100644 index 000000000000..44bc7e93c144 --- /dev/null +++ b/src/main/startup/main-process-runtime-rpc-install.ts @@ -0,0 +1,76 @@ +// The runtime RPC server the launch serves chats, mobile and remote clients over. + +import { app } from 'electron' +import { is } from '@electron-toolkit/utils' +import { + getCanonicalUserDataPath, + migrateMobilePairingDataToCanonicalUserDataPath +} from '../persistence' +import { OrcaRuntimeRpcServer } from '../runtime/runtime-rpc' +import { registerMobileHandlers } from '../ipc/mobile' +import { getDesktopRelayStatus } from './main-process-relay-status' +import { getBundledWebClientRoot, type getServeOptions } from './main-process-serve' +import { mainProcessState as state } from './main-process-state' + +type RuntimeService = NonNullable + +export function installRuntimeRpc( + runtime: RuntimeService, + serveOptions: ReturnType | null +): OrcaRuntimeRpcServer { + // Why: existing installs may have pairing creds under the late app.getPath('userData'); copy them forward before switching to the canonical path. + migrateMobilePairingDataToCanonicalUserDataPath(app.getPath('userData')) + // Why: parallel E2E Electron instances would race the fixed port (EADDRINUSE); port 0 gives each a random OS-assigned port. + const isE2E = Boolean(process.env.ORCA_E2E_USER_DATA_DIR) + const requestedE2EWsPort = process.env.ORCA_E2E_RUNTIME_WS_PORT + const e2eWsPort = requestedE2EWsPort === undefined ? 0 : Number(requestedE2EWsPort) + if (isE2E && (!Number.isInteger(e2eWsPort) || e2eWsPort < 0 || e2eWsPort > 65_535)) { + throw new Error(`Invalid ORCA_E2E_RUNTIME_WS_PORT value: ${requestedE2EWsPort}`) + } + // Why: pin dev to 6769 so `pnpm dev` doesn't race packaged Orca on 6768 and fall back to a random port, breaking deterministic mobile pairing/repro (STA-1511). + const devWsPort = is.dev && !isE2E ? 6769 : undefined + const runtimeRpc = new OrcaRuntimeRpcServer({ + runtime, + // Why: mobile pairing needs the stable pre-setName() path (getCanonicalUserDataPath), not a late app.getPath('userData') that drops paired devices across restarts. + userDataPath: getCanonicalUserDataPath(), + enableWebSocket: true, + // Why: STA-2370 — the desktop app binds the WS listener to loopback until the user pairs a device; + // `orca serve` is an explicit remote opt-in, and E2E keeps the wide bind its harness connects over. + exposeNetworkByDefault: Boolean(serveOptions) || isE2E, + ...(isE2E ? { wsPort: e2eWsPort } : {}), + ...(devWsPort !== undefined ? { wsPort: devWsPort } : {}), + ...(serveOptions?.wsPort !== undefined + ? { + wsPort: serveOptions.wsPort, + // Why: only explicit `orca serve --port` overrides a stale STA-1511 fallback (issue #8535); default/dev stay fallback-first for pairing stability. + preferPinnedWsPort: true + } + : {}), + webClientRoot: getBundledWebClientRoot() + }) + state.runtimeRpc = runtimeRpc + registerMobileHandlers(runtimeRpc, { + getRelayStatus: getDesktopRelayStatus, + consumePendingUnpairedDeviceAuthFailure: (webContentsId) => { + if ( + !state.mainWindow || + state.mainWindow.isDestroyed() || + state.mainWindow.webContents.id !== webContentsId || + !state.pendingUnpairedDeviceAuthFailure + ) { + return false + } + state.pendingUnpairedDeviceAuthFailure = false + return true + } + }) + // Why: repeated direct auth failures otherwise look like a client that never connects; point users to re-pairing. + runtimeRpc.setOnUnpairedDeviceAuthFailure(() => { + // Why: runtime startup races renderer mount; retain the one-shot until the listener consumes it. + state.pendingUnpairedDeviceAuthFailure = true + if (state.mainWindow && !state.mainWindow.isDestroyed()) { + state.mainWindow.webContents.send('mobile:unpairedDeviceAuthFailure') + } + }) + return runtimeRpc +} diff --git a/src/renderer/src/components/activity/activity-answered-ask-timeline.test.tsx b/src/renderer/src/components/activity/activity-answered-ask-timeline.test.tsx index 49e53f64f7e3..7c8c8837c084 100644 --- a/src/renderer/src/components/activity/activity-answered-ask-timeline.test.tsx +++ b/src/renderer/src/components/activity/activity-answered-ask-timeline.test.tsx @@ -129,11 +129,11 @@ function thread(activity: ReturnType) { return only } -async function connect(): Promise<(event: AgentSessionStatusEvent) => void> { +async function connect(): Promise<(event: AgentSessionStatusEvent) => Promise> { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) const emit: (event: AgentSessionStatusEvent) => void = mocks.subscribeStatus.mock.calls[0]?.[1] - return (event) => act(() => emit(event)) + return (event) => act(async () => emit(event)) } describe("an answered subagent ask on a settled parent's Activity row", () => { @@ -163,15 +163,15 @@ describe("an answered subagent ask on a settled parent's Activity row", () => { it('reads done, lists Blocked then done, and leaves the answer read', async () => { const emit = await connect() - emit({ type: 'snapshot', sessions: [summary()] }) + await emit({ type: 'snapshot', sessions: [summary()] }) acknowledge(SETTLED + 1_000) - emit({ + await emit({ type: 'status', session: summary({ status: 'attention', statusStartedAt: ASKED, updatedAt: ASKED }) }) // The user reads the ask and answers it; the parent returns to its own turn's end. acknowledge(ASKED + 500) - emit({ type: 'status', session: summary({ updatedAt: ANSWERED }) }) + await emit({ type: 'status', session: summary({ updatedAt: ANSWERED }) }) const row = thread(renderActivity()) expect(activityThreadStatusId(row)).toBe('done') @@ -190,7 +190,7 @@ describe("an answered subagent ask on a settled parent's Activity row", () => { expect(countActivityUnread(store().getState())).toBe(0) // Clearing the answered row must also pass the ask, which is dated after the done. - act(() => { + await act(async () => { expect(clearActivityThread(row)).toBe(true) }) expect(renderActivity().result.current.allThreads).toHaveLength(0) @@ -199,13 +199,13 @@ describe("an answered subagent ask on a settled parent's Activity row", () => { it('keeps each answered done between the asks around it', async () => { const asks = [ASKED, ASKED + 400, ASKED + 800] const emit = await connect() - emit({ type: 'snapshot', sessions: [summary()] }) + await emit({ type: 'snapshot', sessions: [summary()] }) for (const askedAt of asks) { - emit({ + await emit({ type: 'status', session: summary({ status: 'attention', statusStartedAt: askedAt, updatedAt: askedAt }) }) - emit({ type: 'status', session: summary({ updatedAt: askedAt + 200 }) }) + await emit({ type: 'status', session: summary({ updatedAt: askedAt + 200 }) }) } // Every done repeats the turn's end, so only when each was seen keeps them apart and in order; @@ -223,23 +223,23 @@ describe("an answered subagent ask on a settled parent's Activity row", () => { it('reads done after a clear hid that done, while the later ask stays listed', async () => { const emit = await connect() - emit({ type: 'snapshot', sessions: [summary()] }) + await emit({ type: 'snapshot', sessions: [summary()] }) acknowledge(SETTLED + 1_000) const beforeAsk = renderActivity() let cleared = false - act(() => { + await act(async () => { cleared = clearActivityThread(thread(beforeAsk)) }) expect(cleared).toBe(true) beforeAsk.unmount() expect(store().getState().activityClearedAtByPaneKey).toEqual({ [paneKey()]: SETTLED }) - emit({ + await emit({ type: 'status', session: summary({ status: 'attention', statusStartedAt: ASKED, updatedAt: ASKED }) }) acknowledge(ASKED + 500) - emit({ type: 'status', session: summary({ updatedAt: ANSWERED }) }) + await emit({ type: 'status', session: summary({ updatedAt: ANSWERED }) }) const row = thread(renderActivity()) // The cleared done stays cleared; only the ask, seen after the clear, is listed. @@ -250,7 +250,7 @@ describe("an answered subagent ask on a settled parent's Activity row", () => { expect(countActivityUnread(store().getState())).toBe(0) // Once a second ask moves the answered done into history, it stays cleared there too. - emit({ + await emit({ type: 'status', session: summary({ status: 'attention', diff --git a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.launch-directory.test.tsx b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.launch-directory.test.tsx index 0fae5234637c..a0e06634de85 100644 --- a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.launch-directory.test.tsx +++ b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.launch-directory.test.tsx @@ -84,13 +84,14 @@ function summary(overrides: Partial = {}): AgentSessi } } -function emit(event: AgentSessionStatusEvent): void { +async function emit(event: AgentSessionStatusEvent): Promise { const call = mocks.subscribeStatus.mock.calls.at(-1) if (!call) { throw new Error('status feed not subscribed') } const send: (event: AgentSessionStatusEvent) => void = call[1] - act(() => send(event)) + // The bridge applies a tick's rows together; awaiting the act lets that tick's write land. + await act(async () => send(event)) } function pins(): AppState['structuredSessionLaunchDirectoryByTabId'] { @@ -118,7 +119,7 @@ describe('StructuredAgentSessionStatusBridge pinned launch directory', () => { const view = render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - emit({ type: 'snapshot', sessions: [summary({ launchDirectory: '/home/me/original' })] }) + await emit({ type: 'snapshot', sessions: [summary({ launchDirectory: '/home/me/original' })] }) expect(pins()).toEqual({ [floatingTab.id]: { sessionId: 'session-1', launchDirectory: '/home/me/original' } @@ -133,7 +134,7 @@ describe('StructuredAgentSessionStatusBridge pinned launch directory', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - emit({ type: 'snapshot', sessions: [summary()] }) + await emit({ type: 'snapshot', sessions: [summary()] }) expect(pins()).toEqual({}) }) @@ -150,7 +151,10 @@ describe('StructuredAgentSessionStatusBridge pinned launch directory', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - emit({ type: 'snapshot', sessions: [summary({ launchDirectory: '/srv/remote/floating' })] }) + await emit({ + type: 'snapshot', + sessions: [summary({ launchDirectory: '/srv/remote/floating' })] + }) // The summary itself landed; only its path was withheld. expect(Object.keys(mocks.store?.getState().agentStatusByPaneKey ?? {})).toHaveLength(1) diff --git a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx index beeb27600cbb..ef1d24b741ff 100644 --- a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx +++ b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx @@ -30,12 +30,25 @@ const mocks = vi.hoisted(() => ({ vi.mock('@/store', async () => { const { createTestStore } = await import('@/store/slices/store-test-helpers') const useAppStore = createTestStore() - const { setAgentStatus, removeAgentStatus } = useAppStore.getState() + const { setAgentStatus, removeAgentStatus, transactAgentStatuses } = useAppStore.getState() useAppStore.setState({ setAgentStatus: (...args) => { mocks.setAgentStatus(...args) setAgentStatus(...args) }, + // The bridge writes rows through one transaction per tick; each applied row is one write. + transactAgentStatuses: (operation) => + transactAgentStatuses((transaction) => + operation({ + ...transaction, + apply: (update) => { + if (update.kind !== 'providerSession') { + mocks.setAgentStatus(update.paneKey, update.payload) + } + return transaction.apply(update) + } + }) + ), removeAgentStatus: (paneKey) => { mocks.removeAgentStatus(paneKey) removeAgentStatus(paneKey) @@ -167,7 +180,7 @@ describe('StructuredAgentSessionStatusBridge', () => { expect(feed().target).toEqual({ kind: 'local' }) expect(mocks.subscribeTranscript).not.toHaveBeenCalled() - act(() => feed().emit({ type: 'snapshot', sessions: [summary()] })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary()] })) expect(statuses()).toEqual([ expect.objectContaining({ @@ -191,7 +204,7 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) const updatedAt = Date.now() - 30 * 60 * 1000 - 1 - act(() => feed().emit({ type: 'status', session: summary({ updatedAt }) })) + await act(async () => feed().emit({ type: 'status', session: summary({ updatedAt }) })) const entry = statuses()[0] expect(entry).toEqual(expect.objectContaining({ state: 'working', structuredHostOwned: true })) expect(isExplicitAgentStatusFresh(entry, Date.now(), 30 * 60 * 1000)).toBe(true) @@ -200,9 +213,9 @@ describe('StructuredAgentSessionStatusBridge', () => { it('clears host-held evidence when the status stream disconnects', async () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'status', session: summary() })) + await act(async () => feed().emit({ type: 'status', session: summary() })) expect(statuses()).toHaveLength(1) - act(() => feed().emit({ type: 'end' })) + await act(async () => feed().emit({ type: 'end' })) expect(statuses()).toHaveLength(1) expect(statuses()[0]).not.toHaveProperty('structuredHostOwned') }) @@ -210,15 +223,17 @@ describe('StructuredAgentSessionStatusBridge', () => { it('maps each host status onto the sidebar agent state', async () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'snapshot', sessions: [summary()] })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary()] })) expect(statuses()).toEqual([expect.objectContaining({ state: 'working' })]) - act(() => feed().emit({ type: 'status', session: summary({ status: 'idle', updatedAt: 2 }) })) + await act(async () => + feed().emit({ type: 'status', session: summary({ status: 'idle', updatedAt: 2 }) }) + ) expect(statuses()).toEqual([ expect.objectContaining({ state: 'done', sessionBoundary: false, stateStartedAt: 2 }) ]) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ status: 'attention', updatedAt: 3 }) }) ) expect(statuses()).toEqual([expect.objectContaining({ state: 'blocked' })]) @@ -228,7 +243,7 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [ @@ -265,7 +280,7 @@ describe('StructuredAgentSessionStatusBridge', () => { // An unchanged roster must not rewrite the store. const writes = mocks.setAgentStatus.mock.calls.length - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -285,7 +300,7 @@ describe('StructuredAgentSessionStatusBridge', () => { ) expect(mocks.setAgentStatus.mock.calls.length).toBe(writes) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -302,7 +317,7 @@ describe('StructuredAgentSessionStatusBridge', () => { }) ]) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -320,7 +335,9 @@ describe('StructuredAgentSessionStatusBridge', () => { ).toBe('unverifiable') // A summary without tasks ends the fan-out: children clear with it. - act(() => feed().emit({ type: 'status', session: summary({ status: 'idle', updatedAt: 4 }) })) + await act(async () => + feed().emit({ type: 'status', session: summary({ status: 'idle', updatedAt: 4 }) }) + ) expect(statuses()).toEqual([expect.objectContaining({ subagents: undefined })]) }) @@ -330,7 +347,7 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [ @@ -349,7 +366,7 @@ describe('StructuredAgentSessionStatusBridge', () => { expect.objectContaining({ state: 'working', workingMode: undefined, stateStartedAt: 1 }) ]) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -367,7 +384,7 @@ describe('StructuredAgentSessionStatusBridge', () => { expect.objectContaining({ state: 'working', workingMode: 'monitoring', stateStartedAt: 2 }) ]) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -391,7 +408,7 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [ @@ -407,7 +424,7 @@ describe('StructuredAgentSessionStatusBridge', () => { expect.objectContaining({ state: 'working', workingMode: 'monitoring', stateStartedAt: 1 }) ]) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -433,7 +450,7 @@ describe('StructuredAgentSessionStatusBridge', () => { await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) const before = Date.now() - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [ @@ -461,23 +478,25 @@ describe('StructuredAgentSessionStatusBridge', () => { tab: structuredTab as never, parentIsFresh })[0]?.state - act(() => feed().emit({ type: 'snapshot', sessions: [live] })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [live] })) expect(childState()).toBe('unverifiable') parentIsFresh = true expect(childState()).toBe('working') - act(() => feed().emit({ type: 'end' })) + await act(async () => feed().emit({ type: 'end' })) expect(childState()).toBe('unverifiable') await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledTimes(2)) - act(() => feed(1).emit({ type: 'snapshot', sessions: [] })) + await act(async () => feed(1).emit({ type: 'snapshot', sessions: [] })) expect(childState()).toBe('unverifiable') - act(() => feed(1).emit({ type: 'status', session: live })) + await act(async () => feed(1).emit({ type: 'status', session: live })) expect(childState()).toBe('working') parentIsFresh = false expect(childState()).toBe('unverifiable') const writes = mocks.setAgentStatus.mock.calls.length - act(() => feed(1).emit({ type: 'status', session: live })) + await act(async () => feed(1).emit({ type: 'status', session: live })) expect(mocks.setAgentStatus).toHaveBeenCalledTimes(writes) - act(() => feed(1).emit({ type: 'status', session: summary({ backgroundTasks: [] }) })) + await act(async () => + feed(1).emit({ type: 'status', session: summary({ backgroundTasks: [] }) }) + ) expect(childState()).toBeUndefined() }) @@ -485,7 +504,7 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [ @@ -508,7 +527,7 @@ describe('StructuredAgentSessionStatusBridge', () => { ]) // The tool line describes live work, so a settled turn that omits it must clear it. - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -530,7 +549,7 @@ describe('StructuredAgentSessionStatusBridge', () => { expect(statuses()[0]?.toolInput).toBeUndefined() // Only the message moves here, so the row updates only if the guard compares it. - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -550,20 +569,20 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'snapshot', sessions: [summary({ status: null })] })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary({ status: null })] })) expect(mocks.setAgentStatus).not.toHaveBeenCalled() - act(() => feed().emit({ type: 'status', session: summary({ updatedAt: 2 }) })) + await act(async () => feed().emit({ type: 'status', session: summary({ updatedAt: 2 }) })) expect(statuses()).toEqual([expect.objectContaining({ state: 'working' })]) }) it('keeps the status map reference stable for repeated equal summaries', async () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'snapshot', sessions: [summary()] })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary()] })) const before = mocks.store?.getState().agentStatusByPaneKey - act(() => { + await act(async () => { for (let repeat = 0; repeat < 10; repeat += 1) { feed().emit({ type: 'status', session: summary() }) } @@ -582,7 +601,7 @@ describe('StructuredAgentSessionStatusBridge', () => { }) render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary({ status: 'idle', updatedAt: now - 100 })] @@ -596,7 +615,7 @@ describe('StructuredAgentSessionStatusBridge', () => { updatedAt: now - 100 }) ]) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ status: 'idle', updatedAt: now - 50 }) }) ) expect(statuses()).toEqual([ @@ -611,8 +630,8 @@ describe('StructuredAgentSessionStatusBridge', () => { it('preserves the working age when host metadata advances during the same turn', async () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'status', session: summary({ updatedAt: 100 }) })) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ updatedAt: 100 }) })) + await act(async () => feed().emit({ type: 'status', session: summary({ updatedAt: 200, providerSession: { ...providerSession, id: 'new-id' } }) @@ -626,15 +645,15 @@ describe('StructuredAgentSessionStatusBridge', () => { it('accepts an authoritative older journal age after a host upgrade reconnect', async () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'status', session: summary({ updatedAt: 800 }) })) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ updatedAt: 800 }) })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary({ status: 'idle', updatedAt: 900 })] }) ) const paneKey = statuses()[0].paneKey const history = statuses()[0].stateHistory const acknowledged = { [paneKey]: 950 } mocks.store?.setState({ acknowledgedAgentsByPaneKey: acknowledged }) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary({ status: 'idle', updatedAt: 200 })] }) ) expect(statuses()).toEqual([ @@ -644,7 +663,7 @@ describe('StructuredAgentSessionStatusBridge', () => { const calls = mocks.setAgentStatus.mock.calls.length expect(statuses()[0].stateHistory).toBe(history) expect(mocks.store?.getState().acknowledgedAgentsByPaneKey).toBe(acknowledged) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary({ status: 'idle', updatedAt: 200 })] }) ) expect(mocks.store?.getState().agentStatusByPaneKey).toBe(before) @@ -654,10 +673,10 @@ describe('StructuredAgentSessionStatusBridge', () => { it('drops the status and the feed when the last structured tab closes', async () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'snapshot', sessions: [summary()] })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary()] })) expect(statuses()).toHaveLength(1) - act(() => mocks.store?.setState({ unifiedTabsByWorktree: { 'wt-1': [] } })) + await act(async () => mocks.store?.setState({ unifiedTabsByWorktree: { 'wt-1': [] } })) expect(statuses()).toEqual([]) await waitFor(() => expect(mocks.unsubscribe).toHaveBeenCalledOnce()) @@ -670,7 +689,7 @@ describe('StructuredAgentSessionStatusBridge', () => { await act(() => Promise.resolve()) expect(mocks.subscribeStatus).toHaveBeenCalledOnce() - act(() => feed().emit({ type: 'end' })) + await act(async () => feed().emit({ type: 'end' })) await act(() => vi.advanceTimersByTimeAsync(300)) expect(mocks.unsubscribe).toHaveBeenCalledOnce() @@ -810,7 +829,7 @@ describe('the main agent fact the bridge writes', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [ @@ -832,7 +851,7 @@ describe('the main agent fact the bridge writes', () => { ]) // The shell drains: the row settles, the main agent was done all along, so its clock holds. - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ status: 'idle', updatedAt: 2, turnOutcome: 'cancellation' }) diff --git a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx index 56e93c60e602..a9b6eec4751e 100644 --- a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx +++ b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx @@ -32,6 +32,7 @@ import { agentMainAgentVerdict } from '../../../../shared/agent-main-agent-verdi import { useStructuredAgentSessionLaunchLifecycle } from '@/lib/structured-agent-session-launch-registry' import { useStructuredAgentSessionLaunchFailedAt } from '@/lib/structured-agent-session-launch-failed-at' import { useAppStore } from '@/store' +import type { AgentStatusBatchTransaction } from '@/store/slices/agent-status-contract' import type { RuntimeClientTarget } from '@/runtime/runtime-rpc-client' import { structuredAgentSessionOwnerForTab, @@ -135,6 +136,17 @@ function childWorkFor(summary: AgentSessionStatusSummary): { return subagents ? { subagents } : {} } +type StatusProjection = { + tab: StructuredTab + summary: AgentSessionStatusSummary + observation: 'live' | 'unverifiable' +} + +// Rows projected in one tick, applied as one status publication: a host snapshot reaches every +// projection at once, and a store write per row re-runs every reader once per row. +const queuedProjections = new Map() +let projectionFlushQueued = false + /** A start the host refused leaves it no session to publish, so the launch's own failure is the * row: the same failed verdict the host publishes for a send the agent's start refused. */ function projectFailedStart(tab: StructuredTab, paneKey: string, failedAt: number): void { @@ -183,9 +195,10 @@ function projectStatus( launchFailedAt: number | null ): void { const paneKey = structuredAgentSessionPaneKey(tab.id, tab.entityId) - const store = useAppStore.getState() // No persisted turn yet (or nothing known): the row shows no agent status at all. if (!summary?.status) { + forgetQueuedProjection(paneKey) + const store = useAppStore.getState() if (launchFailedAt !== null) { projectFailedStart(tab, paneKey, launchFailedAt) } else if (store.agentStatusByPaneKey?.[paneKey]) { @@ -193,6 +206,40 @@ function projectStatus( } return } + queuedProjections.set(paneKey, { tab, summary, observation }) + if (!projectionFlushQueued) { + projectionFlushQueued = true + queueMicrotask(flushQueuedProjections) + } +} + +function forgetQueuedProjection(paneKey: string): void { + queuedProjections.delete(paneKey) +} + +function flushQueuedProjections(): void { + projectionFlushQueued = false + const projections = [...queuedProjections.values()] + queuedProjections.clear() + if (projections.length === 0) { + return + } + useAppStore.getState().transactAgentStatuses((transaction) => { + for (const projection of projections) { + applyStatusProjection(transaction, projection) + } + }) +} + +function applyStatusProjection( + transaction: AgentStatusBatchTransaction, + { tab, summary, observation }: StatusProjection +): void { + const paneKey = structuredAgentSessionPaneKey(tab.id, tab.entityId) + const store = transaction.getState() + if (!summary.status) { + return + } const { children, subagents } = childWorkFor(summary) // Shared with `worktree ps`, so the CLI and this row cannot disagree about one session. const agentStatus = structuredAgentSessionAgentStatus({ @@ -256,11 +303,11 @@ function projectStatus( ) { return } - store.setAgentStatus( + transaction.apply({ paneKey, - desired, - tab.label, - { + payload: desired, + terminalTitle: tab.label, + timing: { updatedAt: summary.updatedAt, // This ordered host feed can correct a legacy publication clock after upgrade. allowOlderTimestamp: true, @@ -277,13 +324,13 @@ function projectStatus( // row held open by child work alone is dated by when this client saw it instead. evidenceObservedAt: isAgentStatusHeldOpenByChildWork(desired) ? Date.now() : summary.updatedAt }, - { tabId: tab.id, worktreeId: tab.worktreeId }, - { + routing: { tabId: tab.id, worktreeId: tab.worktreeId }, + metadata: { ...(summary.providerSession ? { providerSession: summary.providerSession } : {}), terminalResumeEligible: false, ...(summary.hostExecutionOwned ? { structuredHostOwned: true as const } : {}) } - ) + }) } /** Reads the chat's status from the host recorded on its tab; a chat no host can be named for has @@ -331,8 +378,12 @@ function StructuredAgentSessionOwnedStatusProjection({ [tab.entityId, tab.id] ) useEffect( - () => () => - useAppStore.getState().removeAgentStatus(structuredAgentSessionPaneKey(tab.id, tab.entityId)), + () => () => { + const paneKey = structuredAgentSessionPaneKey(tab.id, tab.entityId) + // A row queued before the unmount must not land after it. + forgetQueuedProjection(paneKey) + useAppStore.getState().removeAgentStatus(paneKey) + }, [tab.entityId, tab.id] ) return null diff --git a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx new file mode 100644 index 000000000000..58d30cc011b1 --- /dev/null +++ b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx @@ -0,0 +1,379 @@ +// @vitest-environment happy-dom + +// A host status snapshot reaches every chat's projection at once: its rows must land as one store +// publication, not one per row, or every reader of the status map runs once per row. + +import { act, cleanup, render, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { + AgentSessionStatusEvent, + AgentSessionStatusSummary +} from '../../../../shared/agent-session-wire' +import type { AgentStatusEntry } from '../../../../shared/agent-status-types' +import type { Tab } from '../../../../shared/tab-types' +import type * as RuntimeRpcClientModule from '@/runtime/runtime-rpc-client' +import type * as AgentStatusProjectionModule from '@/runtime/sync-runtime-graph/agent-status-projection' + +const mocks = vi.hoisted(() => ({ + mobileStatusProjections: vi.fn(), + removeAgentStatus: vi.fn(), + setAgentStatus: vi.fn(), + setGeneratedTitle: vi.fn(), + setGeneratedTitles: vi.fn(), + subscribeStatus: vi.fn(), + subscribeTranscript: vi.fn(), + supportsCapability: vi.fn(), + unsubscribe: vi.fn(), + writeEachRowAlone: false +})) + +vi.mock('@/store', async () => { + const { createTestStore } = await import('@/store/slices/store-test-helpers') + const useAppStore = createTestStore() + const { + setAgentStatus, + removeAgentStatus, + transactAgentStatuses, + setGeneratedTabTitleFromAgentPrompt, + setGeneratedTabTitlesFromAgentPrompts + } = useAppStore.getState() + useAppStore.setState({ + setGeneratedTabTitleFromAgentPrompt: (...args) => { + mocks.setGeneratedTitle(...args) + setGeneratedTabTitleFromAgentPrompt(...args) + }, + setGeneratedTabTitlesFromAgentPrompts: (updates) => { + mocks.setGeneratedTitles(updates) + setGeneratedTabTitlesFromAgentPrompts(updates) + }, + setAgentStatus: (...args) => { + mocks.setAgentStatus(...args) + setAgentStatus(...args) + }, + // The bridge writes rows through one transaction per tick; each applied row is one write. + transactAgentStatuses: (operation) => + mocks.writeEachRowAlone + ? operation({ + // The bridge before batching: each row written straight to the store, read live. + getState: useAppStore.getState, + apply: (update) => { + const state = useAppStore.getState() + if (update.kind === 'providerSession') { + state.recordAgentProviderSession( + update.paneKey, + update.agent, + update.providerSession, + update.timing, + update.routing, + update.metadata + ) + } else { + state.setAgentStatus( + update.paneKey, + update.payload, + update.terminalTitle, + update.timing, + update.routing, + update.metadata + ) + } + return true + }, + afterCommit: (effect) => effect() + }) + : transactAgentStatuses((transaction) => + operation({ + ...transaction, + apply: (update) => { + if (update.kind !== 'providerSession') { + mocks.setAgentStatus(update.paneKey, update.payload) + } + return transaction.apply(update) + } + }) + ), + removeAgentStatus: (paneKey) => { + mocks.removeAgentStatus(paneKey) + removeAgentStatus(paneKey) + } + }) + return { useAppStore } +}) + +vi.mock('@/runtime/sync-runtime-graph/agent-status-projection', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + buildRuntimeMobileAgentStatusProjection: ( + ...args: Parameters + ) => { + mocks.mobileStatusProjections() + return actual.buildRuntimeMobileAgentStatusProjection(...args) + } + } +}) + +// Every chat runs on the local host. +vi.mock('@/lib/worktree-runtime-owner', () => ({ + getRuntimeEnvironmentIdForWorktree: () => null, + getExecutionHostIdForWorktree: () => 'local' +})) + +vi.mock('@/runtime/runtime-rpc-client', async (importOriginal) => ({ + ...(await importOriginal()), + runtimeEnvironmentSupportsCapability: mocks.supportsCapability +})) + +vi.mock('@/runtime/structured-agent-session-client', () => ({ + callStructuredAgentSession: vi.fn(), + subscribeStructuredAgentSession: mocks.subscribeTranscript, + subscribeStructuredAgentSessionStatus: mocks.subscribeStatus +})) + +import { StructuredAgentSessionStatusBridge } from './StructuredAgentSessionStatusBridge' +import { useAppStore } from '@/store' +import { resetStructuredAgentSessionStatusFeedsForTests } from '@/runtime/structured-agent-session-status-feed' +import { + canSkipRuntimeMobileSessionSyncKeyBuild, + getRuntimeMobileSessionSyncKey +} from '@/runtime/sync-runtime-graph/sync-key' +import { getDefaultSettings } from '../../../../shared/constants' +import { structuredAgentSessionPaneKey } from '../../../../shared/structured-agent-session-projection' + +const structuredTab = { + id: 'structured-tab-1', + worktreeId: 'wt-1', + groupId: 'group-1', + contentType: 'agent-session', + entityId: 'session-1', + label: 'Codex Chat', + customLabel: null, + color: null, + sortOrder: 0, + createdAt: 0, + isPinned: false, + agentSessionAgent: 'codex' +} satisfies Tab + +const providerSession = { key: 'session_id', id: '01a002e9-9a1c-7d42-a642-e481f64446f1' } as const + +function summary(overrides: Partial = {}): AgentSessionStatusSummary { + return { + sessionId: 'session-1', + workspaceId: 'wt-1', + agent: 'codex', + status: 'working', + hostExecutionOwned: true, + latestPrompt: 'hello', + providerSession, + updatedAt: 1, + ...overrides + } +} + +function statuses(): AgentStatusEntry[] { + return Object.values(useAppStore.getState().agentStatusByPaneKey ?? {}) +} + +/** The host side of the most recent status subscription. */ +function feed(index = 0): { target: unknown; emit: (event: AgentSessionStatusEvent) => void } { + const call = mocks.subscribeStatus.mock.calls[index] + if (!call) { + throw new Error('status feed not subscribed') + } + return { target: call[0], emit: call[1] as (event: AgentSessionStatusEvent) => void } +} + +// A seeded launch lists a few hundred chats; the host's first snapshot carries all of them. +const CHAT_COUNT = 247 + +const tabs = Array.from({ length: CHAT_COUNT }, (_, index) => ({ + ...structuredTab, + id: `structured-tab-${index}`, + entityId: `session-${index}` +})) + +function chatSummaries(): AgentSessionStatusSummary[] { + return tabs.map((tab, index) => + summary({ + sessionId: tab.entityId, + status: index % 3 === 0 ? 'idle' : 'working', + latestPrompt: `prompt ${index}`, + updatedAt: index + 1 + }) + ) +} + +/** Every store notification, and how many changed the status map. */ +function countPublications(): { all: number; status: number; stop: () => void } { + const counts = { all: 0, status: 0, stop: () => {} } + counts.stop = useAppStore.subscribe((state, previous) => { + counts.all += 1 + if (state.agentStatusByPaneKey !== previous.agentStatusByPaneKey) { + counts.status += 1 + } + }) + return counts +} + +/** The mobile session sync's store subscriber, which rebuilds the status projection per change. */ +function subscribeMobileSync(): () => void { + let previousKey = getRuntimeMobileSessionSyncKey(useAppStore.getState()) + return useAppStore.subscribe((state, previous) => { + if (!canSkipRuntimeMobileSessionSyncKeyBuild(state, previous)) { + previousKey = getRuntimeMobileSessionSyncKey(state, previous, previousKey) + } + }) +} + +/** Every chat in one of five states, shifted each round so each row changes state between rounds. */ +function variedSummaries(round: number): AgentSessionStatusSummary[] { + return tabs.map((tab, index) => { + const kind = (index + round) % 5 + return summary({ + sessionId: tab.entityId, + status: kind === 0 ? 'idle' : kind === 1 ? 'attention' : 'working', + latestPrompt: `prompt ${index} r${round}`, + updatedAt: 100 * round + index + 1, + ...(kind === 0 + ? { turnOutcome: 'interruption' as const, lastAssistantMessage: `said ${index}` } + : {}), + ...(kind === 2 ? { toolName: 'Bash', toolInput: `ls ${index}`, model: 'gpt-x' } : {}), + ...(kind === 3 + ? { + backgroundTasks: [ + { + id: `child-${index}`, + kind: 'agent' as const, + name: 'deep_review', + description: 'Review', + state: 'working' as const, + startedAt: 500 + } + ] + } + : {}), + ...(kind === 4 && round > 0 ? { hostExecutionOwned: undefined } : {}) + }) + }) +} + +/** Three snapshots, then single updates; the status map after each step, and the tab labels. */ +async function snapshotRounds(): Promise<{ rows: unknown[]; labels: unknown[] }> { + await renderBridge() + const rows: unknown[] = [] + for (const round of [0, 1, 2]) { + vi.mocked(Date.now).mockReturnValue(1_000 + round) + await act(async () => feed().emit({ type: 'snapshot', sessions: variedSummaries(round) })) + rows.push(structuredClone(useAppStore.getState().agentStatusByPaneKey)) + } + for (const index of [3, 7, 11]) { + const session = variedSummaries(2)[index]! + await act(async () => + feed().emit({ + type: 'status', + session: { + ...session, + status: 'idle', + turnOutcome: 'interruption', + updatedAt: 50_000 + index + } + }) + ) + } + rows.push(structuredClone(useAppStore.getState().agentStatusByPaneKey)) + const labels = (useAppStore.getState().unifiedTabsByWorktree['wt-1'] ?? []).map((tab) => [ + tab.id, + tab.label, + tab.generatedLabel ?? null + ]) + cleanup() + return { rows, labels } +} + +async function renderBridge(): Promise { + render() + await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) +} + +describe('a status snapshot of many chats', () => { + beforeEach(() => { + vi.clearAllMocks() + resetStructuredAgentSessionStatusFeedsForTests() + mocks.subscribeStatus.mockResolvedValue({ unsubscribe: mocks.unsubscribe }) + mocks.supportsCapability.mockResolvedValue(true) + // The status evidence clock, so rows written on different ticks compare equal. + vi.spyOn(Date, 'now').mockReturnValue(1_000) + useAppStore.setState({ + agentStatusByPaneKey: {}, + settings: { ...getDefaultSettings('/tmp'), tabAutoGenerateTitle: true }, + unifiedTabsByWorktree: { 'wt-1': tabs } + }) + }) + + afterEach(() => { + cleanup() + resetStructuredAgentSessionStatusFeedsForTests() + vi.restoreAllMocks() + }) + + it("applies a snapshot of every listed chat's status as one publication", async () => { + await renderBridge() + const stopMobileSync = subscribeMobileSync() + mocks.mobileStatusProjections.mockClear() + // Every publication re-runs every store reader, so a publication per row is the burst. + const publications = countPublications() + + await act(async () => feed().emit({ type: 'snapshot', sessions: chatSummaries() })) + publications.stop() + stopMobileSync() + + expect(statuses()).toHaveLength(CHAT_COUNT) + expect(publications.status).toBe(1) + expect(publications.all).toBe(1) + expect(mocks.mobileStatusProjections).toHaveBeenCalledOnce() + // The rows' generated-title requests go out as one batch, not a call per row. + expect(mocks.setGeneratedTitle).not.toHaveBeenCalled() + expect(mocks.setGeneratedTitles).toHaveBeenCalledOnce() + expect(mocks.setGeneratedTitles.mock.calls[0][0]).toHaveLength(CHAT_COUNT) + }) + + it('writes the same rows and titles as writing each row on its own', async () => { + const tabsBefore = structuredClone(useAppStore.getState().unifiedTabsByWorktree) + const batched = await snapshotRounds() + resetStructuredAgentSessionStatusFeedsForTests() + mocks.subscribeStatus.mockClear() + useAppStore.setState({ agentStatusByPaneKey: {}, unifiedTabsByWorktree: tabsBefore }) + mocks.writeEachRowAlone = true + const perRow = await snapshotRounds() + mocks.writeEachRowAlone = false + + expect(Object.keys(batched.rows[0] ?? {})).toHaveLength(CHAT_COUNT) + expect(batched).toEqual(perRow) + }) + + it('still removes a row, and applies a later single update on the next tick', async () => { + await renderBridge() + await act(async () => feed().emit({ type: 'snapshot', sessions: chatSummaries() })) + const [first, second] = chatSummaries() + const paneKey = (tab: (typeof tabs)[number]): string => + structuredAgentSessionPaneKey(tab.id, tab.entityId) + expect(useAppStore.getState().agentStatusByPaneKey[paneKey(tabs[0])]).toBeDefined() + + await act(async () => feed().emit({ type: 'status', session: { ...first, status: null } })) + expect(mocks.removeAgentStatus).toHaveBeenCalledWith(paneKey(tabs[0])) + expect(useAppStore.getState().agentStatusByPaneKey[paneKey(tabs[0])]).toBeUndefined() + + const publications = countPublications() + await act(async () => + feed().emit({ type: 'status', session: { ...second, status: 'idle', updatedAt: 9_999 } }) + ) + publications.stop() + expect(publications.status).toBe(1) + expect(useAppStore.getState().agentStatusByPaneKey[paneKey(tabs[1])]).toMatchObject({ + state: 'done', + updatedAt: 9_999 + }) + expect(statuses()).toHaveLength(CHAT_COUNT - 1) + }) +}) diff --git a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-clock.test.tsx b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-clock.test.tsx index a52fd0897c70..0083c6eecb0b 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-clock.test.tsx +++ b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-clock.test.tsx @@ -104,11 +104,11 @@ function unread(): number { return countActivityUnread(state, ACKNOWLEDGED + 60_000) } -async function connect(): Promise<(event: AgentSessionStatusEvent) => void> { +async function connect(): Promise<(event: AgentSessionStatusEvent) => Promise> { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) const emit: (event: AgentSessionStatusEvent) => void = mocks.subscribeStatus.mock.calls[0]?.[1] - return (event) => act(() => emit(event)) + return (event) => act(async () => emit(event)) } describe("the structured row's state clock", () => { @@ -131,43 +131,43 @@ describe("the structured row's state clock", () => { it('keeps a settled parent read while its subagent writes, until its own next turn ends', async () => { const emit = await connect() - emit({ type: 'snapshot', sessions: [summary({ statusStartedAt: SETTLED })] }) + await emit({ type: 'snapshot', sessions: [summary({ statusStartedAt: SETTLED })] }) expect(row()).toMatchObject({ state: 'done', stateStartedAt: SETTLED }) acknowledge(ACKNOWLEDGED) expect(unread()).toBe(0) // The child's rows move the evidence clock, and a host may still republish for them. for (const updatedAt of [24_000, 25_000, 26_000]) { - emit({ type: 'status', session: summary({ statusStartedAt: SETTLED, updatedAt }) }) + await emit({ type: 'status', session: summary({ statusStartedAt: SETTLED, updatedAt }) }) } expect(row()).toMatchObject({ state: 'done', updatedAt: 26_000, stateStartedAt: SETTLED }) expect(unread()).toBe(0) - emit({ + await emit({ type: 'status', session: summary({ status: 'working', statusStartedAt: 30_000, updatedAt: 30_000 }) }) - emit({ type: 'status', session: summary({ statusStartedAt: 31_000, updatedAt: 31_000 }) }) + await emit({ type: 'status', session: summary({ statusStartedAt: 31_000, updatedAt: 31_000 }) }) expect(row()).toMatchObject({ state: 'done', stateStartedAt: 31_000 }) expect(unread()).toBe(1) }) it("keeps the old dating for an older host's summary, which carries no clock", async () => { const emit = await connect() - emit({ type: 'snapshot', sessions: [summary()] }) + await emit({ type: 'snapshot', sessions: [summary()] }) acknowledge(ACKNOWLEDGED) expect(unread()).toBe(0) - emit({ type: 'status', session: summary({ updatedAt: 26_000 }) }) + await emit({ type: 'status', session: summary({ updatedAt: 26_000 }) }) expect(row()).toMatchObject({ state: 'done', stateStartedAt: 26_000 }) expect(unread()).toBe(1) }) it("dates a subagent's approval at the ask and leaves the parent's completion where it was", async () => { const emit = await connect() - emit({ type: 'snapshot', sessions: [summary({ statusStartedAt: SETTLED })] }) + await emit({ type: 'snapshot', sessions: [summary({ statusStartedAt: SETTLED })] }) acknowledge(ACKNOWLEDGED) - emit({ + await emit({ type: 'status', session: summary({ status: 'attention', statusStartedAt: 27_000, updatedAt: 27_000 }) }) @@ -180,7 +180,10 @@ describe("the structured row's state clock", () => { // Answered: the parent is idle again, still dated by its own last turn. acknowledge(28_000) - emit({ type: 'status', session: summary({ statusStartedAt: SETTLED, updatedAt: 28_500 }) }) + await emit({ + type: 'status', + session: summary({ statusStartedAt: SETTLED, updatedAt: 28_500 }) + }) expect(row()).toMatchObject({ state: 'done', stateStartedAt: SETTLED }) expect(agentEntryCompletionAt(row())).toBe(SETTLED) expect(unread()).toBe(0) @@ -189,11 +192,11 @@ describe("the structured row's state clock", () => { it('leaves a row child work holds open on its own continuity, and settles it on the parent clock', async () => { const emit = await connect() const child = { id: 'child-1', kind: 'agent', state: 'working' } as const - emit({ + await emit({ type: 'snapshot', sessions: [summary({ status: 'working', statusStartedAt: 10_000, updatedAt: 10_000 })] }) - emit({ + await emit({ type: 'status', session: summary({ statusStartedAt: SETTLED, updatedAt: 24_000, backgroundTasks: [child] }) }) @@ -204,7 +207,7 @@ describe("the structured row's state clock", () => { mainAgent: { state: 'done', stateStartedAt: SETTLED } }) - emit({ + await emit({ type: 'status', session: summary({ statusStartedAt: SETTLED, diff --git a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-failed-start.test.tsx b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-failed-start.test.tsx index f647ed24415f..d1645e5f23e9 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-failed-start.test.tsx +++ b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-failed-start.test.tsx @@ -281,6 +281,8 @@ describe('a chat whose start failed', () => { updatedAt: 50_000 } }) + // Rows of one tick land together, after the tick. + await flush() expect(rows()).toEqual([expect.objectContaining({ state: 'working' })]) }) diff --git a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-stopping.test.tsx b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-stopping.test.tsx index fc1fc8430c30..3dc29b3fa388 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-stopping.test.tsx +++ b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-stopping.test.tsx @@ -108,10 +108,10 @@ describe("the sidebar row's Stopping", () => { await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) const emit: (event: AgentSessionStatusEvent) => void = mocks.subscribeStatus.mock.calls[0]?.[1] - act(() => emit({ type: 'status', session: summary({ stopping: true }) })) + await act(async () => emit({ type: 'status', session: summary({ stopping: true }) })) expect(row().mainAgent).toMatchObject({ state: 'working', stopping: true }) - act(() => emit({ type: 'status', session: summary() })) + await act(async () => emit({ type: 'status', session: summary() })) expect(row().state).toBe('working') expect(row().mainAgent).not.toHaveProperty('stopping') }) diff --git a/src/renderer/src/components/native-chat/structured-child-records-switch.test.tsx b/src/renderer/src/components/native-chat/structured-child-records-switch.test.tsx index a5b4b911793c..f82ba549ad9c 100644 --- a/src/renderer/src/components/native-chat/structured-child-records-switch.test.tsx +++ b/src/renderer/src/components/native-chat/structured-child-records-switch.test.tsx @@ -235,7 +235,7 @@ describe('the switch: both surfaces read the host child records', () => { } const rendered = render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed()({ type: 'snapshot', sessions: [summary({ children })] })) + await act(async () => feed()({ type: 'snapshot', sessions: [summary({ children })] })) const header = rendered.container.querySelector('[aria-expanded]') if (header) { fireEvent.click(header) @@ -275,9 +275,11 @@ describe('the switch: both surfaces read the host child records', () => { const children = [view('a')] await mountWith(children) const first = row().children - act(() => feed()({ type: 'status', session: summary({ children: structuredClone(children) }) })) + await act(async () => + feed()({ type: 'status', session: summary({ children: structuredClone(children) }) }) + ) expect(row().children).toBe(first) - act(() => + await act(async () => feed()({ type: 'status', session: summary({ children: [{ ...children[0]!, lastMessage: 'Found 3 call sites' }] }) @@ -299,7 +301,7 @@ describe('the switch: both surfaces read the host child records', () => { expect(stripRows(container)).toEqual([live]) // The status stream drops: the host is not heard from, so neither surface vouches for it. - act(() => feed()({ type: 'end' })) + await act(async () => feed()({ type: 'end' })) const lost = { dot: 'No recent update', lead: 'Audit a', trail: 'No update in 2m' } await waitFor(() => expect(sidebarRows()).toEqual([lost])) expect(stripRows(container)).toEqual([lost]) @@ -308,7 +310,7 @@ describe('the switch: both surfaces read the host child records', () => { it('reads a stale parent row the same way on both surfaces', async () => { const { container } = await mountWith([view('a')]) // The host no longer owns the session, and its last word is older than the freshness window. - act(() => + await act(async () => feed()({ type: 'status', session: summary({ @@ -334,7 +336,7 @@ describe('the switch: both surfaces read the host child records', () => { // The child finishes: the host sends no running children, so no roster and no summary rows. rerender() - act(() => feed()({ type: 'status', session: summary({ children: undefined }) })) + await act(async () => feed()({ type: 'status', session: summary({ children: undefined }) })) expect(strip()).toBeNull() expect(sidebarRows()).toEqual([]) diff --git a/tests/e2e/codex-child-approval-activity-row.unit.test.ts b/tests/e2e/codex-child-approval-activity-row.unit.test.ts index 9b1dc2306e2c..5079cae25b08 100644 --- a/tests/e2e/codex-child-approval-activity-row.unit.test.ts +++ b/tests/e2e/codex-child-approval-activity-row.unit.test.ts @@ -216,8 +216,9 @@ describe("a Codex subagent's answered approval on the settled parent's Activity const toRenderer: (event: AgentSessionStatusEvent) => void = mocks.subscribeStatus.mock.calls[0]?.[1] let forwarded = 0 - const deliver = (): AgentSessionStatusSummary => { - act(() => { + // Async, so the bridge's once-per-tick store update lands before the row is read. + const deliver = async (): Promise => { + await act(async () => { for (const event of host.events.slice(forwarded)) { toRenderer(event) } @@ -249,7 +250,7 @@ describe("a Codex subagent's answered approval on the settled parent's Activity host.on(CODEX_CHILD, 'turn/started', { turn: { id: 'child-turn' } }) host.on(CODEX_THREAD, 'turn/completed', { turn: { id: 'parent-turn', status: 'completed' } }) await host.drain() - const settled = deliver() + const settled = await deliver() expect(settled).toMatchObject({ status: 'idle', statusStartedAt: expect.any(Number) }) store().setState({ acknowledgedAgentsByPaneKey: { [paneKey()]: host.tick() } }) @@ -263,7 +264,7 @@ describe("a Codex subagent's answered approval on the settled parent's Activity promptKey: 'child-approval' }) await host.drain() - const asked = deliver() + const asked = await deliver() expect(asked).toMatchObject({ status: 'attention' }) expect(asked.statusStartedAt).toBeGreaterThan(settled.statusStartedAt ?? Infinity) // The user reads the ask, then answers it. @@ -272,7 +273,7 @@ describe("a Codex subagent's answered approval on the settled parent's Activity await host.answer(approval ?? '') host.on(CODEX_CHILD, 'turn/completed', { turn: { id: 'child-turn', status: 'completed' } }) await host.drain() - const answered = deliver() + const answered = await deliver() // The host rule under test elsewhere: the answer never re-dates the session's done. expect(answered).toMatchObject({ status: 'idle', statusStartedAt: settled.statusStartedAt }) expect(answered.updatedAt).toBeGreaterThan(asked.updatedAt) diff --git a/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts b/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts index ac18fe159869..921d788e5bfa 100644 --- a/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts +++ b/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts @@ -17,10 +17,13 @@ import { createTrackedJournalOpener, insertTestJournalRowJson, liveTestJournalRows, + loadTestJournal, openTestJournalHostDatabase, + readTestJournalSessionStatus, SAVED_BY_NEWER_ORCA } from '../../../src/main/native-chat/agent-session-journal/journal-host-database-test-support' import type { JournalRow } from '../../../src/main/native-chat/agent-session-journal/journal-row-schema' +import { deriveJournalSessionStatus } from '../../../src/main/native-chat/agent-session-journal/journal-session-state' import { importReleaseCheckoutModule, materializeReleaseCheckout } from './release-checkout' import { agentJournalItemKey } from '../../../src/shared/agent-session-journal-item-key' import { agentSessionFailureFact } from '../../../src/shared/agent-session-failure' @@ -262,7 +265,7 @@ function storedRows(directory: string): string[] { } } -test("an older build opens this build's journal writable and appends to it; the pause survives the round trip", async () => { +test("an older build opens this build's journal writable and appends to it; the pause survives the round trip, and the stale status is derived again", async () => { const directory = mkdtempSync(join(tmpdir(), 'orca-stop-event-writable-downgrade-')) const journals = createTrackedJournalOpener() const itemIds = (journal: Pick) => @@ -279,6 +282,7 @@ test("an older build opens this build's journal writable and appends to it; the ]) await journals.closeAll() const rowsBefore = storedRows(directory) + expect(readTestJournalSessionStatus(directory, IDENTITY.sessionId)).not.toBeNull() const checkout = await materializeReleaseCheckout(WRITABLE_BASELINE_REF) const support = await importReleaseCheckoutModule( @@ -298,9 +302,16 @@ test("an older build opens this build's journal writable and appends to it; the const rowsAfter = storedRows(directory) expect(rowsAfter.slice(0, rowsBefore.length)).toEqual(rowsBefore) expect(rowsAfter).toHaveLength(rowsBefore.length + 1) + // The older build kept no status: the row names a tip that is no longer the chat's, so it reads + // as missing until this build derives it again. + expect(readTestJournalSessionStatus(directory, IDENTITY.sessionId)).toBeNull() // Upgraded again: the older build's row folds, and the person's Stop still pauses the queue. const upgraded = await journals.open({ identity: IDENTITY, stateDirectory: directory }) + upgraded.sessionStatus.backfill() + expect(readTestJournalSessionStatus(directory, IDENTITY.sessionId)).toEqual( + deriveJournalSessionStatus(loadTestJournal(directory, IDENTITY.sessionId)!.state) + ) expect(upgraded.cursor().sequence).toBe(wrote.cursor.sequence + 1) expect(itemIds(upgraded)).toEqual([...wrote.items, 'codex:thread-1:turn-1:2']) expect(upgraded.queuedMessages.pauses('host-a').map((pause) => pause.reason)).toEqual([ diff --git a/tests/e2e/structured-session-reopened-tab-close.unit.test.ts b/tests/e2e/structured-session-reopened-tab-close.unit.test.ts index 35547145c32f..870e48454f7d 100644 --- a/tests/e2e/structured-session-reopened-tab-close.unit.test.ts +++ b/tests/e2e/structured-session-reopened-tab-close.unit.test.ts @@ -128,6 +128,10 @@ async function setup() { // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: restore and tab close call only these host members. { reconcileRestartLeases: async () => undefined, + startup: { + seedStoredStatuses: (ids: readonly string[]) => [...ids], + settleOwedSessions: async () => undefined + }, restoreReadableSessions: async () => undefined, close: async (sessionId: string) => { hostSessionCloses.push(sessionId)