From 23b927e859e93523ca523c654eefcd63f0daceee Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:49:30 -0700 Subject: [PATCH 01/11] Move the turn message ordinals and the turn-row revision to the neutral timeline folder Pure moves so a shared timeline assembler can use them: Codex's message ordinal counter becomes ProviderTurnMessageOrdinals and Claude's turn-row revision becomes the provider-neutral agent-journal turn-row revision. Only names and import paths change. --- config/scripts/ci-shard-timings.json | 2 +- src/main/claude/claude-context-facts.ts | 11 +- src/main/claude/claude-open-turn.ts | 4 +- .../codex-structured-item-translation.test.ts | 28 ++--- .../codex-structured-item-translation.ts | 8 +- .../codex/codex-structured-journal-items.ts | 4 +- .../codex-structured-journal-settlement.ts | 4 +- ...-structured-journal-translation-restore.ts | 7 +- ...ctured-journal-translation-streams.test.ts | 4 +- ...ex-structured-journal-translation-turns.ts | 2 +- src/main/codex/codex-thread-item-identity.ts | 8 +- .../agent-journal-turn-row-revision.ts} | 106 +++++++++++------- .../provider-turn-message-ordinals.test.ts} | 25 +++-- .../provider-turn-message-ordinals.ts} | 19 +--- 14 files changed, 130 insertions(+), 102 deletions(-) rename src/main/{claude/claude-turn-row-revision.ts => native-chat/agent-session-timeline/agent-journal-turn-row-revision.ts} (67%) rename src/main/{codex/codex-turn-ordinals.test.ts => native-chat/agent-session-timeline/provider-turn-message-ordinals.test.ts} (64%) rename src/main/{codex/codex-turn-ordinals.ts => native-chat/agent-session-timeline/provider-turn-message-ordinals.ts} (88%) diff --git a/config/scripts/ci-shard-timings.json b/config/scripts/ci-shard-timings.json index 73d0226943b8..fb242a4788c8 100644 --- a/config/scripts/ci-shard-timings.json +++ b/config/scripts/ci-shard-timings.json @@ -1307,7 +1307,7 @@ "src/main/codex/codex-trust-grant-ledger.test.ts": 54, "src/main/codex/codex-trust-grant-main-thread-boundary.test.ts": 43, "src/main/codex/codex-trust-grant-telemetry.test.ts": 70, - "src/main/codex/codex-turn-ordinals.test.ts": 96, + "src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.test.ts": 96, "src/main/codex/codex-user-hook-trust-rebase-client.test.ts": 104, "src/main/codex/codex-user-hook-trust-moves.test.ts": 127, "src/main/codex/codex-wsl-hook-install-plan.test.ts": 51, diff --git a/src/main/claude/claude-context-facts.ts b/src/main/claude/claude-context-facts.ts index 358fd134c87a..3a1326e09363 100644 --- a/src/main/claude/claude-context-facts.ts +++ b/src/main/claude/claude-context-facts.ts @@ -25,7 +25,10 @@ import type { ClaudeOpenTurn } from './claude-open-turn' import { claudeRecord, claudeText } from './claude-structured-item-translation' import type { ClaudeTurnEnd } from './claude-turn-lifecycle-item' import { isRootClaudeFrame } from './claude-turn-opening' -import { writeClaudeTurnRow, type ClaudeTurnRowTarget } from './claude-turn-row-revision' +import { + writeAgentJournalTurnRow, + type AgentJournalTurnRowTarget +} from '../native-chat/agent-session-timeline/agent-journal-turn-row-revision' const CONVERSATION_FRAME_TYPES = new Set(['assistant', 'user', 'stream_event']) @@ -188,7 +191,7 @@ export class ClaudeContextFacts { this.windowHint = report.windowTokens const contextUsage: AgentSessionContextUsage = part === 'report' ? { used: { kind: 'report', ...report }, window } : { window } - writeClaudeTurnRow( + writeAgentJournalTurnRow( this.sink, target === null ? { newest: true } : { identity: target }, { contextUsage }, @@ -211,9 +214,9 @@ export class ClaudeContextFacts { windowIfNoneHeld?: AgentSessionContextWindow ): void { const identity = this.turn.identity - const target: ClaudeTurnRowTarget = identity ? { identity } : { newest: true } + const target: AgentJournalTurnRowTarget = identity ? { identity } : { newest: true } // A context fact often lands with no later frame to publish it, so it publishes itself. - writeClaudeTurnRow( + writeAgentJournalTurnRow( this.sink, target, { contextUsage, ...(windowIfNoneHeld ? { windowIfNoneHeld } : {}) }, diff --git a/src/main/claude/claude-open-turn.ts b/src/main/claude/claude-open-turn.ts index ddc1bd553907..4e7fe7ff9530 100644 --- a/src/main/claude/claude-open-turn.ts +++ b/src/main/claude/claude-open-turn.ts @@ -18,7 +18,7 @@ import { type ClaudeCurrentTurn, type ClaudeTurnEnd } from './claude-turn-lifecycle-item' -import { writeClaudeTurnRow } from './claude-turn-row-revision' +import { writeAgentJournalTurnRow } from '../native-chat/agent-session-timeline/agent-journal-turn-row-revision' import type { ClaudeCommandTurn } from './claude-command-turn' import { createClaudeTurnOpener, type ClaudeTurnSource } from './claude-turn-opening' @@ -203,7 +203,7 @@ export class ClaudeOpenTurn { contextUsage?: AgentSessionContextUsage ): void { const item = claudeTurnLifecycleItem(turn, end) - writeClaudeTurnRow( + writeAgentJournalTurnRow( this.deps.sink, { identity: item.identity }, { lifecycle: item.body, ...(contextUsage ? { contextUsage } : {}) }, diff --git a/src/main/codex/codex-structured-item-translation.test.ts b/src/main/codex/codex-structured-item-translation.test.ts index ad63d624f58b..29562552a02c 100644 --- a/src/main/codex/codex-structured-item-translation.test.ts +++ b/src/main/codex/codex-structured-item-translation.test.ts @@ -11,9 +11,9 @@ import { codexJournalItem, codexMessageBlocks, codexStreamingJournalItem, - CodexTurnOrdinals, - MAX_CODEX_TURN_ORDINAL_BYTES, - MAX_CODEX_TURN_ORDINAL_ENTRIES, + ProviderTurnMessageOrdinals, + MAX_PROVIDER_TURN_ORDINAL_BYTES, + MAX_PROVIDER_TURN_ORDINAL_ENTRIES, isCodexMessageItemType, readCodexThreadItem, type CodexThreadItem @@ -59,7 +59,7 @@ const RESUMED_TURN: CodexThreadItem[] = [ ] function keysFor(items: CodexThreadItem[]): string[] { - const ordinals = new CodexTurnOrdinals() + const ordinals = new ProviderTurnMessageOrdinals() return items .filter((item) => isCodexMessageItemType(item.type)) .map((item) => @@ -71,15 +71,15 @@ function keysFor(items: CodexThreadItem[]): string[] { describe('codex turn ordinals', () => { it('bounds forgotten turn tombstones while retaining the recent window', () => { - const ordinals = new CodexTurnOrdinals() - const total = MAX_CODEX_TURN_ORDINAL_ENTRIES + 12 + const ordinals = new ProviderTurnMessageOrdinals() + const total = MAX_PROVIDER_TURN_ORDINAL_ENTRIES + 12 for (let index = 0; index < total; index += 1) { const turnId = `turn-${index}` expect(ordinals.ordinalFor('thread-many', turnId, 'item-0')).toBe(0) ordinals.forgetTurn('thread-many', turnId) } - expect(ordinals.forgottenTurnCount).toBe(MAX_CODEX_TURN_ORDINAL_ENTRIES) + expect(ordinals.forgottenTurnCount).toBe(MAX_PROVIDER_TURN_ORDINAL_ENTRIES) // The newest completed turn still keeps its counter for a late frame. expect(ordinals.ordinalFor('thread-many', `turn-${total - 1}`, 'item-late')).toBe(1) // The oldest turn was deterministically evicted and starts a fresh key. @@ -87,7 +87,7 @@ describe('codex turn ordinals', () => { }) it('releases a forgotten turn without ever reusing an ordinal it assigned', () => { - const ordinals = new CodexTurnOrdinals() + const ordinals = new ProviderTurnMessageOrdinals() expect(ordinals.ordinalFor('thread-1', 'turn-1', 'item-1')).toBe(0) expect(ordinals.ordinalFor('thread-1', 'turn-1', 'item-2')).toBe(1) @@ -102,12 +102,12 @@ describe('codex turn ordinals', () => { }) it('bounds aggregate provider identifier bytes retained by one active turn', () => { - const ordinals = new CodexTurnOrdinals() + const ordinals = new ProviderTurnMessageOrdinals() for (let index = 0; index < 3_000; index += 1) { ordinals.ordinalFor('thread', 'turn', `${index}:${'x'.repeat(512)}`) } - expect(ordinals.bytes).toBeLessThanOrEqual(MAX_CODEX_TURN_ORDINAL_BYTES) + expect(ordinals.bytes).toBeLessThanOrEqual(MAX_PROVIDER_TURN_ORDINAL_BYTES) }) }) @@ -117,7 +117,7 @@ describe('codex item identity', () => { }) it('numbers messages 0,1,2 on both sides — the projection skips the dropped command', () => { - const ordinals = new CodexTurnOrdinals() + const ordinals = new ProviderTurnMessageOrdinals() const live = LIVE_TURN.map((item) => codexItemIdentity({ threadId: THREAD_ID, turnId: TURN_ID, item, ordinals }) ) @@ -136,7 +136,7 @@ describe('codex item identity', () => { }) it('assigns an ordinal once and reuses it, so a delta and its completion upsert one row', () => { - const ordinals = new CodexTurnOrdinals() + const ordinals = new ProviderTurnMessageOrdinals() ordinals.ordinalFor(THREAD_ID, TURN_ID, 'item-0') expect(ordinals.ordinalFor(THREAD_ID, TURN_ID, 'item-1')).toBe(1) @@ -144,14 +144,14 @@ describe('codex item identity', () => { }) it('restarts numbering per turn', () => { - const ordinals = new CodexTurnOrdinals() + const ordinals = new ProviderTurnMessageOrdinals() ordinals.ordinalFor(THREAD_ID, TURN_ID, 'item-0') expect(ordinals.ordinalFor(THREAD_ID, 'turn-2', 'item-1')).toBe(0) }) it('keys a non-message item and a turnless message in the orca namespace', () => { - const ordinals = new CodexTurnOrdinals() + const ordinals = new ProviderTurnMessageOrdinals() const command = codexItemIdentity({ threadId: THREAD_ID, turnId: TURN_ID, diff --git a/src/main/codex/codex-structured-item-translation.ts b/src/main/codex/codex-structured-item-translation.ts index 25b821d90d9f..3f4229cb6e54 100644 --- a/src/main/codex/codex-structured-item-translation.ts +++ b/src/main/codex/codex-structured-item-translation.ts @@ -28,10 +28,10 @@ export { type CodexThreadItem } from './codex-thread-item-identity' export { - CodexTurnOrdinals, - MAX_CODEX_TURN_ORDINAL_BYTES, - MAX_CODEX_TURN_ORDINAL_ENTRIES -} from './codex-turn-ordinals' + ProviderTurnMessageOrdinals, + MAX_PROVIDER_TURN_ORDINAL_BYTES, + MAX_PROVIDER_TURN_ORDINAL_ENTRIES +} from '../native-chat/agent-session-timeline/provider-turn-message-ordinals' // Codex thread items → journal item bodies. diff --git a/src/main/codex/codex-structured-journal-items.ts b/src/main/codex/codex-structured-journal-items.ts index ec6ac67df5d4..22bb16c8ef08 100644 --- a/src/main/codex/codex-structured-journal-items.ts +++ b/src/main/codex/codex-structured-journal-items.ts @@ -7,7 +7,7 @@ import { requiresTerminalSettlement } from '../native-chat/agent-session-journal import { codexItemIdentity, codexJournalItem, - CodexTurnOrdinals, + ProviderTurnMessageOrdinals, readCodexThreadItem, type CodexThreadItem } from './codex-structured-item-translation' @@ -35,7 +35,7 @@ import { readCodexDispatchEcho } from './codex-structured-dispatch-echo' import type { CodexRowAttribution } from './codex-subagent-linkage' export class CodexJournalItems { - readonly ordinals = new CodexTurnOrdinals() + readonly ordinals = new ProviderTurnMessageOrdinals() readonly activeItems = new Map() readonly streams private readonly identities = new Map() diff --git a/src/main/codex/codex-structured-journal-settlement.ts b/src/main/codex/codex-structured-journal-settlement.ts index 131de9824aae..e7d0a376eca8 100644 --- a/src/main/codex/codex-structured-journal-settlement.ts +++ b/src/main/codex/codex-structured-journal-settlement.ts @@ -17,7 +17,7 @@ import { codexJournalItem, codexStreamingJournalItem, type CodexThreadItem, - type CodexTurnOrdinals + type ProviderTurnMessageOrdinals } from './codex-structured-item-translation' import type { CodexStructuredItemStreams } from './codex-structured-item-streams' import type { CodexHelperName } from './codex-collab-agent-item-translation' @@ -56,7 +56,7 @@ export function settleCodexJournalSession(input: { pendingPrompts: ReadonlyMap currentTurnIds: ReadonlyMap> primaryThreadId: string | null - ordinals: CodexTurnOrdinals + ordinals: ProviderTurnMessageOrdinals /** Terminal lifecycle for a turn the provider left running when it ended; null for a turn a * conversation command claimed, whose record the host settles. */ settledTurnLifecycle: (threadId: string, turnId: string) => AgentJournalTurnLifecycle | null diff --git a/src/main/codex/codex-structured-journal-translation-restore.ts b/src/main/codex/codex-structured-journal-translation-restore.ts index 9f7efc15ae4c..a3b4652f366e 100644 --- a/src/main/codex/codex-structured-journal-translation-restore.ts +++ b/src/main/codex/codex-structured-journal-translation-restore.ts @@ -1,5 +1,8 @@ import type { AgentJournalTurnLifecycle } from '../../shared/agent-session-journal-types' -import { readCodexThreadItem, type CodexTurnOrdinals } from './codex-structured-item-translation' +import { + readCodexThreadItem, + type ProviderTurnMessageOrdinals +} from './codex-structured-item-translation' import type { CodexJournalCompactions } from './codex-structured-journal-compactions' import type { CodexJournalItems } from './codex-structured-journal-items' import { @@ -55,7 +58,7 @@ export function restoreCodexJournalThread(input: { threadId: string thread: Record currentTurnIds: Map> - ordinals: CodexTurnOrdinals + ordinals: ProviderTurnMessageOrdinals handleItem: (event: CodexHistoryItemEvent) => CodexJournalTranslationAdmission /** Absent when the caller has no session identity to key lifecycle rows by. */ restoreTurnLifecycle?: ( diff --git a/src/main/codex/codex-structured-journal-translation-streams.test.ts b/src/main/codex/codex-structured-journal-translation-streams.test.ts index 54a2c017aa05..7bb88edc3a6f 100644 --- a/src/main/codex/codex-structured-journal-translation-streams.test.ts +++ b/src/main/codex/codex-structured-journal-translation-streams.test.ts @@ -6,7 +6,7 @@ import type { import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key' import { projectStructuredItemsToNativeChat } from '../../shared/structured-agent-session-projection' import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' -import { CodexTurnOrdinals } from './codex-structured-item-translation' +import { ProviderTurnMessageOrdinals } from './codex-structured-item-translation' import { createCodexJournalTranslator, MAX_CODEX_GENERIC_BOOKKEEPING_ENTRIES, @@ -164,7 +164,7 @@ describe('codex journal translation', () => { }) it('releases a turn ordinal map when the turn completes', () => { - const spy = vi.spyOn(CodexTurnOrdinals.prototype, 'forgetTurn') + const spy = vi.spyOn(ProviderTurnMessageOrdinals.prototype, 'forgetTurn') try { const { translator } = translatorWith() translator.handle(TURN_STARTED) diff --git a/src/main/codex/codex-structured-journal-translation-turns.ts b/src/main/codex/codex-structured-journal-translation-turns.ts index 33f138c6578d..dc955f70385c 100644 --- a/src/main/codex/codex-structured-journal-translation-turns.ts +++ b/src/main/codex/codex-structured-journal-translation-turns.ts @@ -8,7 +8,7 @@ import { } from '../../shared/agent-session-journal-types' import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key' import { agentJournalTurnBody } from '../../shared/agent-session-turn-record' -import { CODEX_USER_MESSAGE_ORDINAL } from './codex-turn-ordinals' +import { CODEX_USER_MESSAGE_ORDINAL } from './codex-thread-item-identity' import type { StructuredAgentSessionEventSink, StructuredAgentSessionSinkAdmission diff --git a/src/main/codex/codex-thread-item-identity.ts b/src/main/codex/codex-thread-item-identity.ts index 0488e5c00e98..663b497f7731 100644 --- a/src/main/codex/codex-thread-item-identity.ts +++ b/src/main/codex/codex-thread-item-identity.ts @@ -1,5 +1,5 @@ import type { AgentJournalItemIdentity } from '../../shared/agent-session-journal-types' -import type { CodexTurnOrdinals } from './codex-turn-ordinals' +import type { ProviderTurnMessageOrdinals } from '../native-chat/agent-session-timeline/provider-turn-message-ordinals' // Codex thread items → durable journal identities. // @@ -15,6 +15,10 @@ import type { CodexTurnOrdinals } from './codex-turn-ordinals' // including ones this build does not model — is skipped identically on both // sides, which is what makes the key survive a Codex release that adds one. +/** Codex records the user message first in a turn, so a restored submission is + * ordinal 0 of `(threadId, turnId)`. */ +export const CODEX_USER_MESSAGE_ORDINAL = 0 + /** Only these carry a durable `(threadId, turnId, ordinal)` identity. */ const CODEX_MESSAGE_ITEM_TYPES = new Set(['userMessage', 'agentMessage']) @@ -50,7 +54,7 @@ export function codexItemIdentity(input: { threadId: string turnId: string | null item: CodexThreadItem - ordinals: CodexTurnOrdinals + ordinals: ProviderTurnMessageOrdinals }): AgentJournalItemIdentity { const { item, turnId } = input if (turnId && isCodexMessageItemType(item.type)) { diff --git a/src/main/claude/claude-turn-row-revision.ts b/src/main/native-chat/agent-session-timeline/agent-journal-turn-row-revision.ts similarity index 67% rename from src/main/claude/claude-turn-row-revision.ts rename to src/main/native-chat/agent-session-timeline/agent-journal-turn-row-revision.ts index 1b684ba89aaa..69356c406784 100644 --- a/src/main/claude/claude-turn-row-revision.ts +++ b/src/main/native-chat/agent-session-timeline/agent-journal-turn-row-revision.ts @@ -1,4 +1,4 @@ -// Every write to a Claude turn row is a revision of the row as the journal holds +// Every write to a turn row is a revision of the row as the journal holds // it at execution: a writer overrides only the fields it owns and keeps the // rest, whoever wrote them. Nothing about ended turns is kept in memory, so a // restart or reattach revises the same rows a live translator would. @@ -9,34 +9,37 @@ import { MAX_CONTEXT_MODEL_ID_CHARS, type AgentSessionContextUsage, type AgentSessionContextWindow -} from '../../shared/agent-session-context-usage' +} from '../../../shared/agent-session-context-usage' import { agentJournalItemKey, parseAgentJournalItemKey -} from '../../shared/agent-session-journal-item-key' +} from '../../../shared/agent-session-journal-item-key' import { AGENT_JOURNAL_THREAD_SCOPE, type AgentJournalItemIdentity, type AgentJournalTurnItem -} from '../../shared/agent-session-journal-types' -import { readAgentJournalTurn } from '../../shared/agent-session-turn-record' -import { estimateStructuredAgentSessionItemBytes } from '../native-chat/agent-session-wire/structured-agent-session-event-sink-estimate' +} from '../../../shared/agent-session-journal-types' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { estimateStructuredAgentSessionItemBytes } from '../agent-session-wire/structured-agent-session-event-sink-estimate' import type { StructuredAgentSessionEventSink, StructuredAgentSessionRevisionJournal, - StructuredAgentSessionRevisionOptions -} from '../native-chat/agent-session-wire/structured-agent-session-event-sink' + StructuredAgentSessionRevisionOptions, + StructuredAgentSessionSinkAdmission +} from '../agent-session-wire/structured-agent-session-event-sink' /** The row a write revises: a known one, or the newest turn in the journal. */ -export type ClaudeTurnRowTarget = { identity: AgentJournalItemIdentity } | { newest: true } +export type AgentJournalTurnRowTarget = { identity: AgentJournalItemIdentity } | { newest: true } -export type ClaudeTurnRowWrite = { +export type AgentJournalTurnRowWrite = { /** The lifecycle fields this turn has now; the lifecycle writer owns all of them. */ lifecycle?: AgentJournalTurnItem /** Context parts, each replacing its namesake on the row. */ contextUsage?: AgentSessionContextUsage /** A window written only while no row in the journal holds one. */ windowIfNoneHeld?: AgentSessionContextWindow + /** Lands only while the row is absent or still running, so an ended turn is never rewritten. */ + onlyWhileRunning?: true } function jsonBytes(value: unknown): number { @@ -84,7 +87,7 @@ const TURN_ROW_BYTES_WITHOUT_CONTEXT = 8 * 1024 /** `lifecycle` over the row's current body, keeping every field it does not own. */ function reviseTurnBody( current: AgentJournalTurnItem | null, - write: ClaudeTurnRowWrite + write: AgentJournalTurnRowWrite ): AgentJournalTurnItem | null { const { lifecycle, contextUsage } = write const base = @@ -113,8 +116,8 @@ function withoutLifecycle(turn: AgentJournalTurnItem) { /** The write with its fallback window resolved against every row, since any of them may hold the newest. */ function withFallbackWindow( journal: StructuredAgentSessionRevisionJournal, - { windowIfNoneHeld, ...write }: ClaudeTurnRowWrite -): ClaudeTurnRowWrite { + { windowIfNoneHeld, ...write }: AgentJournalTurnRowWrite +): AgentJournalTurnRowWrite { if (!windowIfNoneHeld || write.contextUsage?.window) { return write } @@ -129,7 +132,7 @@ function withFallbackWindow( function findTurnRow( journal: StructuredAgentSessionRevisionJournal, - target: ClaudeTurnRowTarget + target: AgentJournalTurnRowTarget ): { itemId: string; body: AgentJournalTurnItem } | null { if ('identity' in target) { const itemId = agentJournalItemKey(target.identity) @@ -147,23 +150,60 @@ function findTurnRow( } /** How a turn-row write reaches live subscribers. */ -export type ClaudeTurnRowDelivery = { +export type AgentJournalTurnRowDelivery = { /** False only for a writer that publishes each write itself right after queueing it. */ publish: boolean options?: Omit } +/** Bytes a turn-row write may resolve to. */ +export function agentJournalTurnRowReservedBytes( + target: AgentJournalTurnRowTarget, + write: AgentJournalTurnRowWrite +): number { + const known = 'identity' in target ? target.identity : null + return ( + (known && write.lifecycle + ? estimateStructuredAgentSessionItemBytes(known, write.lifecycle) + : TURN_ROW_BYTES_WITHOUT_CONTEXT) + contextBytesBound(write.contextUsage) + ) +} + +/** The row a turn-row write lands as, read from the journal at execution; null writes nothing. */ +export function resolveAgentJournalTurnRowWrite( + journal: StructuredAgentSessionRevisionJournal, + target: AgentJournalTurnRowTarget, + write: AgentJournalTurnRowWrite, + reservedBytes: number +): { identity: AgentJournalItemIdentity; body: AgentJournalTurnItem } | null { + const known = 'identity' in target ? target.identity : null + const row = findTurnRow(journal, target) + if (write.onlyWhileRunning && row && row.body.state !== 'running') { + return null + } + const identity = known ?? (row ? parseAgentJournalItemKey(row.itemId) : null) + const body = reviseTurnBody(row?.body ?? null, withFallbackWindow(journal, write)) + if (!identity || !body) { + return null + } + if (estimateStructuredAgentSessionItemBytes(identity, body) <= reservedBytes) { + return { identity, body } + } + // Only a row grown by fields this build does not know gets here; the lifecycle still lands. + return write.lifecycle ? { identity, body: write.lifecycle } : null +} + /** - * Queue one revision of a Claude turn row. A lifecycle write creates the row + * Queue one revision of a turn row. A lifecycle write creates the row * when it is absent; a context write only ever revises one that exists. * Without a journal-reading sink only the lifecycle is written, as it was built. */ -export function writeClaudeTurnRow( +export function writeAgentJournalTurnRow( sink: StructuredAgentSessionEventSink, - target: ClaudeTurnRowTarget, - write: ClaudeTurnRowWrite, - { publish, options: delivery = {} }: ClaudeTurnRowDelivery -): void { + target: AgentJournalTurnRowTarget, + write: AgentJournalTurnRowWrite, + { publish, options: delivery = {} }: AgentJournalTurnRowDelivery +): StructuredAgentSessionSinkAdmission { // A turn record belongs to no turn. const options = { ...delivery, turnScope: AGENT_JOURNAL_THREAD_SCOPE } const revise = publish ? sink.tryReviseResolvedItemAndPublish : sink.tryReviseResolvedItem @@ -174,29 +214,13 @@ export function writeClaudeTurnRow( sink.publish() } } - return + return { accepted: true } } - const known = 'identity' in target ? target.identity : null - const reservedBytes = - (known && write.lifecycle - ? estimateStructuredAgentSessionItemBytes(known, write.lifecycle) - : TURN_ROW_BYTES_WITHOUT_CONTEXT) + contextBytesBound(write.contextUsage) - revise.call( + const reservedBytes = agentJournalTurnRowReservedBytes(target, write) + return revise.call( sink, reservedBytes, - (journal) => { - const row = findTurnRow(journal, target) - const identity = known ?? (row ? parseAgentJournalItemKey(row.itemId) : null) - const body = reviseTurnBody(row?.body ?? null, withFallbackWindow(journal, write)) - if (!identity || !body) { - return null - } - if (estimateStructuredAgentSessionItemBytes(identity, body) <= reservedBytes) { - return { identity, body } - } - // Only a row grown by fields this build does not know gets here; the lifecycle still lands. - return write.lifecycle ? { identity, body: write.lifecycle } : null - }, + (journal) => resolveAgentJournalTurnRowWrite(journal, target, write, reservedBytes), options ) } diff --git a/src/main/codex/codex-turn-ordinals.test.ts b/src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.test.ts similarity index 64% rename from src/main/codex/codex-turn-ordinals.test.ts rename to src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.test.ts index 5ae12011a855..b070b449f264 100644 --- a/src/main/codex/codex-turn-ordinals.test.ts +++ b/src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.test.ts @@ -1,16 +1,17 @@ import { expect, it } from 'vitest' import { - CodexTurnOrdinals, - MAX_CODEX_TURN_ORDINAL_BYTES, - MAX_CODEX_TURN_ORDINAL_ENTRIES -} from './codex-turn-ordinals' + ProviderTurnMessageOrdinals, + MAX_PROVIDER_TURN_ORDINAL_BYTES, + MAX_PROVIDER_TURN_ORDINAL_ENTRIES +} from './provider-turn-message-ordinals' it('does not rescan the forgotten window for each new streamed item', () => { - const ordinals = new CodexTurnOrdinals() - for (let index = 0; index < MAX_CODEX_TURN_ORDINAL_ENTRIES; index += 1) { + const ordinals = new ProviderTurnMessageOrdinals() + for (let index = 0; index < MAX_PROVIDER_TURN_ORDINAL_ENTRIES; index += 1) { ordinals.ordinalFor('thread', String(index), 'item') ordinals.forgetTurn('thread', String(index)) } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this test counts reads of the class's private turn map, whose entries carry `active`. const turns = (ordinals as unknown as { turns: Map }).turns let reads = 0 for (const turn of turns.values()) { @@ -29,30 +30,30 @@ it('does not rescan the forgotten window for each new streamed item', () => { expect(ordinals.ordinalFor('thread', 'live', String(index))).toBe(index) } expect(reads).toBe(0) - expect(ordinals.forgottenTurnCount).toBe(MAX_CODEX_TURN_ORDINAL_ENTRIES) + expect(ordinals.forgottenTurnCount).toBe(MAX_PROVIDER_TURN_ORDINAL_ENTRIES) }) it('retains ordinal continuity on late reactivation and evicts oldest forgotten turns', () => { - const ordinals = new CodexTurnOrdinals() + const ordinals = new ProviderTurnMessageOrdinals() expect(ordinals.ordinalFor('t', 'first', 'a')).toBe(0) ordinals.forgetTurn('t', 'first') expect(ordinals.ordinalFor('t', 'first', 'b')).toBe(1) expect(ordinals.forgottenTurnCount).toBe(0) ordinals.forgetTurn('t', 'first') - for (let index = 0; index < MAX_CODEX_TURN_ORDINAL_ENTRIES; index += 1) { + for (let index = 0; index < MAX_PROVIDER_TURN_ORDINAL_ENTRIES; index += 1) { ordinals.ordinalFor('t', String(index), 'a') ordinals.forgetTurn('t', String(index)) } - expect(ordinals.forgottenTurnCount).toBe(MAX_CODEX_TURN_ORDINAL_ENTRIES) + expect(ordinals.forgottenTurnCount).toBe(MAX_PROVIDER_TURN_ORDINAL_ENTRIES) expect(ordinals.ordinalFor('t', 'first', 'c')).toBe(0) }) it('keeps byte eviction bounded for active and forgotten turns', () => { - const ordinals = new CodexTurnOrdinals() + const ordinals = new ProviderTurnMessageOrdinals() for (let index = 0; index < 4000; index += 1) { ordinals.ordinalFor('thread', 'live', `${index}-${'x'.repeat(240)}`) } - expect(ordinals.bytes).toBeLessThanOrEqual(MAX_CODEX_TURN_ORDINAL_BYTES) + expect(ordinals.bytes).toBeLessThanOrEqual(MAX_PROVIDER_TURN_ORDINAL_BYTES) ordinals.forgetTurn('thread', 'live') expect(ordinals.bytes).toBeLessThan(100) expect(ordinals.forgottenTurnCount).toBe(1) diff --git a/src/main/codex/codex-turn-ordinals.ts b/src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.ts similarity index 88% rename from src/main/codex/codex-turn-ordinals.ts rename to src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.ts index 7e21b900d965..9405ff9281ef 100644 --- a/src/main/codex/codex-turn-ordinals.ts +++ b/src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.ts @@ -1,18 +1,11 @@ -import { - boundPayload, - digestPayload -} from '../native-chat/agent-session-journal/journal-payload-bounds' - -/** Codex records the user message first in a turn, so a restored submission is - * ordinal 0 of `(threadId, turnId)`. */ -export const CODEX_USER_MESSAGE_ORDINAL = 0 +import { boundPayload, digestPayload } from '../agent-session-journal/journal-payload-bounds' /** Maximum forgotten turn keys retained for late-frame reconciliation. */ -export const MAX_CODEX_TURN_ORDINAL_ENTRIES = 256 -export const MAX_CODEX_TURN_ORDINAL_BYTES = 512 * 1024 +export const MAX_PROVIDER_TURN_ORDINAL_ENTRIES = 256 +export const MAX_PROVIDER_TURN_ORDINAL_BYTES = 512 * 1024 /** Assigns stable message ordinals while retaining a bounded late-frame window. */ -export class CodexTurnOrdinals { +export class ProviderTurnMessageOrdinals { private readonly turns = new Map< string, { assigned: Map; next: number; active: boolean } @@ -46,7 +39,7 @@ export class CodexTurnOrdinals { } private trimForgotten(): void { - while (this.forgottenTurnCount > MAX_CODEX_TURN_ORDINAL_ENTRIES) { + while (this.forgottenTurnCount > MAX_PROVIDER_TURN_ORDINAL_ENTRIES) { const oldest = this.forgottenTurns.values().next().value if (!oldest) { break @@ -67,7 +60,7 @@ export class CodexTurnOrdinals { private trimBytes(currentTurnKey: string): void { this.trimForgotten() - while (this.retainedBytes > MAX_CODEX_TURN_ORDINAL_BYTES) { + while (this.retainedBytes > MAX_PROVIDER_TURN_ORDINAL_BYTES) { const forgotten = this.forgottenTurns.values().next().value const oldest = forgotten ?? this.turns.keys().next().value if (typeof oldest !== 'string') { From 8a42c335c541df45eee0b42c7419d9d2b13680dc Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:52:04 -0700 Subject: [PATCH 02/11] Admit one provider event's writes as one transition, and let rows be found again after a restart - A sink transition is admitted whole or not at all; its steps run back to back at their turn in the journal's write queue, and each resolver reads the fold with every earlier write landed. A resolver may also say where the row belongs (turn scope, provider reference), and the writer always hears how the transition landed. A resolved lifecycle batch chooses its settlement mutations from the fold at execution. - New optional row field providerItemRef: the provider's own reference for the item a row is, written only where the row's identity cannot spell it (Codex keys messages by their place in the turn and renumbers its item ids on resume). Set by the creating write, kept by revisions, indexed by the journal fold, never read by clients. A downgrade test shows an older host and client render such rows unchanged. - Provider timeline identity schemes (shared legacy arm, Codex) and the join index that resolves a provider item to its row from memory or the fold: ordinals and request incarnations are read back from the rows, so a restart or an evicted entry finds the original row instead of placing a new one. --- .../codex/codex-provider-timeline-identity.ts | 43 +++ .../journal-item-appender.ts | 9 +- .../journal-item-fold.ts | 10 +- .../journal-lifecycle-batch-appender.ts | 30 +- .../journal-provider-item-ref.test.ts | 75 ++++ .../agent-session-journal/journal-reducer.ts | 3 + .../journal-render-item.ts | 1 + .../journal-row-builders.ts | 5 +- .../journal-row-schema.ts | 12 +- .../journal-row-writer.ts | 55 +-- .../journal-store-collaborators.ts | 32 +- .../journal-store-contracts.ts | 10 +- .../agent-session-journal/journal-store.ts | 11 + .../provider-timeline-identity.ts | 133 +++++++ .../provider-timeline-joins.test.ts | 176 ++++++++++ .../provider-timeline-joins.ts | 330 ++++++++++++++++++ .../agent-session-delta-coalescer.ts | 25 ++ ...ructured-agent-session-event-sink-queue.ts | 3 + .../structured-agent-session-event-sink.ts | 9 +- ...ed-agent-session-journal-append-options.ts | 1 + ...tructured-agent-session-transition.test.ts | 265 ++++++++++++++ .../structured-agent-session-transition.ts | 156 +++++++++ src/shared/agent-session-journal-types.ts | 9 + ...n-provider-item-ref-downgrade.unit.test.ts | 127 +++++++ 24 files changed, 1486 insertions(+), 44 deletions(-) create mode 100644 src/main/codex/codex-provider-timeline-identity.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-provider-item-ref.test.ts create mode 100644 src/main/native-chat/agent-session-timeline/provider-timeline-identity.ts create mode 100644 src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts create mode 100644 src/main/native-chat/agent-session-timeline/provider-timeline-joins.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts create mode 100644 tests/e2e/cross-version-wire/agent-session-provider-item-ref-downgrade.unit.test.ts diff --git a/src/main/codex/codex-provider-timeline-identity.ts b/src/main/codex/codex-provider-timeline-identity.ts new file mode 100644 index 000000000000..029c13fe56e2 --- /dev/null +++ b/src/main/codex/codex-provider-timeline-identity.ts @@ -0,0 +1,43 @@ +// Codex's persisted identities, spelled for the shared timeline assembler. +// +// The same rows Codex writes today (`codexItemIdentity`, `codexTurnLifecycleIdentity`): a user or +// assistant message inside a turn keys by (thread, turn, message ordinal), which survives a resume +// because the ordinal counts messages only; anything else keys by its Codex item id; a turn keys by +// its Codex turn id. Codex ids name one conversation for good, so the namespace spells nothing. + +import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key' +import type { ProviderTimelineIdentityScheme } from '../native-chat/agent-session-timeline/provider-timeline-identity' +import { + codexTurnLifecycleIdentity, + codexTurnUserItemId +} from './codex-structured-journal-translation-turns' + +export function createCodexProviderTimelineIdentityScheme(input: { + sessionId: string + primaryThreadId: () => string | null +}): ProviderTimelineIdentityScheme { + const turn: ProviderTimelineIdentityScheme['turn'] = (address) => + codexTurnLifecycleIdentity(input.sessionId, address.key.value) + return { + ordinalMessages: true, + turn, + turnId: (address) => address.key.value, + turnOpener: (address) => { + const thread = input.primaryThreadId() + return thread === null + ? agentJournalItemKey(turn(address)) + : codexTurnUserItemId(thread, address.key.value) + }, + item: (address) => { + const threadId = address.thread ?? input.primaryThreadId() ?? '' + return address.itemClass === 'message' && address.turn && address.messageOrdinal !== null + ? { + provider: 'codex', + threadId, + turnId: address.turn.value, + ordinal: address.messageOrdinal + } + : { provider: 'orca', clientMessageId: `codex-item:${threadId}:${address.key.value}` } + } + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-item-appender.ts b/src/main/native-chat/agent-session-journal/journal-item-appender.ts index 146078d830c9..479f0bc04a9d 100644 --- a/src/main/native-chat/agent-session-journal/journal-item-appender.ts +++ b/src/main/native-chat/agent-session-journal/journal-item-appender.ts @@ -10,7 +10,12 @@ import type { JournalRow } from './journal-row-schema' const NOTHING_RESOLVED = new Error('journal_item_resolved_to_nothing') -export type JournalResolvedItem = { identity: AgentJournalItemIdentity; body: AgentJournalItemBody } +export type JournalResolvedItem = { + identity: AgentJournalItemIdentity + body: AgentJournalItemBody + /** Replaces the call's options when only the fold could say where the row belongs. */ + options?: JournalItemAppendOptions +} export class JournalItemAppender { constructor( @@ -48,7 +53,7 @@ export class JournalItemAppender { this.deps.state, resolved.identity, resolved.body, - options + resolved.options ?? options )(seq, ts) }) .then( diff --git a/src/main/native-chat/agent-session-journal/journal-item-fold.ts b/src/main/native-chat/agent-session-journal/journal-item-fold.ts index 10aee4a99c41..aa1a02e4a70a 100644 --- a/src/main/native-chat/agent-session-journal/journal-item-fold.ts +++ b/src/main/native-chat/agent-session-journal/journal-item-fold.ts @@ -39,6 +39,9 @@ export function upsertJournalItem( return } state.derivedTurnScope.observe(itemId, isRootAgentJournalItem(next), existing?.body, next.body) + if (next.providerItemRef !== undefined && existing?.providerItemRef === undefined) { + state.providerItemRefs.set(next.providerItemRef, itemId) + } if (!existing) { state.items.set(itemId, next) state.itemFences.set(itemId, fence) @@ -65,7 +68,9 @@ export function upsertJournalItem( sequence: existing.sequence, ...(existing.sequenceIndex !== undefined ? { sequenceIndex: existing.sequenceIndex } : {}), observedAt: existing.observedAt, - turnScope: existing.turnScope ?? next.turnScope + turnScope: existing.turnScope ?? next.turnScope, + // The creating write's, as the scope is: a revision never re-points the row at another item. + ...(existing.providerItemRef === undefined ? {} : { providerItemRef: existing.providerItemRef }) }) state.tombstones.delete(itemId) } @@ -92,6 +97,9 @@ export function removeJournalItem( ) } state.tombstones.set(itemId, revision) + if (existing?.providerItemRef !== undefined) { + state.providerItemRefs.delete(existing.providerItemRef) + } state.items.delete(itemId) state.itemFences.delete(itemId) } diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts index c05d22a87457..682f1b3c1d5f 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts @@ -1,8 +1,13 @@ import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { JournalReducerState } from './journal-reducer' +import { partitionJournalLifecycleMutations } from './journal-lifecycle-batch-partition' import { journalLifecycleBatchRowBuilder } from './journal-row-builders' -import type { JournalLifecycleBatchInput } from './journal-store-contracts' +import type { + JournalLifecycleBatchInput, + JournalResolvedLifecycleBatchInput +} from './journal-store-contracts' import type { JournalRow } from './journal-row-schema' +import type { JournalRowWriter } from './journal-row-writer' const SETTLEMENT_ALREADY_APPLIED = new Error('journal_settlement_already_applied') @@ -12,6 +17,7 @@ export class JournalLifecycleBatchAppender { state: () => JournalReducerState cursor: () => AgentJournalCursor enqueue: (build: (seq: number, ts: number) => JournalRow) => Promise + enqueueEach: JournalRowWriter['enqueueEach'] } ) {} @@ -41,6 +47,28 @@ export class JournalLifecycleBatchAppender { }) } + /** Chooses the mutations at its turn in the queue; one too large for a row becomes consecutive + * rows that nothing else lands between, in the order resolved. */ + appendResolved(input: JournalResolvedLifecycleBatchInput): Promise { + return this.deps + .enqueueEach(() => + partitionJournalLifecycleMutations(input.settlementId, input.resolve()) + .filter((chunk) => !this.wasApplied(chunk.settlementId)) + .map((chunk) => + journalLifecycleBatchRowBuilder( + this.deps.state, + chunk.settlementId, + chunk.mutations, + input + ) + ) + ) + .then((rows) => { + const last = rows.at(-1) + return last ? { epoch: last.epoch, sequence: last.seq } : null + }) + } + private wasApplied(settlementId: string): boolean { return this.deps.state().appliedSettlementIds.has(settlementId) } diff --git a/src/main/native-chat/agent-session-journal/journal-provider-item-ref.test.ts b/src/main/native-chat/agent-session-journal/journal-provider-item-ref.test.ts new file mode 100644 index 000000000000..d80d5a539965 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-provider-item-ref.test.ts @@ -0,0 +1,75 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import type { AgentJournalItemIdentity } from '../../../shared/agent-session-journal-types' +import { createTrackedJournalOpener } from './journal-host-database-test-support' + +const journals = createTrackedJournalOpener() +const roots: string[] = [] + +afterEach(async () => { + await journals.closeAll() + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }) + } +}) + +async function open(root?: string) { + const directory = root ?? (await mkdtemp(join(tmpdir(), 'orca-provider-item-ref-'))) + if (!root) { + roots.push(directory) + } + const journal = await journals.open({ + identity: { + sessionId: 'session-ref', + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: 'root' } + }, + stateDirectory: directory, + now: () => 1_000 + }) + return { journal, directory } +} + +const row: AgentJournalItemIdentity = { + provider: 'codex', + threadId: 'root', + turnId: 't1', + ordinal: 0 +} +const text = (value: string) => ({ + kind: 'message' as const, + role: 'assistant' as const, + blocks: [{ type: 'text' as const, text: value }] +}) +const scope = { kind: 'turn', turnItemId: 'turn:t1' } as const + +describe('a row’s provider item reference', () => { + it('is the creating write’s, kept by revisions and indexed for its producer', async () => { + const { journal } = await open() + await journal.appendItem(row, text('a'), { fence: 1, turnScope: scope, providerItemRef: 'm0' }) + await journal.appendItem(row, text('ab'), { fence: 1, turnScope: scope, providerItemRef: 'm9' }) + await journal.appendItem(row, text('abc'), { fence: 1, turnScope: scope }) + + expect(journal.item('codex:root:t1:0')).toMatchObject({ + providerItemRef: 'm0', + body: text('abc') + }) + expect(journal.itemIdForProviderItemRef('m0')).toBe('codex:root:t1:0') + expect(journal.itemIdForProviderItemRef('m9')).toBeNull() + }) + + it('survives a reopen, and leaves the index with its row', async () => { + const { journal, directory } = await open() + await journal.appendItem(row, text('a'), { fence: 1, turnScope: scope, providerItemRef: 'm0' }) + await journals.closeAll() + + const { journal: reopened } = await open(directory) + expect(reopened.itemIdForProviderItemRef('m0')).toBe('codex:root:t1:0') + await reopened.appendTombstone(row, { fence: 1 }) + expect(reopened.itemIdForProviderItemRef('m0')).toBeNull() + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index a7d6f150e70c..d26b52b144c4 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -56,6 +56,8 @@ export type JournalReducerState = { /** Provider item id → the submission slot that adopted it. Stops an accepted * echo from appending a second copy of the user's own message. */ aliases: Map + /** `providerItemRef` → the row that carries it, so a producer finds that row again in O(1). */ + providerItemRefs: Map appliedSettlementIds: Set /** Scope for rows stored without one; rebuilt by replay, never persisted. */ derivedTurnScope: JournalDerivedTurnScope @@ -80,6 +82,7 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou submissions: new Map(), receipts: new Map(), aliases: new Map(), + providerItemRefs: new Map(), appliedSettlementIds: new Set(), derivedTurnScope: new JournalDerivedTurnScope(), latestPersonTurnSequence: 0, diff --git a/src/main/native-chat/agent-session-journal/journal-render-item.ts b/src/main/native-chat/agent-session-journal/journal-render-item.ts index fd58b2c25e4e..76a7abf054ec 100644 --- a/src/main/native-chat/agent-session-journal/journal-render-item.ts +++ b/src/main/native-chat/agent-session-journal/journal-render-item.ts @@ -35,6 +35,7 @@ export function journalRenderItem( ...(row.recovered ? { recoveredAt: row.ts } : {}), ...(row.recovered ? { recovered: row.recovered } : {}), turnScope, + ...(row.providerItemRef === undefined ? {} : { providerItemRef: row.providerItemRef }), ...agentJournalLinkageFields(producer) } } diff --git a/src/main/native-chat/agent-session-journal/journal-row-builders.ts b/src/main/native-chat/agent-session-journal/journal-row-builders.ts index 0865df3a41fa..16bf1960e400 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-builders.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-builders.ts @@ -49,7 +49,8 @@ export function journalItemRowBuilder( ts: options.observedAt ?? ts, recovered: options.recovered, linkage: options, - turnScope: options.turnScope + turnScope: options.turnScope, + ...(options.providerItemRef === undefined ? {} : { providerItemRef: options.providerItemRef }) }) } @@ -249,6 +250,7 @@ export function buildJournalItemRow(input: { recovered?: true linkage?: AgentJournalProducerLinkage turnScope: AgentJournalTurnScope + providerItemRef?: string }): JournalItemRow { const itemId = agentJournalItemKey(input.identity) const resolved = input.state.aliases.get(itemId) ?? itemId @@ -268,6 +270,7 @@ export function buildJournalItemRow(input: { ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts, [body]), ...(input.recovered ? { recovered: input.recovered } : {}), turnScope: input.turnScope, + ...(input.providerItemRef === undefined ? {} : { providerItemRef: input.providerItemRef }), ...agentJournalLinkageFields(input.linkage) } } diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.ts index 9276b1a8c05f..e8d1746ceb8d 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.ts @@ -20,6 +20,7 @@ import { type AgentJournalItemBody, type AgentJournalMessageItem, type AgentJournalProducerLinkage, + type AgentJournalProviderItemRef, type AgentJournalTurnScope, type AgentSessionProviderHandle } from '../../../shared/agent-session-journal-types' @@ -51,6 +52,9 @@ type JournalRowBase = AgentJournalProducerLinkage & { * for the reason linkage does. Absent on rows from hosts that predate it: the reducer * derives one for them on read. */ turnScope?: AgentJournalTurnScope + /** On an item row: the provider's reference for the item it creates. Rides the base for the + * reason linkage does. */ + providerItemRef?: AgentJournalProviderItemRef } /** First row of every epoch: binds the epoch to a provider handle and records why it opened. */ @@ -249,7 +253,13 @@ export function parseJournalRow(line: string): JournalRowParse { * survives is a real one: the reader scopes on PRESENCE, so `''` or a * non-string left in place would hide the row from its own author for good. */ function dropUnusableProducerLinkage(record: Record): void { - for (const field of ['agentId', 'parentAgentId', 'providerParentRef', 'producerKind']) { + for (const field of [ + 'agentId', + 'parentAgentId', + 'providerParentRef', + 'producerKind', + 'providerItemRef' + ]) { const value = record[field] if (value !== undefined && (typeof value !== 'string' || value.length === 0)) { delete record[field] diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.ts index b07fda9459bc..17a7f551370d 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.ts @@ -37,27 +37,40 @@ export class JournalRowWriter { build: (seq: number, ts: number) => JournalRow, hook?: JournalRowTransactionHook ): Promise { - return this.deps.serialize(() => { - assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) - const row = build(this.deps.nextSequence(), this.deps.now()) - assertJournalFence(row.fence, this.deps.highestFence()) - try { - // One INSERT: the chat's epoch pointer moves only when the epoch does. - this.deps.database().transaction((db) => { - insertJournalRow(db, this.deps.sessionId, row) - hook?.(db, row) - this.runBookkeeping(db, row) - }) - } catch (error) { - this.deps.rolledBack?.() - throw error - } - // COMMIT landed, so the row is durable: adopt it before anything that can - // fail. Rejecting here instead would leave the next append reusing a - // sequence the table already holds. - this.deps.commit(row) - return row - }) + return this.deps.serialize(() => this.write(build, hook)) + } + + /** Rows chosen at this write's turn in the queue and written back to back, so no other write + * lands between them; each is built after the one before it is adopted. */ + enqueueEach( + resolve: () => readonly ((seq: number, ts: number) => JournalRow)[] + ): Promise { + return this.deps.serialize(() => resolve().map((build) => this.write(build))) + } + + private write( + build: (seq: number, ts: number) => JournalRow, + hook?: JournalRowTransactionHook + ): JournalRow { + assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) + const row = build(this.deps.nextSequence(), this.deps.now()) + assertJournalFence(row.fence, this.deps.highestFence()) + try { + // One INSERT: the chat's epoch pointer moves only when the epoch does. + this.deps.database().transaction((db) => { + insertJournalRow(db, this.deps.sessionId, row) + hook?.(db, row) + this.runBookkeeping(db, row) + }) + } catch (error) { + this.deps.rolledBack?.() + throw error + } + // COMMIT landed, so the row is durable: adopt it before anything that can + // fail. Rejecting here instead would leave the next append reusing a + // sequence the table already holds. + this.deps.commit(row) + return row } /** Assign the next sequence, make the row durable, and fold it through the SAME reducer diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 65ea665879b5..c00de57f6a66 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -92,6 +92,20 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal wroteBeforeOpen: (sequence) => host.journal().wroteBeforeOpen(sequence), committed: host.notifyCommitted }) + const rowWriter = new JournalRowWriter({ + sessionId: host.identity.sessionId, + now: host.now, + serialize: host.serialize, + database: host.database, + readOnly: host.readOnly, + highestFence: () => host.state().highestFence, + nextSequence: () => host.state().lastSequence + 1, + commit: host.commit, + // Every rejection is a dispatch row through this one writer; the draft + // returned-transition rides it so no path can bypass the hook. + inTransaction: (db, row) => queuedMessages.onRowInTransaction(db, row), + rolledBack: () => queuedMessages.invalidate() + }) return { epochController, queuedMessages, @@ -102,20 +116,7 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal restoreJournalStore(host, { epochController }).then(() => queuedMessages.repairAndPruneAtOpen() ), - rowWriter: new JournalRowWriter({ - sessionId: host.identity.sessionId, - now: host.now, - serialize: host.serialize, - database: host.database, - readOnly: host.readOnly, - highestFence: () => host.state().highestFence, - nextSequence: () => host.state().lastSequence + 1, - commit: host.commit, - // Every rejection is a dispatch row through this one writer; the draft - // returned-transition rides it so no path can bypass the hook. - inTransaction: (db, row) => queuedMessages.onRowInTransaction(db, row), - rolledBack: () => queuedMessages.invalidate() - }), + rowWriter, itemAppender: new JournalItemAppender({ state: host.state, enqueue: host.enqueue @@ -123,7 +124,8 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal lifecycleBatchAppender: new JournalLifecycleBatchAppender({ state: host.state, cursor: host.cursor, - enqueue: host.enqueue + enqueue: host.enqueue, + enqueueEach: (resolve) => rowWriter.enqueueEach(resolve) }) } } diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index 85e6b99f9029..173e0c062e97 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -62,7 +62,10 @@ export type JournalItemLinkageVisitor = ( itemId: string, sequence: number, body: AgentJournalItemBody, - attribution: AgentJournalProducerLinkage & { turnScope?: AgentJournalTurnScope } + attribution: AgentJournalProducerLinkage & { + turnScope?: AgentJournalTurnScope + providerItemRef?: string + } ) => void export type JournalLifecycleBatchInput = { @@ -72,6 +75,11 @@ export type JournalLifecycleBatchInput = { recovered?: true } +export type JournalResolvedLifecycleBatchInput = Omit & { + /** Read from the fold with every earlier write landed; may return none. */ + resolve: () => readonly JournalLifecycleMutationInput[] +} + export type JournalSubmissionInput = { clientMessageId: string payloadFingerprint: string diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 5a4880155dec..53929d836e77 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -8,6 +8,7 @@ import type { AgentJournalCursor, AgentJournalItemBody, AgentJournalItemIdentity, + AgentJournalRenderItem, AgentJournalSnapshot, AgentJournalSubmission, AgentJournalThreadGoal, @@ -227,6 +228,11 @@ export class AgentSessionJournal { itemBody = (itemId: string): AgentJournalItemBody | null => this.state.items.get(itemId)?.body ?? null + /** One reduced item with its attribution, for a writer that needs the turn a row joined. */ + item = (itemId: string): AgentJournalRenderItem | null => this.state.items.get(itemId) ?? null + /** The row that carries this `providerItemRef`, if any. */ + itemIdForProviderItemRef = (ref: string) => this.state.providerItemRefs.get(ref) ?? null + /** Visits reduced items with the producer that wrote each, for a producer re-deriving what an * earlier run of this session left. */ visitItemsWithLinkage = (visit: JournalItemLinkageVisitor): void => { @@ -331,6 +337,11 @@ export class AgentSessionJournal { return this.lifecycleBatchAppender.append(input) } + /** A lifecycle batch whose mutations are chosen from the fold at its own turn in the queue; + * null when none are. */ + appendResolvedLifecycleBatch: JournalLifecycleBatchAppender['appendResolved'] = (input) => + this.lifecycleBatchAppender.appendResolved(input) + /** * Write-ahead submission row. It is durable before the caller dispatches * anything, and it doubles as the optimistic user bubble so an accepted echo diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-identity.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-identity.ts new file mode 100644 index 000000000000..d364b4d31acf --- /dev/null +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-identity.ts @@ -0,0 +1,133 @@ +// How the assembler's addresses become persisted journal identities. +// +// The assembler decides WHICH rows are the same row: it resolves every provider join to an +// address — the provider session's namespace, the key, the thread and turn the item belongs to, +// the item's class, and its place among its turn's messages — or mints a key when the provider +// names nothing. A scheme only spells an address as an `AgentJournalItemIdentity`. It must be pure, +// and it is the one place a provider's persisted identity shape lives, so moving a provider to a +// new identity arm is a change here. + +import type { + AgentJournalItemBody, + AgentJournalItemIdentity, + AgentType +} from '../../../shared/agent-session-journal-types' +import { boundPayload, digestPayload } from '../agent-session-journal/journal-payload-bounds' + +/** A key the provider vouched for, or one the assembler minted because it named nothing. + * A minted value is unique per acquisition, so it never needs a namespace. */ +export type ProviderTimelineKey = { source: 'provider' | 'minted'; value: string } + +export type ProviderTimelineTurnAddress = { + /** The provider session whose ids the key belongs to. */ + namespace: string + key: ProviderTimelineKey +} + +/** Which family an item belongs to; families never share a key space. Streamed text and full + * item snapshots are one family, so a provider-named message is one row however it arrives. */ +export type ProviderTimelineItemFamily = 'item' | 'request' | 'frame' + +/** What kind of thing an item is, derived from what it carries: `message` is user or assistant + * prose; every other class is its row kind (`reasoning` text included). */ +export type ProviderTimelineItemClass = + | 'message' + | 'reasoning' + | Exclude + | 'frame' + +export type ProviderTimelineItemAddress = { + namespace: string + family: ProviderTimelineItemFamily + key: ProviderTimelineKey + /** The provider thread it came from; null when the provider named none. */ + thread: string | null + /** The provider turn it belongs to; null for a row outside any turn. */ + turn: ProviderTimelineKey | null + itemClass: ProviderTimelineItemClass + /** Its place among its (thread, turn)'s `message` items, by first write; null for any other, and + * for every item of a scheme without `ordinalMessages`. */ + messageOrdinal: number | null + /** 1 for the first request under a key; a key reused after it settled is the next one. */ + incarnation: number +} + +export type ProviderTimelineIdentityScheme = { + /** Messages inside a turn are keyed by their place among the turn's messages, not by their key. + * Such an identity must spell only (thread, turn, ordinal), and its row carries the key as + * `providerItemRef`, since nothing else could find it again. */ + ordinalMessages: boolean + turn(address: ProviderTimelineTurnAddress): AgentJournalItemIdentity + /** The id the turn row carries and a client's Stop names. */ + turnId(address: ProviderTimelineTurnAddress): string + /** The user item a turn names when no send of Orca's opened it; absent ⇒ the turn row itself. */ + turnOpener?(address: ProviderTimelineTurnAddress): string + /** With a null `messageOrdinal` it must spell only (namespace, family, key, thread, incarnation), + * so a row is found again from its key alone. */ + item(address: ProviderTimelineItemAddress): AgentJournalItemIdentity +} + +export function providerTimelineItemClass(body: AgentJournalItemBody): ProviderTimelineItemClass { + if (body.kind !== 'message') { + return body.kind + } + return body.role === 'user' || body.role === 'assistant' ? 'message' : 'reasoning' +} + +const MAX_KEY_PART_BYTES = 256 + +/** A provider id as a bounded identity part: escaped so `:` cannot forge another part, and + * digest-suffixed when long so two long ids never share a prefix-only spelling. */ +export function providerTimelineKeyPart(value: string): string { + const encoded = encodeURIComponent(value) + if (Buffer.byteLength(encoded, 'utf8') <= MAX_KEY_PART_BYTES) { + return encoded + } + const suffix = `#${digestPayload(value).slice(0, 24)}` + const head = boundPayload(encoded, { + inlineHeadBytes: MAX_KEY_PART_BYTES - Buffer.byteLength(suffix, 'utf8') + }).head + return `${head}${suffix}` +} + +/** A key as one bounded string inside its namespace (and thread, for a per-thread key). */ +export function spellProviderTimelineKey( + namespace: string, + key: ProviderTimelineKey, + thread: string | null = null +): string { + // A provider's item ids are its own per thread, so a subagent thread's ids are spelled apart. + const scope = thread === null ? '' : `${providerTimelineKeyPart(thread)}/` + return key.source === 'provider' + ? `p:${providerTimelineKeyPart(namespace)}:${scope}${providerTimelineKeyPart(key.value)}` + : `m:${providerTimelineKeyPart(key.value)}` +} + +/** + * The scheme for a provider with no identity arm of its own: the existing `legacy` arm, so no row + * shape changes. Turn rows keep the `turn-lifecycle:` record prefix the other lanes write. + * Provider keys are spelled inside their namespace, and apart from minted ones. + */ +export function createLegacyProviderTimelineIdentityScheme(input: { + agent: AgentType + sessionId: string +}): ProviderTimelineIdentityScheme { + const identity = (recordId: string): AgentJournalItemIdentity => ({ + provider: 'legacy', + agent: input.agent, + sessionId: input.sessionId, + recordId + }) + return { + ordinalMessages: false, + turn: (address) => + identity(`turn-lifecycle:${spellProviderTimelineKey(address.namespace, address.key)}`), + turnId: (address) => spellProviderTimelineKey(address.namespace, address.key), + item: (address) => + identity( + `${address.family}:${spellProviderTimelineKey(address.namespace, address.key, address.family === 'request' ? null : address.thread)}${ + address.incarnation > 1 ? `#${address.incarnation}` : '' + }` + ) + } +} diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts new file mode 100644 index 000000000000..b132802369f0 --- /dev/null +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts @@ -0,0 +1,176 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalApprovalItem, + type AgentJournalItemIdentity +} from '../../../shared/agent-session-journal-types' +import { createCodexProviderTimelineIdentityScheme } from '../../codex/codex-provider-timeline-identity' +import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { createLegacyProviderTimelineIdentityScheme } from './provider-timeline-identity' +import { ProviderTimelineJoins, type ProviderTimelineItemJoin } from './provider-timeline-joins' + +const journals = createTrackedJournalOpener() +const roots: string[] = [] + +afterEach(async () => { + await journals.closeAll() + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }) + } +}) + +async function openJournal(): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-provider-timeline-joins-')) + roots.push(root) + return journals.open({ + identity: { + sessionId: 'session-joins', + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: 'root' } + }, + stateDirectory: root, + now: () => 1_000 + }) +} + +const codexJoins = () => + new ProviderTimelineJoins({ + scheme: createCodexProviderTimelineIdentityScheme({ + sessionId: 'session-joins', + primaryThreadId: () => 'root' + }), + generation: 'gen-1', + namespace: 'ns' + }) + +const message = (id: string): ProviderTimelineItemJoin => ({ + family: 'item', + key: { source: 'provider', value: id }, + thread: 'root' +}) + +const inTurn = (turn: string) => ({ + thread: 'root', + turn: { source: 'provider', value: turn } as const, + scope: { kind: 'turn', turnItemId: `turn:${turn}` } as const +}) + +async function write( + journal: AgentSessionJournal, + row: { identity: AgentJournalItemIdentity; ref?: string }, + text: string +) { + await journal.appendItem( + row.identity, + { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text }] }, + { + fence: 1, + turnScope: { kind: 'turn', turnItemId: 'turn:t1' }, + ...(row.ref === undefined ? {} : { providerItemRef: row.ref }) + } + ) +} + +describe('provider timeline joins', () => { + it('finds an ordinal message again from the reference its row carries, after a restart', async () => { + const journal = await openJournal() + const before = codexJoins() + const m0 = before.place(message('m0'), 'message', inTurn('t1'), journal) + await write(journal, m0, 'first') + const m1 = before.place(message('m1'), 'message', inTurn('t1'), journal) + await write(journal, m1, 'second') + expect([m0.itemId, m1.itemId]).toEqual(['codex:root:t1:0', 'codex:root:t1:1']) + + const after = codexJoins() + expect(after.find(message('m1'), journal)).toMatchObject({ + itemId: 'codex:root:t1:1', + scope: { kind: 'turn', turnItemId: 'turn:t1' } + }) + // A new message takes the next free place, past every row the turn holds. + expect(after.place(message('m2'), 'message', inTurn('t1'), journal).itemId).toBe( + 'codex:root:t1:2' + ) + }) + + it('skips a place an echoed send holds through its submission alias', async () => { + const journal = await openJournal() + await journal.appendSubmission({ + clientMessageId: 'send-1', + payloadFingerprint: 'fp', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }, + fence: 1 + }) + await journal.resolveDispatch({ + clientMessageId: 'send-1', + state: 'accepted', + providerIdentity: { provider: 'codex', threadId: 'root', turnId: 't1', ordinal: 0 }, + fence: 1 + }) + expect(codexJoins().place(message('reply'), 'message', inTurn('t1'), journal).itemId).toBe( + 'codex:root:t1:1' + ) + }) + + it('finds a row whose identity spells its key without any reference', async () => { + const journal = await openJournal() + const scheme = createLegacyProviderTimelineIdentityScheme({ + agent: 'grok', + sessionId: 'session-joins' + }) + const joins = () => new ProviderTimelineJoins({ scheme, generation: 'g', namespace: 'ns' }) + const tool = joins().place(message('tool'), 'tool-call', inTurn('t1'), journal) + expect(tool.ref).toBeUndefined() + await write(journal, tool, 'x') + expect(joins().find(message('tool'), journal)?.itemId).toBe(tool.itemId) + expect(joins().find({ ...message('tool'), thread: 'other' }, journal)).toBeNull() + }) + + it('reads the current request incarnation back from the rows', async () => { + const journal = await openJournal() + const scheme = createLegacyProviderTimelineIdentityScheme({ + agent: 'grok', + sessionId: 'session-joins' + }) + const joins = () => new ProviderTimelineJoins({ scheme, generation: 'g', namespace: 'ns' }) + const key = { source: 'provider', value: 'p1' } as const + const thread = { thread: null, turn: null, scope: AGENT_JOURNAL_THREAD_SCOPE } + const approval: AgentJournalApprovalItem = { + kind: 'approval', + title: 'Run?', + detail: null, + options: [], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + } + const before = joins() + for (let incarnation = 1; incarnation <= 2; incarnation += 1) { + const row = before.nextRequest(key, 'approval', thread, journal) + expect(row.incarnation).toBe(incarnation) + await journal.appendItem(row.identity, approval, { + fence: 1, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) + } + expect(joins().request(key, journal)).toMatchObject({ + incarnation: 2, + itemId: agentJournalItemKey( + scheme.item({ + namespace: 'ns', + family: 'request', + key, + thread: null, + turn: null, + itemClass: 'approval', + messageOrdinal: null, + incarnation: 2 + }) + ) + }) + }) +}) diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-joins.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-joins.ts new file mode 100644 index 000000000000..1b85f32a0aa1 --- /dev/null +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-joins.ts @@ -0,0 +1,330 @@ +// Provider joins → journal rows: the assembler's canonical join index. +// +// Every row the assembler writes is found again through here, and every entry is a cache of a +// fact the journal holds. A row whose identity spells its provider key is found by recomputing +// that identity; a row whose identity does not (a message keyed by its place among its turn's +// messages) carries the key as `providerItemRef`, which the journal indexes. So a miss — after a +// restart or an eviction — recovers the original row; it never re-places it. Message ordinals and +// request incarnations are read back from the rows that hold them. Only transition resolvers call +// the allocating members, so allocation happens in journal order and a refused event allocates +// nothing. + +import { + agentJournalItemKey, + parseAgentJournalItemKey +} from '../../../shared/agent-session-journal-item-key' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemIdentity, + type AgentJournalRenderItem, + type AgentJournalTurnScope +} from '../../../shared/agent-session-journal-types' +import { BoundedMap } from '../../../shared/bounded-map' +import type { StructuredAgentSessionTransitionJournal } from '../agent-session-wire/structured-agent-session-transition' +import { + providerTimelineKeyPart, + spellProviderTimelineKey, + type ProviderTimelineIdentityScheme, + type ProviderTimelineItemAddress, + type ProviderTimelineItemClass, + type ProviderTimelineItemFamily, + type ProviderTimelineKey, + type ProviderTimelineTurnAddress +} from './provider-timeline-identity' + +const MAX_JOINED_ROWS = 1_024 +const MAX_JOINED_ROW_BYTES = 1024 * 1024 +const MAX_TURN_ORDINALS = 256 +/** Echoed sends whose slot no row holds yet; a lane records each on its send soon after. */ +const MAX_RESERVED_ECHOES = 64 + +/** One provider item as the provider names it. Item keys are the provider's per thread. */ +export type ProviderTimelineItemJoin = { + family: ProviderTimelineItemFamily + key: ProviderTimelineKey + thread: string | null +} + +export type ProviderTimelineRow = { + identity: AgentJournalItemIdentity + itemId: string + scope: AgentJournalTurnScope + /** The join's reference, on a row whose identity does not spell its key. */ + ref?: string + /** Which request under its key, 1 for the first. */ + incarnation: number +} + +export type ProviderTimelineTurnRef = { + address: ProviderTimelineTurnAddress + identity: AgentJournalItemIdentity + itemId: string + turnId: string +} + +/** Where a new row goes: the provider thread and turn, and the journal turn scope. */ +export type ProviderTimelinePlacement = { + thread: string | null + turn: ProviderTimelineKey | null + scope: AgentJournalTurnScope +} + +type Journal = StructuredAgentSessionTransitionJournal | null + +function bytes(value: string): number { + return Buffer.byteLength(value, 'utf8') +} + +export class ProviderTimelineJoins { + private readonly rows = new BoundedMap({ + maxEntries: MAX_JOINED_ROWS, + maxBytes: MAX_JOINED_ROW_BYTES, + sizeOf: (row, key) => bytes(key) + 2 * bytes(row.itemId) + 64 + }) + private readonly ordinals = new BoundedMap({ + maxEntries: MAX_TURN_ORDINALS, + maxBytes: MAX_TURN_ORDINALS * 1024, + sizeOf: (_next, place) => bytes(place) + 8 + }) + private readonly echoes = new BoundedMap({ maxEntries: MAX_RESERVED_ECHOES }) + private serial = 0 + + constructor( + private readonly deps: { + scheme: ProviderTimelineIdentityScheme + generation: string + namespace: string + } + ) {} + + get namespace(): string { + return this.deps.namespace + } + + /** A new provider session: its ids name new rows. Minted keys stay unique as they are. */ + reset(namespace: string): void { + this.deps.namespace = namespace + this.rows.clear() + this.ordinals.clear() + } + + /** A key unique to this acquisition, for something the provider names nothing. */ + mint(kind: string): ProviderTimelineKey { + this.serial += 1 + return { source: 'minted', value: `${this.deps.generation}:${kind}${this.serial}` } + } + + /** A turn's row. `namespace` defaults to the current one; a planner names the one it expects. */ + turn(key: ProviderTimelineKey, namespace = this.deps.namespace): ProviderTimelineTurnRef { + const address = { namespace, key } + const identity = this.deps.scheme.turn(address) + return { + address, + identity, + itemId: agentJournalItemKey(identity), + turnId: this.deps.scheme.turnId(address) + } + } + + /** The user item a turn names when no send of Orca's opened it. */ + turnOpener(turn: ProviderTimelineTurnRef): string { + return this.deps.scheme.turnOpener?.(turn.address) ?? turn.itemId + } + + /** The join as one bounded string: what rows carry as `providerItemRef` and caches key by. */ + reference(join: ProviderTimelineItemJoin, namespace = this.deps.namespace): string { + const thread = join.family === 'request' ? null : join.thread + return `${join.family}:${spellProviderTimelineKey(namespace, join.key, thread)}` + } + + /** The row the join already names, from memory or the fold; null when the journal holds none. */ + find( + join: ProviderTimelineItemJoin, + journal: Journal, + namespace = this.deps.namespace + ): ProviderTimelineRow | null { + const ref = this.reference(join, namespace) + const cached = this.rows.get(ref) + // A planner ahead of a reset asks about a namespace whose rows this index does not spell yet. + if (cached || !journal || namespace !== this.deps.namespace) { + return cached ?? null + } + const itemId = + journal.itemIdForProviderItemRef(ref) ?? agentJournalItemKey(this.spelled(join, 1)) + const row = rowOf(journal.item(itemId), 1) + if (row) { + this.rows.set(ref, row) + } + return row + } + + /** A new row for a join no row names yet. Allocates its message ordinal: resolvers only. */ + place( + join: ProviderTimelineItemJoin, + itemClass: ProviderTimelineItemClass, + placement: ProviderTimelinePlacement, + journal: Journal + ): ProviderTimelineRow { + const ordinal = this.ordinalFor(itemClass, placement, journal) + const identity = this.deps.scheme.item({ + ...this.address(join, 1), + turn: placement.turn, + itemClass, + messageOrdinal: ordinal + }) + const ref = this.reference(join) + const row: ProviderTimelineRow = { + identity, + itemId: agentJournalItemKey(identity), + scope: placement.scope, + ...(ordinal === null ? {} : { ref }), + incarnation: 1 + } + this.rows.set(ref, row) + return row + } + + /** An echoed send takes its message's place without a row (its send is the bubble). */ + reserveEcho( + join: ProviderTimelineItemJoin, + placement: ProviderTimelinePlacement, + journal: Journal + ) { + if (!this.deps.scheme.ordinalMessages || !placement.turn || this.find(join, journal)) { + return + } + const row = this.place(join, 'message', placement, journal) + this.echoes.set(row.itemId, true) + } + + /** The request under `key` the journal holds last: the highest incarnation with a row. */ + request( + key: ProviderTimelineKey, + journal: Journal, + namespace = this.deps.namespace + ): ProviderTimelineRow | null { + const join = { family: 'request' as const, key, thread: null } + const ref = this.reference(join, namespace) + const cached = this.rows.get(ref) + if (cached || !journal || namespace !== this.deps.namespace) { + return cached ?? null + } + let current: ProviderTimelineRow | null = null + for (let incarnation = 1; ; incarnation += 1) { + const itemId = agentJournalItemKey(this.spelled(join, incarnation)) + // A scheme that spells no incarnation names one row for every request under the key. + const row = itemId === current?.itemId ? null : rowOf(journal.item(itemId), incarnation) + if (!row) { + break + } + current = row + } + if (current) { + this.rows.set(ref, current) + } + return current + } + + /** The next request under `key`: resolvers only. */ + nextRequest( + key: ProviderTimelineKey, + itemClass: ProviderTimelineItemClass, + placement: ProviderTimelinePlacement, + journal: Journal + ): ProviderTimelineRow { + const join = { family: 'request' as const, key, thread: null } + const incarnation = (this.request(key, journal)?.incarnation ?? 0) + 1 + const identity = this.deps.scheme.item({ + ...this.address(join, incarnation), + thread: placement.thread, + turn: placement.turn, + itemClass + }) + const row = { + identity, + itemId: agentJournalItemKey(identity), + scope: placement.scope, + incarnation + } + this.rows.set(this.reference(join), row) + return row + } + + private address( + join: ProviderTimelineItemJoin, + incarnation: number + ): ProviderTimelineItemAddress { + return { + namespace: this.deps.namespace, + family: join.family, + key: join.key, + thread: join.thread, + turn: null, + itemClass: 'status', + messageOrdinal: null, + incarnation + } + } + + private spelled(join: ProviderTimelineItemJoin, incarnation: number): AgentJournalItemIdentity { + return this.deps.scheme.item(this.address(join, incarnation)) + } + + /** The next free place among the turn's messages: past every row and echoed send holding one. */ + private ordinalFor( + itemClass: ProviderTimelineItemClass, + placement: ProviderTimelinePlacement, + journal: Journal + ): number | null { + const turn = placement.turn + if (!this.deps.scheme.ordinalMessages || itemClass !== 'message' || !turn) { + return null + } + const place = `${providerTimelineKeyPart(placement.thread ?? '')}\u001f${spellProviderTimelineKey( + this.deps.namespace, + turn + )}` + let ordinal = this.ordinals.get(place) ?? 0 + while (this.taken(this.slot(placement.thread, turn, ordinal), journal)) { + ordinal += 1 + } + this.ordinals.set(place, ordinal + 1) + return ordinal + } + + private slot(thread: string | null, turn: ProviderTimelineKey, ordinal: number): string { + return agentJournalItemKey( + this.deps.scheme.item({ + ...this.address({ family: 'item', key: turn, thread }, 1), + turn, + itemClass: 'message', + messageOrdinal: ordinal + }) + ) + } + + private taken(itemId: string, journal: Journal): boolean { + return ( + this.echoes.has(itemId) || + (journal !== null && + (journal.itemBody(itemId) !== null || journal.canonicalItemId(itemId) !== itemId)) + ) + } +} + +function rowOf( + item: AgentJournalRenderItem | null, + incarnation: number +): ProviderTimelineRow | null { + const identity = item ? parseAgentJournalItemKey(item.itemId) : null + if (!item || !identity) { + return null + } + return { + identity, + itemId: item.itemId, + scope: item.turnScope ?? AGENT_JOURNAL_THREAD_SCOPE, + ...(item.providerItemRef === undefined ? {} : { ref: item.providerItemRef }), + incarnation + } +} diff --git a/src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts b/src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts index b1d4c483d9bd..eff850ae8e05 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts @@ -56,6 +56,10 @@ export type AgentSessionDeltaCoalescer = { dispose: () => void /** Bounded last-known state for terminalizing a rejected completion. */ snapshot: (key: string) => AgentSessionDeltaSnapshot | null + /** Streams with text not yet emitted, in arrival order, for a caller that writes them itself. */ + dirty: () => { key: string; snapshot: AgentSessionDeltaSnapshot }[] + /** The caller wrote this stream's current text itself; nothing is owed until it grows. */ + markFlushed: (key: string) => void } function defaultSchedule(run: () => void, ms: number): () => void { @@ -215,6 +219,27 @@ export function createAgentSessionDeltaCoalescer( truncated: stream.truncated } : null + }, + dirty: () => + [...streams].flatMap(([key, stream]) => + stream.dirty + ? [ + { + key, + snapshot: { + text: stream.chunks.join(''), + observedBytes: stream.observedBytes, + truncated: stream.truncated + } + } + ] + : [] + ), + markFlushed: (key) => { + const stream = streams.get(key) + if (stream) { + stream.dirty = false + } } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink-queue.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink-queue.ts index bdc07a3b3d54..bfa4b7520078 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink-queue.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink-queue.ts @@ -8,6 +8,7 @@ import type { StructuredAgentSessionSinkState, StructuredAgentSessionSinkWatermarks } from './structured-agent-session-event-sink' +import type { StructuredAgentSessionTransitionJournal } from './structured-agent-session-transition' export type StructuredAgentSessionSinkOperation = { sequence: number @@ -85,6 +86,8 @@ export class StructuredAgentSessionSinkQueue { journalLinkage = (): StructuredAgentSessionLinkageJournal | null => this.target?.journal ?? null + journalItems = (): StructuredAgentSessionTransitionJournal | null => this.target?.journal ?? null + journalStopDecidesTurn = (turnId: string, endedAt: number, openedBy?: string): boolean => this.target?.journal.stopMarks.personStopDecides(turnId, endedAt, openedBy) ?? false diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts index 49572abdcc00..5cdabdce713d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts @@ -12,6 +12,10 @@ import { estimateStructuredAgentSessionItemBytes } from './structured-agent-sess import { StructuredAgentSessionSinkQueue } from './structured-agent-session-event-sink-queue' import { structuredAgentSessionJournalAppendOptions } from './structured-agent-session-journal-append-options' import { createStructuredAgentSessionResolvedAppend } from './structured-agent-session-resolved-append' +import { + createStructuredAgentSessionTransitionMembers, + type StructuredAgentSessionTransitionSink +} from './structured-agent-session-transition' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' export type StructuredAgentSessionSinkAdmission = @@ -47,6 +51,8 @@ export type StructuredAgentSessionPublishOptions = Pick< /** An item write states which turn its row belongs to; the write that creates the row decides. */ export type StructuredAgentSessionItemAppendOptions = StructuredAgentSessionAppendOptions & { turnScope: AgentJournalTurnScope + /** The provider's reference for the item, on the write that creates its row. */ + providerItemRef?: string } export type StructuredAgentSessionLifecycleJournal = Pick< @@ -80,7 +86,7 @@ export type StructuredAgentSessionRevisionOptions = StructuredAgentSessionItemAp /** Compatibility alias for lifecycle callers that already use this resolver. */ export type StructuredAgentSessionLifecycleIdentityResolver = StructuredAgentSessionIdentityResolver -export type StructuredAgentSessionEventSink = { +export type StructuredAgentSessionEventSink = StructuredAgentSessionTransitionSink & { appendItem( identity: AgentJournalItemIdentity, body: AgentJournalItemBody, @@ -286,6 +292,7 @@ export function createDeferredStructuredAgentSessionEventSink(deps: { }, tryAppendItem: appendItem, ...resolvedAppend, + ...createStructuredAgentSessionTransitionMembers(queue), journalEpoch: queue.journalEpoch, journalLinkage: queue.journalLinkage, journalStopDecidesTurn: queue.journalStopDecidesTurn, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts index e7ce667ed2b6..57bb482f7c2b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts @@ -16,6 +16,7 @@ export function structuredAgentSessionJournalAppendOptions( fence, ...(options.observedAt === undefined ? {} : { observedAt: options.observedAt }), turnScope: options.turnScope, + ...(options.providerItemRef === undefined ? {} : { providerItemRef: options.providerItemRef }), ...agentJournalLinkageFields(options) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts new file mode 100644 index 000000000000..1b242dd8d983 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts @@ -0,0 +1,265 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemIdentity, + type AgentJournalToolCallItem +} from '../../../shared/agent-session-journal-types' +import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { createAgentSessionDeltaCoalescer } from './agent-session-delta-coalescer' +import { + createDeferredStructuredAgentSessionEventSink, + type StructuredAgentSessionSinkWatermarks +} from './structured-agent-session-event-sink' +import { testEventSinkLogging } from './structured-agent-session-logger-test-support' +import type { + StructuredAgentSessionTransition, + StructuredAgentSessionTransitionStep +} from './structured-agent-session-transition' + +const SESSION = 'session-transition' +const journals = createTrackedJournalOpener() +const roots: string[] = [] + +afterEach(async () => { + await journals.closeAll() + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) +}) + +function identity(recordId: string): AgentJournalItemIdentity { + return { provider: 'legacy', agent: 'grok', sessionId: SESSION, recordId } +} + +function tool(name: string, state: AgentJournalToolCallItem['state']): AgentJournalToolCallItem { + return { kind: 'tool-call', name, input: { name }, state } +} + +function itemStep( + resolve: Extract['resolve'] +): StructuredAgentSessionTransitionStep { + return { + kind: 'item', + reservedBytes: 4096, + resolve, + options: { turnScope: AGENT_JOURNAL_THREAD_SCOPE } + } +} + +async function rig(watermarks: Partial = {}) { + const root = await mkdtemp(join(tmpdir(), 'orca-transition-')) + roots.push(root) + const journal: AgentSessionJournal = await journals.open({ + identity: { + sessionId: SESSION, + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'grok', + providerHandle: { kind: 'opaque', agent: 'grok', value: 'provider-session-1' } + }, + stateDirectory: root, + now: () => 1_000 + }) + const publishes: number[] = [] + const deferred = createDeferredStructuredAgentSessionEventSink({ + ...testEventSinkLogging(SESSION), + watermarks + }) + const bind = () => deferred.bind({ journal, fence: 1, publish: () => publishes.push(1) }) + return { journal, deferred, sink: deferred.sink, publishes, bind } +} + +describe('structured agent-session transitions', () => { + it('lands its steps back to back, each resolved after the one before it', async () => { + const { journal, deferred, sink, publishes, bind } = await rig() + bind() + const first = identity('first') + const transition: StructuredAgentSessionTransition = { + lifecycle: false, + publish: true, + steps: [ + itemStep(() => ({ identity: first, body: tool('read', 'running') })), + // Reads the row the step before it wrote. + itemStep((view) => { + const before = view.itemBody(agentJournalItemKey(first)) + return before?.kind === 'tool-call' + ? { identity: identity('second'), body: tool(`after-${before.name}`, 'running') } + : null + }) + ] + } + expect(sink.tryAppendTransition?.(transition)).toEqual({ accepted: true }) + sink.tryAppendItem?.(identity('later'), tool('later', 'running'), { + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) + await deferred.drained() + + const items = journal.snapshot().items + const start = items[0]?.sequence ?? 0 + expect(items.map((item) => [item.itemId, item.sequence - start])).toEqual([ + [agentJournalItemKey(first), 0], + [agentJournalItemKey(identity('second')), 1], + [agentJournalItemKey(identity('later')), 2] + ]) + expect(items[1]?.body).toMatchObject({ name: 'after-read' }) + expect(publishes).toHaveLength(1) + }) + + it('places a row where its resolver says, and hears every landing', async () => { + const { journal, deferred, sink, bind } = await rig() + bind() + const landed: (readonly boolean[])[] = [] + const scope = { kind: 'turn', turnItemId: 'turn-row' } as const + sink.tryAppendTransition?.({ + lifecycle: false, + publish: true, + steps: [ + itemStep(() => ({ + identity: identity('placed'), + body: tool('read', 'running'), + options: { turnScope: scope, providerItemRef: 'item:read' } + })), + itemStep(() => null) + ], + landed: (wrote) => landed.push(wrote) + }) + await deferred.drained() + + expect(journal.item(agentJournalItemKey(identity('placed')))).toMatchObject({ + turnScope: scope, + providerItemRef: 'item:read' + }) + expect(landed).toEqual([[true, false]]) + }) + + it('refuses a transition whole, so none of its steps ever lands', async () => { + const { journal, deferred, sink, bind } = await rig({ maxQueuedOperations: 1 }) + const step = (recordId: string) => + itemStep(() => ({ identity: identity(recordId), body: tool(recordId, 'running') })) + const admitted = { lifecycle: false, publish: false, steps: [step('a')] } + const refused = { lifecycle: false, publish: false, steps: [step('b'), step('c')] } + + expect(sink.tryAppendTransition?.(admitted)).toEqual({ accepted: true }) + expect(sink.tryAppendTransition?.(refused)).toEqual({ + accepted: false, + reason: 'backpressure' + }) + bind() + await deferred.drained() + + expect(journal.snapshot().items.map((item) => item.itemId)).toEqual([ + agentJournalItemKey(identity('a')) + ]) + }) + + it('settles from the rows as they stand, in consecutive rows when one cannot hold them', async () => { + const { journal, deferred, sink, publishes, bind } = await rig() + bind() + const running = Array.from({ length: 250 }, (_, index) => identity(`tool-${index}`)) + for (const id of running) { + sink.tryAppendItem?.(id, tool('read', 'running'), { turnScope: AGENT_JOURNAL_THREAD_SCOPE }) + } + sink.tryAppendItem?.(identity('done'), tool('read', 'completed'), { + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) + let wrote: readonly boolean[] = [] + expect( + sink.tryAppendTransition?.({ + lifecycle: true, + publish: true, + landed: (result) => { + wrote = result + }, + steps: [ + { + kind: 'settlement', + settlementId: 'settle-all', + reservedBytes: 1, + // The settled row is a candidate too; the fold, not the caller, rules it out. + resolve: (view) => + [...running, identity('done')].flatMap((id) => { + const body = view.itemBody(agentJournalItemKey(id)) + return body?.kind === 'tool-call' && body.state === 'running' + ? [ + { + kind: 'item' as const, + identity: id, + body: { ...body, state: 'failed' as const }, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + } + ] + : [] + }) + } + ] + }) + ).toEqual({ accepted: true }) + sink.tryAppendItem?.(identity('after'), tool('after', 'running'), { + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) + await deferred.drained() + + const items = journal.snapshot().items + const sequenceOf = (recordId: string) => + items.find((item) => item.itemId === agentJournalItemKey(identity(recordId)))?.sequence ?? 0 + expect( + items.filter((item) => item.body.kind === 'tool-call' && item.body.state === 'failed') + ).toHaveLength(250) + expect( + items.find((item) => item.itemId === agentJournalItemKey(identity('done')))?.body + ).toMatchObject({ state: 'completed' }) + // Two batch rows, back to back, between the last append before it and the first after it. + expect(sequenceOf('after') - sequenceOf('done')).toBe(3) + expect(wrote).toEqual([true]) + expect(publishes).toHaveLength(1) + }) + + it('writes and announces nothing when a step resolves to nothing', async () => { + const { journal, deferred, sink, publishes, bind } = await rig() + bind() + let wrote: readonly boolean[] = [] + sink.tryAppendTransition?.({ + lifecycle: true, + publish: true, + landed: (result) => { + wrote = result + }, + steps: [ + itemStep(() => null), + { kind: 'settlement', settlementId: 'none', reservedBytes: 1, resolve: () => [] } + ] + }) + await deferred.drained() + + expect(journal.snapshot().items).toEqual([]) + expect(wrote).toEqual([false, false]) + expect(publishes).toEqual([]) + }) +}) + +describe('coalescer text a caller writes itself', () => { + it('reports unwritten streams and owes nothing once the caller marks them written', () => { + const emitted: string[] = [] + const coalescer = createAgentSessionDeltaCoalescer({ + emit: (_key, text) => { + emitted.push(text) + }, + schedule: () => () => {} + }) + coalescer.append('a', 'Hel') + coalescer.append('b', 'Wor') + coalescer.append('a', 'lo') + + expect(coalescer.dirty().map(({ key, snapshot }) => [key, snapshot.text])).toEqual([ + ['a', 'Hello'], + ['b', 'Wor'] + ]) + coalescer.markFlushed('a') + coalescer.flushAll() + expect(emitted).toEqual(['Wor']) + expect(coalescer.dirty()).toEqual([]) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts new file mode 100644 index 000000000000..4304338a148f --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts @@ -0,0 +1,156 @@ +// One provider event's journal writes, admitted as a single sink operation. +// +// A producer that keeps state about what it wrote must change that state only for writes the sink +// took; when an event needs several rows, a refusal of the third after the first two were taken +// would leave the producer and the journal disagreeing. A transition is admitted whole or not at +// all. At execution its steps are issued in the same tick, so they sit together in the journal's +// write queue, and each step resolves against the fold with every earlier write landed — the +// steps before it included — so what a step writes is decided by the journal, not by memory. + +import type { + AgentJournalItemBody, + AgentJournalItemIdentity +} from '../../../shared/agent-session-journal-types' +import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { estimateStructuredAgentSessionItemBytes } from './structured-agent-session-event-sink-estimate' +import type { + StructuredAgentSessionItemAppendOptions, + StructuredAgentSessionSinkAdmission +} from './structured-agent-session-event-sink' +import type { StructuredAgentSessionSinkQueue } from './structured-agent-session-event-sink-queue' +import { structuredAgentSessionJournalAppendOptions } from './structured-agent-session-journal-append-options' + +/** What a transition step reads: rows by key or provider reference, every row, and their turns. */ +export type StructuredAgentSessionTransitionJournal = Pick< + AgentSessionJournal, + | 'epoch' + | 'visitItems' + | 'itemBody' + | 'item' + | 'itemIdForProviderItemRef' + | 'canonicalItemId' + | 'visitItemsWithLinkage' +> + +export type StructuredAgentSessionTransitionStep = + | { + kind: 'item' + /** Bounds what `resolve` may write; a larger write fails the sink, unless `paced`. */ + reservedBytes: number + /** The size only paces the queue: the row grows by text the fold holds (a resumed message). */ + paced?: true + /** The row and its whole body; null writes nothing. Resolved `options` replace the planned + * ones, for a writer that learns the row's turn or producer only from the fold. */ + resolve: (journal: StructuredAgentSessionTransitionJournal) => { + identity: AgentJournalItemIdentity + body: AgentJournalItemBody + options?: StructuredAgentSessionItemAppendOptions + } | null + options: StructuredAgentSessionItemAppendOptions + } + | { + kind: 'settlement' + /** Unique per settlement: the journal applies one id once. */ + settlementId: string + /** Paces the queue only; the mutations are the journal's to choose. */ + reservedBytes: number + /** Read at execution; none writes nothing. */ + resolve: ( + journal: StructuredAgentSessionTransitionJournal + ) => readonly JournalLifecycleMutationInput[] + } + +export type StructuredAgentSessionTransition = { + steps: readonly StructuredAgentSessionTransitionStep[] + /** Rides the sink's lifecycle budget: it ends or settles something. */ + lifecycle: boolean + /** Announce the writes once they land, when any step wrote. */ + publish: boolean + /** Which steps wrote a row, once every step has landed or one failed (then none counts). */ + landed?: (wrote: readonly boolean[]) => void +} + +const STEP_OVERFLOW = 'structured agent-session transition step exceeded its reserved size' + +/** The sink members a transition writer uses. */ +export type StructuredAgentSessionTransitionSink = { + /** Queues one event's writes as a single admitted operation. */ + tryAppendTransition?( + transition: StructuredAgentSessionTransition + ): StructuredAgentSessionSinkAdmission + /** The bound journal's rows as they stand now; null until bound. */ + journalItems?(): StructuredAgentSessionTransitionJournal | null +} + +export function createStructuredAgentSessionTransitionMembers( + queue: StructuredAgentSessionSinkQueue +): Required { + return { tryAppendTransition: transitionAppend(queue), journalItems: queue.journalItems } +} + +function transitionAppend( + queue: StructuredAgentSessionSinkQueue +): (transition: StructuredAgentSessionTransition) => StructuredAgentSessionSinkAdmission { + return (transition) => + queue.submit({ + bytes: + transition.steps.reduce((total, step) => total + step.reservedBytes, 0) + + (transition.publish ? 1 : 0), + lifecycle: transition.lifecycle, + run: async (bound) => { + const { journal, fence } = bound + // Issued in one tick, so no write submitted after this transition lands between its steps. + const writes = transition.steps.map((step) => + step.kind === 'item' + ? journal + .appendResolvedItem( + () => { + const resolved = step.resolve(journal) + if (!resolved) { + return null + } + if ( + !step.paced && + estimateStructuredAgentSessionItemBytes(resolved.identity, resolved.body) > + step.reservedBytes + ) { + throw new Error(STEP_OVERFLOW) + } + return { + identity: resolved.identity, + body: resolved.body, + ...(resolved.options + ? { + options: structuredAgentSessionJournalAppendOptions( + fence, + resolved.options + ) + } + : {}) + } + }, + structuredAgentSessionJournalAppendOptions(fence, step.options) + ) + .then((landed) => landed !== null) + : journal + .appendResolvedLifecycleBatch({ + settlementId: step.settlementId, + resolve: () => step.resolve(journal), + fence + }) + .then((landed) => landed !== null) + ) + let wrote: boolean[] = transition.steps.map(() => false) + try { + wrote = await Promise.all(writes) + } finally { + // Heard even when a write failed, so a writer's speculation about it always ends. + transition.landed?.(wrote) + } + if (transition.publish && wrote.some(Boolean)) { + bound.publish() + } + } + }) +} diff --git a/src/shared/agent-session-journal-types.ts b/src/shared/agent-session-journal-types.ts index e5f8125a5c0f..271053b1b214 100644 --- a/src/shared/agent-session-journal-types.ts +++ b/src/shared/agent-session-journal-types.ts @@ -360,8 +360,16 @@ export const AGENT_JOURNAL_THREAD_SCOPE: AgentJournalTurnScope = { kind: 'thread /** Who produced a row and which turn it belongs to: what every item write states. */ export type AgentJournalRowAttribution = AgentJournalProducerLinkage & { turnScope: AgentJournalTurnScope + providerItemRef?: AgentJournalProviderItemRef } +/** The provider's own reference for the item a row is, as its producer joined it, on a row whose + * identity does not spell it (a Codex message keys by its turn ordinal, and Codex renumbers its + * item ids on resume). Set by the write that creates the row and kept by every revision, like the + * turn scope. Only its producer reads it back, to find the row again after a restart; no client + * depends on it. */ +export type AgentJournalProviderItemRef = string + /** Where the journal placed an item: the sequence of the row that created it, * then its place among that row's writes. The timeline's only ordering key. */ export type AgentJournalPosition = { @@ -387,6 +395,7 @@ export type AgentJournalRenderItem = AgentJournalProducerLinkage & { recoveredAt?: number /** Absent only from a host that predates it. */ turnScope?: AgentJournalTurnScope + providerItemRef?: AgentJournalProviderItemRef } // ─── Submissions ──────────────────────────────────────────────────────────── diff --git a/tests/e2e/cross-version-wire/agent-session-provider-item-ref-downgrade.unit.test.ts b/tests/e2e/cross-version-wire/agent-session-provider-item-ref-downgrade.unit.test.ts new file mode 100644 index 000000000000..ff0ecc5fed33 --- /dev/null +++ b/tests/e2e/cross-version-wire/agent-session-provider-item-ref-downgrade.unit.test.ts @@ -0,0 +1,127 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, test } from 'vitest' +import type { AgentJournalItemIdentity } from '../../../src/shared/agent-session-journal-types' +import { createTrackedJournalOpener } from '../../../src/main/native-chat/agent-session-journal/journal-host-database-test-support' +import type { JournalRow } from '../../../src/main/native-chat/agent-session-journal/journal-row-schema' +import { importReleaseCheckoutModule, materializeReleaseCheckout } from './release-checkout' + +// A release that predates `providerItemRef` on journal rows and render items. +const BASELINE_REF = 'v1.4.218' + +const SESSION = 'session-provider-item-ref' + +function message(ordinal: number): AgentJournalItemIdentity { + return { provider: 'codex', threadId: 'thread-1', turnId: 'turn-1', ordinal } +} + +function releaseExport(module: Record, name: string): T { + const value = module[name] + if (value === undefined) { + throw new Error(`the pinned release exports no ${name}`) + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: an export the pinned release has; each caller names the shape it uses, and a changed one fails the test. + return value as T +} + +type OldRenderItem = { itemId: string; [field: string]: unknown } + +// Loads a real old build, including cold extraction and transforms. +test('an older host and client render a row carrying a provider item reference as if it had none', async () => { + const directory = mkdtempSync(join(tmpdir(), 'orca-provider-item-ref-downgrade-')) + const journals = createTrackedJournalOpener() + try { + // This build: the same message twice, the first carrying the reference. + const journal = await journals.open({ + identity: { + sessionId: SESSION, + workspaceId: 'ws-1', + hostId: 'host-1', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: 'thread-1' } + }, + stateDirectory: directory, + now: () => 1_000 + }) + const body = { + kind: 'message' as const, + role: 'assistant' as const, + blocks: [{ type: 'text' as const, text: 'Hello' }] + } + const scope = { kind: 'thread' as const } + await journal.appendItem(message(0), body, { + fence: 1, + turnScope: scope, + providerItemRef: 'item:p:thread-1/item-1' + }) + await journal.appendItem(message(1), body, { fence: 1, turnScope: scope }) + const since = journal.readSince({ epoch: journal.epoch, sequence: 0 }) + if (!since.ok) { + throw new Error(`expected rows, got reset ${since.reset}`) + } + const rows: JournalRow[] = since.rows + const current = journal.snapshot().items + expect(current[0]?.providerItemRef).toBe('item:p:thread-1/item-1') + + const checkout = await materializeReleaseCheckout(BASELINE_REF) + const [reducer, schemas, projection] = await Promise.all( + [ + 'src/main/native-chat/agent-session-journal/journal-reducer.ts', + 'src/shared/agent-session-journal-schemas.ts', + 'src/shared/structured-agent-session-projection.ts' + ].map((path) => importReleaseCheckoutModule(checkout, path)) + ) + const createState = releaseExport<(sessionId: string, epoch: string) => unknown>( + reducer, + 'createJournalReducerState' + ) + const applyRow = releaseExport<(state: unknown, row: JournalRow) => void>( + reducer, + 'applyJournalRow' + ) + const render = releaseExport<(state: unknown) => { items: OldRenderItem[] }>( + reducer, + 'renderJournalState' + ) + const renderItemSchema = releaseExport<{ parse: (value: unknown) => OldRenderItem }>( + schemas, + 'AgentJournalRenderItemSchema' + ) + const project = releaseExport<(item: unknown) => unknown>( + projection, + 'projectStructuredItemToNativeChat' + ) + + // The older host, after a downgrade, folds the rows as it always did. + const state = createState(SESSION, journal.epoch) + for (const row of rows) { + applyRow(state, row) + } + const folded = render(state).items + expect(folded.map((item) => item.itemId)).toEqual([ + 'codex:thread-1:turn-1:0', + 'codex:thread-1:turn-1:1' + ]) + const [first, second] = folded + if (!first || !second) { + throw new Error('the older host folded both rows') + } + expect(first).not.toHaveProperty('providerItemRef') + const same = ({ itemId: _id, sequence: _seq, ...rest }: OldRenderItem) => rest + expect(same(first)).toEqual(same(second)) + + // The older client reads this host's render item without the field, and renders it the same. + const [withRef, withoutRef] = current.map((item) => renderItemSchema.parse(item)) + if (!withRef || !withoutRef) { + throw new Error('the older client read both items') + } + expect(withRef).not.toHaveProperty('providerItemRef') + expect(project(withRef)).toEqual( + project({ ...withoutRef, itemId: withRef.itemId, sequence: withRef.sequence }) + ) + } finally { + await journals.closeAll() + rmSync(directory, { recursive: true, force: true }) + } +}) From 7ad901be3caa50af68b5c3558142d7cb45abf2fc Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:51:51 -0700 Subject: [PATCH 03/11] Recover message ordinals from the journal's highest place, and forget joins read from a replaced epoch A fresh join index continued a turn's messages at the first free place, so a journal holding only a later ordinal (an imported or removed earlier row) had its sequence back-filled. The place is now one past the highest ordinal any row or echoed send holds there, read through a pure scheme reader. The join caches also drop what they read when the journal's epoch is replaced. --- .../codex/codex-provider-timeline-identity.ts | 4 + .../provider-timeline-identity.ts | 5 + .../provider-timeline-joins.test.ts | 55 ++++++++ .../provider-timeline-joins.ts | 86 +++++------- .../provider-timeline-message-places.ts | 122 ++++++++++++++++++ .../structured-agent-session-transition.ts | 4 +- 6 files changed, 220 insertions(+), 56 deletions(-) create mode 100644 src/main/native-chat/agent-session-timeline/provider-timeline-message-places.ts diff --git a/src/main/codex/codex-provider-timeline-identity.ts b/src/main/codex/codex-provider-timeline-identity.ts index 029c13fe56e2..7da2bbd33aae 100644 --- a/src/main/codex/codex-provider-timeline-identity.ts +++ b/src/main/codex/codex-provider-timeline-identity.ts @@ -20,6 +20,10 @@ export function createCodexProviderTimelineIdentityScheme(input: { codexTurnLifecycleIdentity(input.sessionId, address.key.value) return { ordinalMessages: true, + messageSlot: (identity) => + identity.provider === 'codex' + ? { place: `${identity.threadId}\u001f${identity.turnId}`, ordinal: identity.ordinal } + : null, turn, turnId: (address) => address.key.value, turnOpener: (address) => { diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-identity.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-identity.ts index d364b4d31acf..77286d029055 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-identity.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-identity.ts @@ -52,11 +52,16 @@ export type ProviderTimelineItemAddress = { incarnation: number } +export type ProviderTimelineMessageSlot = { place: string; ordinal: number } + export type ProviderTimelineIdentityScheme = { /** Messages inside a turn are keyed by their place among the turn's messages, not by their key. * Such an identity must spell only (thread, turn, ordinal), and its row carries the key as * `providerItemRef`, since nothing else could find it again. */ ordinalMessages: boolean + /** Reads an ordinal message identity back: its (thread, turn) place, spelled by the scheme, and + * its ordinal; null for any other identity. Required with `ordinalMessages`. */ + messageSlot?(identity: AgentJournalItemIdentity): ProviderTimelineMessageSlot | null turn(address: ProviderTimelineTurnAddress): AgentJournalItemIdentity /** The id the turn row carries and a client's Stop names. */ turnId(address: ProviderTimelineTurnAddress): string diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts index b132802369f0..8cddc98a2278 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts @@ -62,6 +62,13 @@ const inTurn = (turn: string) => ({ scope: { kind: 'turn', turnItemId: `turn:${turn}` } as const }) +const codexSlot = (turn: string, ordinal: number): AgentJournalItemIdentity => ({ + provider: 'codex', + threadId: 'root', + turnId: turn, + ordinal +}) + async function write( journal: AgentSessionJournal, row: { identity: AgentJournalItemIdentity; ref?: string }, @@ -118,6 +125,54 @@ describe('provider timeline joins', () => { ) }) + it('continues past the highest ordinal the journal holds, never into a gap below it', async () => { + const journal = await openJournal() + await write(journal, { identity: codexSlot('t1', 2), ref: 'item:old' }, 'old') + // Another turn and another thread keep their own places. + await write(journal, { identity: codexSlot('t2', 7) }, 'other turn') + await write(journal, { identity: { ...codexSlot('t1', 9), threadId: 'sub' } }, 'subagent') + const joins = codexJoins() + expect(joins.place(message('new'), 'message', inTurn('t1'), journal).itemId).toBe( + 'codex:root:t1:3' + ) + expect(joins.place(message('next'), 'message', inTurn('t1'), journal).itemId).toBe( + 'codex:root:t1:4' + ) + }) + + it('continues past an echoed send above the highest row', async () => { + const journal = await openJournal() + await write(journal, { identity: codexSlot('t1', 0) }, 'first') + await journal.appendSubmission({ + clientMessageId: 'send-1', + payloadFingerprint: 'fp', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }, + fence: 1 + }) + await journal.resolveDispatch({ + clientMessageId: 'send-1', + state: 'accepted', + providerIdentity: codexSlot('t1', 4), + fence: 1 + }) + expect(codexJoins().place(message('reply'), 'message', inTurn('t1'), journal).itemId).toBe( + 'codex:root:t1:5' + ) + }) + + it('forgets what it read from an epoch the journal replaced', async () => { + const journal = await openJournal() + const joins = codexJoins() + const m0 = joins.place(message('m0'), 'message', inTurn('t1'), journal) + await write(journal, m0, 'first') + expect(joins.find(message('m0'), journal)?.itemId).toBe('codex:root:t1:0') + await journal.replaceEpochItems('handle_forked', 1, []) + expect(joins.find(message('m0'), journal)).toBeNull() + expect(joins.place(message('m1'), 'message', inTurn('t1'), journal).itemId).toBe( + 'codex:root:t1:0' + ) + }) + it('finds a row whose identity spells its key without any reference', async () => { const journal = await openJournal() const scheme = createLegacyProviderTimelineIdentityScheme({ diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-joins.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-joins.ts index 1b85f32a0aa1..7cb48073933d 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-joins.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-joins.ts @@ -22,7 +22,6 @@ import { import { BoundedMap } from '../../../shared/bounded-map' import type { StructuredAgentSessionTransitionJournal } from '../agent-session-wire/structured-agent-session-transition' import { - providerTimelineKeyPart, spellProviderTimelineKey, type ProviderTimelineIdentityScheme, type ProviderTimelineItemAddress, @@ -31,12 +30,10 @@ import { type ProviderTimelineKey, type ProviderTimelineTurnAddress } from './provider-timeline-identity' +import { ProviderTimelineMessagePlaces } from './provider-timeline-message-places' const MAX_JOINED_ROWS = 1_024 const MAX_JOINED_ROW_BYTES = 1024 * 1024 -const MAX_TURN_ORDINALS = 256 -/** Echoed sends whose slot no row holds yet; a lane records each on its send soon after. */ -const MAX_RESERVED_ECHOES = 64 /** One provider item as the provider names it. Item keys are the provider's per thread. */ export type ProviderTimelineItemJoin = { @@ -81,12 +78,9 @@ export class ProviderTimelineJoins { maxBytes: MAX_JOINED_ROW_BYTES, sizeOf: (row, key) => bytes(key) + 2 * bytes(row.itemId) + 64 }) - private readonly ordinals = new BoundedMap({ - maxEntries: MAX_TURN_ORDINALS, - maxBytes: MAX_TURN_ORDINALS * 1024, - sizeOf: (_next, place) => bytes(place) + 8 - }) - private readonly echoes = new BoundedMap({ maxEntries: MAX_RESERVED_ECHOES }) + private readonly places: ProviderTimelineMessagePlaces + /** The journal epoch every cache entry was read from; a replaced epoch holds other rows. */ + private epoch: string | null = null private serial = 0 constructor( @@ -95,7 +89,12 @@ export class ProviderTimelineJoins { generation: string namespace: string } - ) {} + ) { + this.places = new ProviderTimelineMessagePlaces({ + scheme: deps.scheme, + namespace: () => this.deps.namespace + }) + } get namespace(): string { return this.deps.namespace @@ -105,7 +104,19 @@ export class ProviderTimelineJoins { reset(namespace: string): void { this.deps.namespace = namespace this.rows.clear() - this.ordinals.clear() + this.places.clear() + } + + /** Drops every cache read from an epoch the journal has since replaced (a rewind, an import). */ + private sync(journal: Journal): void { + if (!journal || journal.epoch === this.epoch) { + return + } + if (this.epoch !== null) { + this.rows.clear() + this.places.clear() + } + this.epoch = journal.epoch } /** A key unique to this acquisition, for something the provider names nothing. */ @@ -143,6 +154,7 @@ export class ProviderTimelineJoins { journal: Journal, namespace = this.deps.namespace ): ProviderTimelineRow | null { + this.sync(journal) const ref = this.reference(join, namespace) const cached = this.rows.get(ref) // A planner ahead of a reset asks about a namespace whose rows this index does not spell yet. @@ -165,7 +177,11 @@ export class ProviderTimelineJoins { placement: ProviderTimelinePlacement, journal: Journal ): ProviderTimelineRow { - const ordinal = this.ordinalFor(itemClass, placement, journal) + this.sync(journal) + const ordinal = + this.deps.scheme.ordinalMessages && itemClass === 'message' && placement.turn + ? this.places.next(placement.thread, placement.turn, journal) + : null const identity = this.deps.scheme.item({ ...this.address(join, 1), turn: placement.turn, @@ -194,7 +210,7 @@ export class ProviderTimelineJoins { return } const row = this.place(join, 'message', placement, journal) - this.echoes.set(row.itemId, true) + this.places.reserve(row.itemId) } /** The request under `key` the journal holds last: the highest incarnation with a row. */ @@ -203,6 +219,7 @@ export class ProviderTimelineJoins { journal: Journal, namespace = this.deps.namespace ): ProviderTimelineRow | null { + this.sync(journal) const join = { family: 'request' as const, key, thread: null } const ref = this.reference(join, namespace) const cached = this.rows.get(ref) @@ -269,47 +286,6 @@ export class ProviderTimelineJoins { private spelled(join: ProviderTimelineItemJoin, incarnation: number): AgentJournalItemIdentity { return this.deps.scheme.item(this.address(join, incarnation)) } - - /** The next free place among the turn's messages: past every row and echoed send holding one. */ - private ordinalFor( - itemClass: ProviderTimelineItemClass, - placement: ProviderTimelinePlacement, - journal: Journal - ): number | null { - const turn = placement.turn - if (!this.deps.scheme.ordinalMessages || itemClass !== 'message' || !turn) { - return null - } - const place = `${providerTimelineKeyPart(placement.thread ?? '')}\u001f${spellProviderTimelineKey( - this.deps.namespace, - turn - )}` - let ordinal = this.ordinals.get(place) ?? 0 - while (this.taken(this.slot(placement.thread, turn, ordinal), journal)) { - ordinal += 1 - } - this.ordinals.set(place, ordinal + 1) - return ordinal - } - - private slot(thread: string | null, turn: ProviderTimelineKey, ordinal: number): string { - return agentJournalItemKey( - this.deps.scheme.item({ - ...this.address({ family: 'item', key: turn, thread }, 1), - turn, - itemClass: 'message', - messageOrdinal: ordinal - }) - ) - } - - private taken(itemId: string, journal: Journal): boolean { - return ( - this.echoes.has(itemId) || - (journal !== null && - (journal.itemBody(itemId) !== null || journal.canonicalItemId(itemId) !== itemId)) - ) - } } function rowOf( diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-message-places.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-message-places.ts new file mode 100644 index 000000000000..e5350c39c93a --- /dev/null +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-message-places.ts @@ -0,0 +1,122 @@ +// Where a new message goes among its (thread, turn)'s messages, for a scheme that keys messages +// by that place. Read back from the journal on a miss, past every place a row or an echoed send +// holds, so a restart continues the sequence the journal already has. + +import { + agentJournalItemKey, + parseAgentJournalItemKey +} from '../../../shared/agent-session-journal-item-key' +import type { AgentJournalItemIdentity } from '../../../shared/agent-session-journal-types' +import { BoundedMap } from '../../../shared/bounded-map' +import type { StructuredAgentSessionTransitionJournal } from '../agent-session-wire/structured-agent-session-transition' +import { + providerTimelineKeyPart, + spellProviderTimelineKey, + type ProviderTimelineIdentityScheme, + type ProviderTimelineKey +} from './provider-timeline-identity' + +const MAX_TURN_ORDINALS = 256 +/** Echoed sends whose slot no row holds yet; a lane records each on its send soon after. */ +const MAX_RESERVED_ECHOES = 64 + +type Journal = StructuredAgentSessionTransitionJournal | null + +function bytes(value: string): number { + return Buffer.byteLength(value, 'utf8') +} + +export class ProviderTimelineMessagePlaces { + /** The next ordinal per place. */ + private readonly ordinals = new BoundedMap({ + maxEntries: MAX_TURN_ORDINALS, + maxBytes: MAX_TURN_ORDINALS * 1024, + sizeOf: (_next, place) => bytes(place) + 8 + }) + private readonly echoes = new BoundedMap({ + maxEntries: MAX_RESERVED_ECHOES, + maxBytes: MAX_RESERVED_ECHOES * 1024, + sizeOf: (_held, itemId) => bytes(itemId) + 8 + }) + + constructor( + private readonly deps: { + scheme: ProviderTimelineIdentityScheme + namespace: () => string + } + ) {} + + clear(): void { + this.ordinals.clear() + this.echoes.clear() + } + + /** Holds a place no row takes (an echoed send's). */ + reserve(itemId: string): void { + this.echoes.set(itemId, true) + } + + /** The next place in (thread, turn): allocating, so resolvers only. */ + next(thread: string | null, turn: ProviderTimelineKey, journal: Journal): number { + const place = `${providerTimelineKeyPart(thread ?? '')}\u001f${spellProviderTimelineKey( + this.deps.namespace(), + turn + )}` + let ordinal = this.ordinals.get(place) ?? this.highWater(thread, turn, journal) + // Another writer of the same turn may have taken places since this one cached its mark. + while (this.taken(agentJournalItemKey(this.slot(thread, turn, ordinal)), journal)) { + ordinal += 1 + } + this.ordinals.set(place, ordinal + 1) + return ordinal + } + + /** One past the highest ordinal the journal holds in the place, like the ordinal counter it + * continues: a gap below it (a removed or never-imported row) is never filled. */ + private highWater(thread: string | null, turn: ProviderTimelineKey, journal: Journal): number { + const read = this.deps.scheme.messageSlot + const place = read?.(this.slot(thread, turn, 0))?.place + if (!read || place === undefined || !journal) { + return 0 + } + let next = 0 + const consider = (itemId: string | null) => { + const identity = itemId === null ? null : parseAgentJournalItemKey(itemId) + const slot = identity ? read(identity) : null + if (slot?.place === place) { + next = Math.max(next, slot.ordinal + 1) + } + } + journal.visitItems((itemId) => consider(itemId)) + // An echoed send holds its place through its submission alias, with no row of its own. + for (const submission of journal.submissions()) { + consider(submission.providerItemId) + } + return next + } + + private slot( + thread: string | null, + turn: ProviderTimelineKey, + ordinal: number + ): AgentJournalItemIdentity { + return this.deps.scheme.item({ + namespace: this.deps.namespace(), + family: 'item', + key: turn, + thread, + turn, + itemClass: 'message', + messageOrdinal: ordinal, + incarnation: 1 + }) + } + + private taken(itemId: string, journal: Journal): boolean { + return ( + this.echoes.has(itemId) || + (journal !== null && + (journal.itemBody(itemId) !== null || journal.canonicalItemId(itemId) !== itemId)) + ) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts index 4304338a148f..c67dab93edef 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts @@ -21,10 +21,12 @@ import type { import type { StructuredAgentSessionSinkQueue } from './structured-agent-session-event-sink-queue' import { structuredAgentSessionJournalAppendOptions } from './structured-agent-session-journal-append-options' -/** What a transition step reads: rows by key or provider reference, every row, and their turns. */ +/** What a transition step reads: rows by key or provider reference, every row, their turns, and + * the sends whose echo holds a provider item's place. */ export type StructuredAgentSessionTransitionJournal = Pick< AgentSessionJournal, | 'epoch' + | 'submissions' | 'visitItems' | 'itemBody' | 'item' From 620b8d7deb28d287d327de731468e2446c0d60c4 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:19:08 -0700 Subject: [PATCH 04/11] Spell the subagent thread's message slot without spreading an identity union --- .../agent-session-timeline/provider-timeline-joins.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts index 8cddc98a2278..6a3f1e8d9d3f 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts @@ -62,9 +62,9 @@ const inTurn = (turn: string) => ({ scope: { kind: 'turn', turnItemId: `turn:${turn}` } as const }) -const codexSlot = (turn: string, ordinal: number): AgentJournalItemIdentity => ({ +const codexSlot = (turn: string, ordinal: number, threadId = 'root'): AgentJournalItemIdentity => ({ provider: 'codex', - threadId: 'root', + threadId, turnId: turn, ordinal }) @@ -130,7 +130,7 @@ describe('provider timeline joins', () => { await write(journal, { identity: codexSlot('t1', 2), ref: 'item:old' }, 'old') // Another turn and another thread keep their own places. await write(journal, { identity: codexSlot('t2', 7) }, 'other turn') - await write(journal, { identity: { ...codexSlot('t1', 9), threadId: 'sub' } }, 'subagent') + await write(journal, { identity: codexSlot('t1', 9, 'sub') }, 'subagent') const joins = codexJoins() expect(joins.place(message('new'), 'message', inTurn('t1'), journal).itemId).toBe( 'codex:root:t1:3' From ebb07e6461a8eb0d23a54811ccaa571a7b34253d Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 4 Oct 2026 17:05:07 -0700 Subject: [PATCH 05/11] Keep the journal store under its line limit after the main merge --- .../native-chat/agent-session-journal/journal-store.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index c101c21fd556..19a8b61a62cd 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -69,7 +69,7 @@ import type { JournalEpochController } from './journal-epoch-controller' import { JournalWriteQueue } from './journal-write-queue' import { createJournalStoreCollaborators } from './journal-store-collaborators' import { journalStoreLoadedFields } from './journal-store-open' -import type { JournalItemAppender, JournalResolvedItem } from './journal-item-appender' +import type { JournalItemAppender } from './journal-item-appender' import type { JournalLifecycleBatchAppender } from './journal-lifecycle-batch-appender' import type { JournalStopMarks } from './journal-stop-marks' @@ -306,12 +306,8 @@ export class AgentSessionJournal { } /** An upsert whose row is chosen from the fold at its own turn in the queue; null writes nothing. */ - appendResolvedItem( - resolve: () => JournalResolvedItem | null, - options: JournalItemAppendOptions - ): Promise { - return this.itemAppender.appendResolved(resolve, options) - } + appendResolvedItem: JournalItemAppender['appendResolved'] = (resolve, options) => + this.itemAppender.appendResolved(resolve, options) appendTombstone( identity: AgentJournalItemIdentity, From 03b9f2f3d2b358d7ea3e64e0a0f680feda58f6b0 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:13:40 -0700 Subject: [PATCH 06/11] Drop the provider item reference, join index and identity schemes from the transition PR Nothing in production reaches the state they defended (an assembler that lost its memory while its child keeps streaming the same turn), and the stored Codex id was positional. The legacy identity scheme moves to the assembler PR with its first caller; the Codex scheme and any persisted reference wait for Codex to move onto the assembler. The Codex ordinal counter goes back to codex/, since no neutral code imports it. --- config/scripts/ci-shard-timings.json | 2 +- .../codex/codex-provider-timeline-identity.ts | 47 --- .../codex-structured-item-translation.test.ts | 28 +- .../codex-structured-item-translation.ts | 8 +- .../codex/codex-structured-journal-items.ts | 4 +- .../codex-structured-journal-settlement.ts | 4 +- ...-structured-journal-translation-restore.ts | 7 +- ...ctured-journal-translation-streams.test.ts | 4 +- ...ex-structured-journal-translation-turns.ts | 2 +- src/main/codex/codex-thread-item-identity.ts | 8 +- .../codex-turn-ordinals.test.ts} | 25 +- .../codex-turn-ordinals.ts} | 19 +- .../journal-item-fold.ts | 10 +- .../journal-provider-item-ref.test.ts | 75 ----- .../agent-session-journal/journal-reducer.ts | 3 - .../journal-render-item.ts | 1 - .../journal-row-builders.ts | 5 +- .../journal-row-schema.ts | 12 +- .../journal-store-contracts.ts | 5 +- .../agent-session-journal/journal-store.ts | 2 - .../provider-timeline-identity.ts | 138 -------- .../provider-timeline-joins.test.ts | 231 ------------- .../provider-timeline-joins.ts | 306 ------------------ .../provider-timeline-message-places.ts | 122 ------- .../structured-agent-session-event-sink.ts | 2 - ...ed-agent-session-journal-append-options.ts | 1 - src/shared/agent-session-journal-types.ts | 9 - ...n-provider-item-ref-downgrade.unit.test.ts | 127 -------- 28 files changed, 59 insertions(+), 1148 deletions(-) delete mode 100644 src/main/codex/codex-provider-timeline-identity.ts rename src/main/{native-chat/agent-session-timeline/provider-turn-message-ordinals.test.ts => codex/codex-turn-ordinals.test.ts} (64%) rename src/main/{native-chat/agent-session-timeline/provider-turn-message-ordinals.ts => codex/codex-turn-ordinals.ts} (88%) delete mode 100644 src/main/native-chat/agent-session-journal/journal-provider-item-ref.test.ts delete mode 100644 src/main/native-chat/agent-session-timeline/provider-timeline-identity.ts delete mode 100644 src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts delete mode 100644 src/main/native-chat/agent-session-timeline/provider-timeline-joins.ts delete mode 100644 src/main/native-chat/agent-session-timeline/provider-timeline-message-places.ts delete mode 100644 tests/e2e/cross-version-wire/agent-session-provider-item-ref-downgrade.unit.test.ts diff --git a/config/scripts/ci-shard-timings.json b/config/scripts/ci-shard-timings.json index fb242a4788c8..73d0226943b8 100644 --- a/config/scripts/ci-shard-timings.json +++ b/config/scripts/ci-shard-timings.json @@ -1307,7 +1307,7 @@ "src/main/codex/codex-trust-grant-ledger.test.ts": 54, "src/main/codex/codex-trust-grant-main-thread-boundary.test.ts": 43, "src/main/codex/codex-trust-grant-telemetry.test.ts": 70, - "src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.test.ts": 96, + "src/main/codex/codex-turn-ordinals.test.ts": 96, "src/main/codex/codex-user-hook-trust-rebase-client.test.ts": 104, "src/main/codex/codex-user-hook-trust-moves.test.ts": 127, "src/main/codex/codex-wsl-hook-install-plan.test.ts": 51, diff --git a/src/main/codex/codex-provider-timeline-identity.ts b/src/main/codex/codex-provider-timeline-identity.ts deleted file mode 100644 index 7da2bbd33aae..000000000000 --- a/src/main/codex/codex-provider-timeline-identity.ts +++ /dev/null @@ -1,47 +0,0 @@ -// Codex's persisted identities, spelled for the shared timeline assembler. -// -// The same rows Codex writes today (`codexItemIdentity`, `codexTurnLifecycleIdentity`): a user or -// assistant message inside a turn keys by (thread, turn, message ordinal), which survives a resume -// because the ordinal counts messages only; anything else keys by its Codex item id; a turn keys by -// its Codex turn id. Codex ids name one conversation for good, so the namespace spells nothing. - -import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key' -import type { ProviderTimelineIdentityScheme } from '../native-chat/agent-session-timeline/provider-timeline-identity' -import { - codexTurnLifecycleIdentity, - codexTurnUserItemId -} from './codex-structured-journal-translation-turns' - -export function createCodexProviderTimelineIdentityScheme(input: { - sessionId: string - primaryThreadId: () => string | null -}): ProviderTimelineIdentityScheme { - const turn: ProviderTimelineIdentityScheme['turn'] = (address) => - codexTurnLifecycleIdentity(input.sessionId, address.key.value) - return { - ordinalMessages: true, - messageSlot: (identity) => - identity.provider === 'codex' - ? { place: `${identity.threadId}\u001f${identity.turnId}`, ordinal: identity.ordinal } - : null, - turn, - turnId: (address) => address.key.value, - turnOpener: (address) => { - const thread = input.primaryThreadId() - return thread === null - ? agentJournalItemKey(turn(address)) - : codexTurnUserItemId(thread, address.key.value) - }, - item: (address) => { - const threadId = address.thread ?? input.primaryThreadId() ?? '' - return address.itemClass === 'message' && address.turn && address.messageOrdinal !== null - ? { - provider: 'codex', - threadId, - turnId: address.turn.value, - ordinal: address.messageOrdinal - } - : { provider: 'orca', clientMessageId: `codex-item:${threadId}:${address.key.value}` } - } - } -} diff --git a/src/main/codex/codex-structured-item-translation.test.ts b/src/main/codex/codex-structured-item-translation.test.ts index 29562552a02c..ad63d624f58b 100644 --- a/src/main/codex/codex-structured-item-translation.test.ts +++ b/src/main/codex/codex-structured-item-translation.test.ts @@ -11,9 +11,9 @@ import { codexJournalItem, codexMessageBlocks, codexStreamingJournalItem, - ProviderTurnMessageOrdinals, - MAX_PROVIDER_TURN_ORDINAL_BYTES, - MAX_PROVIDER_TURN_ORDINAL_ENTRIES, + CodexTurnOrdinals, + MAX_CODEX_TURN_ORDINAL_BYTES, + MAX_CODEX_TURN_ORDINAL_ENTRIES, isCodexMessageItemType, readCodexThreadItem, type CodexThreadItem @@ -59,7 +59,7 @@ const RESUMED_TURN: CodexThreadItem[] = [ ] function keysFor(items: CodexThreadItem[]): string[] { - const ordinals = new ProviderTurnMessageOrdinals() + const ordinals = new CodexTurnOrdinals() return items .filter((item) => isCodexMessageItemType(item.type)) .map((item) => @@ -71,15 +71,15 @@ function keysFor(items: CodexThreadItem[]): string[] { describe('codex turn ordinals', () => { it('bounds forgotten turn tombstones while retaining the recent window', () => { - const ordinals = new ProviderTurnMessageOrdinals() - const total = MAX_PROVIDER_TURN_ORDINAL_ENTRIES + 12 + const ordinals = new CodexTurnOrdinals() + const total = MAX_CODEX_TURN_ORDINAL_ENTRIES + 12 for (let index = 0; index < total; index += 1) { const turnId = `turn-${index}` expect(ordinals.ordinalFor('thread-many', turnId, 'item-0')).toBe(0) ordinals.forgetTurn('thread-many', turnId) } - expect(ordinals.forgottenTurnCount).toBe(MAX_PROVIDER_TURN_ORDINAL_ENTRIES) + expect(ordinals.forgottenTurnCount).toBe(MAX_CODEX_TURN_ORDINAL_ENTRIES) // The newest completed turn still keeps its counter for a late frame. expect(ordinals.ordinalFor('thread-many', `turn-${total - 1}`, 'item-late')).toBe(1) // The oldest turn was deterministically evicted and starts a fresh key. @@ -87,7 +87,7 @@ describe('codex turn ordinals', () => { }) it('releases a forgotten turn without ever reusing an ordinal it assigned', () => { - const ordinals = new ProviderTurnMessageOrdinals() + const ordinals = new CodexTurnOrdinals() expect(ordinals.ordinalFor('thread-1', 'turn-1', 'item-1')).toBe(0) expect(ordinals.ordinalFor('thread-1', 'turn-1', 'item-2')).toBe(1) @@ -102,12 +102,12 @@ describe('codex turn ordinals', () => { }) it('bounds aggregate provider identifier bytes retained by one active turn', () => { - const ordinals = new ProviderTurnMessageOrdinals() + const ordinals = new CodexTurnOrdinals() for (let index = 0; index < 3_000; index += 1) { ordinals.ordinalFor('thread', 'turn', `${index}:${'x'.repeat(512)}`) } - expect(ordinals.bytes).toBeLessThanOrEqual(MAX_PROVIDER_TURN_ORDINAL_BYTES) + expect(ordinals.bytes).toBeLessThanOrEqual(MAX_CODEX_TURN_ORDINAL_BYTES) }) }) @@ -117,7 +117,7 @@ describe('codex item identity', () => { }) it('numbers messages 0,1,2 on both sides — the projection skips the dropped command', () => { - const ordinals = new ProviderTurnMessageOrdinals() + const ordinals = new CodexTurnOrdinals() const live = LIVE_TURN.map((item) => codexItemIdentity({ threadId: THREAD_ID, turnId: TURN_ID, item, ordinals }) ) @@ -136,7 +136,7 @@ describe('codex item identity', () => { }) it('assigns an ordinal once and reuses it, so a delta and its completion upsert one row', () => { - const ordinals = new ProviderTurnMessageOrdinals() + const ordinals = new CodexTurnOrdinals() ordinals.ordinalFor(THREAD_ID, TURN_ID, 'item-0') expect(ordinals.ordinalFor(THREAD_ID, TURN_ID, 'item-1')).toBe(1) @@ -144,14 +144,14 @@ describe('codex item identity', () => { }) it('restarts numbering per turn', () => { - const ordinals = new ProviderTurnMessageOrdinals() + const ordinals = new CodexTurnOrdinals() ordinals.ordinalFor(THREAD_ID, TURN_ID, 'item-0') expect(ordinals.ordinalFor(THREAD_ID, 'turn-2', 'item-1')).toBe(0) }) it('keys a non-message item and a turnless message in the orca namespace', () => { - const ordinals = new ProviderTurnMessageOrdinals() + const ordinals = new CodexTurnOrdinals() const command = codexItemIdentity({ threadId: THREAD_ID, turnId: TURN_ID, diff --git a/src/main/codex/codex-structured-item-translation.ts b/src/main/codex/codex-structured-item-translation.ts index 3f4229cb6e54..25b821d90d9f 100644 --- a/src/main/codex/codex-structured-item-translation.ts +++ b/src/main/codex/codex-structured-item-translation.ts @@ -28,10 +28,10 @@ export { type CodexThreadItem } from './codex-thread-item-identity' export { - ProviderTurnMessageOrdinals, - MAX_PROVIDER_TURN_ORDINAL_BYTES, - MAX_PROVIDER_TURN_ORDINAL_ENTRIES -} from '../native-chat/agent-session-timeline/provider-turn-message-ordinals' + CodexTurnOrdinals, + MAX_CODEX_TURN_ORDINAL_BYTES, + MAX_CODEX_TURN_ORDINAL_ENTRIES +} from './codex-turn-ordinals' // Codex thread items → journal item bodies. diff --git a/src/main/codex/codex-structured-journal-items.ts b/src/main/codex/codex-structured-journal-items.ts index 22bb16c8ef08..ec6ac67df5d4 100644 --- a/src/main/codex/codex-structured-journal-items.ts +++ b/src/main/codex/codex-structured-journal-items.ts @@ -7,7 +7,7 @@ import { requiresTerminalSettlement } from '../native-chat/agent-session-journal import { codexItemIdentity, codexJournalItem, - ProviderTurnMessageOrdinals, + CodexTurnOrdinals, readCodexThreadItem, type CodexThreadItem } from './codex-structured-item-translation' @@ -35,7 +35,7 @@ import { readCodexDispatchEcho } from './codex-structured-dispatch-echo' import type { CodexRowAttribution } from './codex-subagent-linkage' export class CodexJournalItems { - readonly ordinals = new ProviderTurnMessageOrdinals() + readonly ordinals = new CodexTurnOrdinals() readonly activeItems = new Map() readonly streams private readonly identities = new Map() diff --git a/src/main/codex/codex-structured-journal-settlement.ts b/src/main/codex/codex-structured-journal-settlement.ts index e7d0a376eca8..131de9824aae 100644 --- a/src/main/codex/codex-structured-journal-settlement.ts +++ b/src/main/codex/codex-structured-journal-settlement.ts @@ -17,7 +17,7 @@ import { codexJournalItem, codexStreamingJournalItem, type CodexThreadItem, - type ProviderTurnMessageOrdinals + type CodexTurnOrdinals } from './codex-structured-item-translation' import type { CodexStructuredItemStreams } from './codex-structured-item-streams' import type { CodexHelperName } from './codex-collab-agent-item-translation' @@ -56,7 +56,7 @@ export function settleCodexJournalSession(input: { pendingPrompts: ReadonlyMap currentTurnIds: ReadonlyMap> primaryThreadId: string | null - ordinals: ProviderTurnMessageOrdinals + ordinals: CodexTurnOrdinals /** Terminal lifecycle for a turn the provider left running when it ended; null for a turn a * conversation command claimed, whose record the host settles. */ settledTurnLifecycle: (threadId: string, turnId: string) => AgentJournalTurnLifecycle | null diff --git a/src/main/codex/codex-structured-journal-translation-restore.ts b/src/main/codex/codex-structured-journal-translation-restore.ts index a3b4652f366e..9f7efc15ae4c 100644 --- a/src/main/codex/codex-structured-journal-translation-restore.ts +++ b/src/main/codex/codex-structured-journal-translation-restore.ts @@ -1,8 +1,5 @@ import type { AgentJournalTurnLifecycle } from '../../shared/agent-session-journal-types' -import { - readCodexThreadItem, - type ProviderTurnMessageOrdinals -} from './codex-structured-item-translation' +import { readCodexThreadItem, type CodexTurnOrdinals } from './codex-structured-item-translation' import type { CodexJournalCompactions } from './codex-structured-journal-compactions' import type { CodexJournalItems } from './codex-structured-journal-items' import { @@ -58,7 +55,7 @@ export function restoreCodexJournalThread(input: { threadId: string thread: Record currentTurnIds: Map> - ordinals: ProviderTurnMessageOrdinals + ordinals: CodexTurnOrdinals handleItem: (event: CodexHistoryItemEvent) => CodexJournalTranslationAdmission /** Absent when the caller has no session identity to key lifecycle rows by. */ restoreTurnLifecycle?: ( diff --git a/src/main/codex/codex-structured-journal-translation-streams.test.ts b/src/main/codex/codex-structured-journal-translation-streams.test.ts index 7bb88edc3a6f..54a2c017aa05 100644 --- a/src/main/codex/codex-structured-journal-translation-streams.test.ts +++ b/src/main/codex/codex-structured-journal-translation-streams.test.ts @@ -6,7 +6,7 @@ import type { import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key' import { projectStructuredItemsToNativeChat } from '../../shared/structured-agent-session-projection' import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' -import { ProviderTurnMessageOrdinals } from './codex-structured-item-translation' +import { CodexTurnOrdinals } from './codex-structured-item-translation' import { createCodexJournalTranslator, MAX_CODEX_GENERIC_BOOKKEEPING_ENTRIES, @@ -164,7 +164,7 @@ describe('codex journal translation', () => { }) it('releases a turn ordinal map when the turn completes', () => { - const spy = vi.spyOn(ProviderTurnMessageOrdinals.prototype, 'forgetTurn') + const spy = vi.spyOn(CodexTurnOrdinals.prototype, 'forgetTurn') try { const { translator } = translatorWith() translator.handle(TURN_STARTED) diff --git a/src/main/codex/codex-structured-journal-translation-turns.ts b/src/main/codex/codex-structured-journal-translation-turns.ts index dc955f70385c..33f138c6578d 100644 --- a/src/main/codex/codex-structured-journal-translation-turns.ts +++ b/src/main/codex/codex-structured-journal-translation-turns.ts @@ -8,7 +8,7 @@ import { } from '../../shared/agent-session-journal-types' import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key' import { agentJournalTurnBody } from '../../shared/agent-session-turn-record' -import { CODEX_USER_MESSAGE_ORDINAL } from './codex-thread-item-identity' +import { CODEX_USER_MESSAGE_ORDINAL } from './codex-turn-ordinals' import type { StructuredAgentSessionEventSink, StructuredAgentSessionSinkAdmission diff --git a/src/main/codex/codex-thread-item-identity.ts b/src/main/codex/codex-thread-item-identity.ts index 663b497f7731..0488e5c00e98 100644 --- a/src/main/codex/codex-thread-item-identity.ts +++ b/src/main/codex/codex-thread-item-identity.ts @@ -1,5 +1,5 @@ import type { AgentJournalItemIdentity } from '../../shared/agent-session-journal-types' -import type { ProviderTurnMessageOrdinals } from '../native-chat/agent-session-timeline/provider-turn-message-ordinals' +import type { CodexTurnOrdinals } from './codex-turn-ordinals' // Codex thread items → durable journal identities. // @@ -15,10 +15,6 @@ import type { ProviderTurnMessageOrdinals } from '../native-chat/agent-session-t // including ones this build does not model — is skipped identically on both // sides, which is what makes the key survive a Codex release that adds one. -/** Codex records the user message first in a turn, so a restored submission is - * ordinal 0 of `(threadId, turnId)`. */ -export const CODEX_USER_MESSAGE_ORDINAL = 0 - /** Only these carry a durable `(threadId, turnId, ordinal)` identity. */ const CODEX_MESSAGE_ITEM_TYPES = new Set(['userMessage', 'agentMessage']) @@ -54,7 +50,7 @@ export function codexItemIdentity(input: { threadId: string turnId: string | null item: CodexThreadItem - ordinals: ProviderTurnMessageOrdinals + ordinals: CodexTurnOrdinals }): AgentJournalItemIdentity { const { item, turnId } = input if (turnId && isCodexMessageItemType(item.type)) { diff --git a/src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.test.ts b/src/main/codex/codex-turn-ordinals.test.ts similarity index 64% rename from src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.test.ts rename to src/main/codex/codex-turn-ordinals.test.ts index b070b449f264..5ae12011a855 100644 --- a/src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.test.ts +++ b/src/main/codex/codex-turn-ordinals.test.ts @@ -1,17 +1,16 @@ import { expect, it } from 'vitest' import { - ProviderTurnMessageOrdinals, - MAX_PROVIDER_TURN_ORDINAL_BYTES, - MAX_PROVIDER_TURN_ORDINAL_ENTRIES -} from './provider-turn-message-ordinals' + CodexTurnOrdinals, + MAX_CODEX_TURN_ORDINAL_BYTES, + MAX_CODEX_TURN_ORDINAL_ENTRIES +} from './codex-turn-ordinals' it('does not rescan the forgotten window for each new streamed item', () => { - const ordinals = new ProviderTurnMessageOrdinals() - for (let index = 0; index < MAX_PROVIDER_TURN_ORDINAL_ENTRIES; index += 1) { + const ordinals = new CodexTurnOrdinals() + for (let index = 0; index < MAX_CODEX_TURN_ORDINAL_ENTRIES; index += 1) { ordinals.ordinalFor('thread', String(index), 'item') ordinals.forgetTurn('thread', String(index)) } - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this test counts reads of the class's private turn map, whose entries carry `active`. const turns = (ordinals as unknown as { turns: Map }).turns let reads = 0 for (const turn of turns.values()) { @@ -30,30 +29,30 @@ it('does not rescan the forgotten window for each new streamed item', () => { expect(ordinals.ordinalFor('thread', 'live', String(index))).toBe(index) } expect(reads).toBe(0) - expect(ordinals.forgottenTurnCount).toBe(MAX_PROVIDER_TURN_ORDINAL_ENTRIES) + expect(ordinals.forgottenTurnCount).toBe(MAX_CODEX_TURN_ORDINAL_ENTRIES) }) it('retains ordinal continuity on late reactivation and evicts oldest forgotten turns', () => { - const ordinals = new ProviderTurnMessageOrdinals() + const ordinals = new CodexTurnOrdinals() expect(ordinals.ordinalFor('t', 'first', 'a')).toBe(0) ordinals.forgetTurn('t', 'first') expect(ordinals.ordinalFor('t', 'first', 'b')).toBe(1) expect(ordinals.forgottenTurnCount).toBe(0) ordinals.forgetTurn('t', 'first') - for (let index = 0; index < MAX_PROVIDER_TURN_ORDINAL_ENTRIES; index += 1) { + for (let index = 0; index < MAX_CODEX_TURN_ORDINAL_ENTRIES; index += 1) { ordinals.ordinalFor('t', String(index), 'a') ordinals.forgetTurn('t', String(index)) } - expect(ordinals.forgottenTurnCount).toBe(MAX_PROVIDER_TURN_ORDINAL_ENTRIES) + expect(ordinals.forgottenTurnCount).toBe(MAX_CODEX_TURN_ORDINAL_ENTRIES) expect(ordinals.ordinalFor('t', 'first', 'c')).toBe(0) }) it('keeps byte eviction bounded for active and forgotten turns', () => { - const ordinals = new ProviderTurnMessageOrdinals() + const ordinals = new CodexTurnOrdinals() for (let index = 0; index < 4000; index += 1) { ordinals.ordinalFor('thread', 'live', `${index}-${'x'.repeat(240)}`) } - expect(ordinals.bytes).toBeLessThanOrEqual(MAX_PROVIDER_TURN_ORDINAL_BYTES) + expect(ordinals.bytes).toBeLessThanOrEqual(MAX_CODEX_TURN_ORDINAL_BYTES) ordinals.forgetTurn('thread', 'live') expect(ordinals.bytes).toBeLessThan(100) expect(ordinals.forgottenTurnCount).toBe(1) diff --git a/src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.ts b/src/main/codex/codex-turn-ordinals.ts similarity index 88% rename from src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.ts rename to src/main/codex/codex-turn-ordinals.ts index 9405ff9281ef..7e21b900d965 100644 --- a/src/main/native-chat/agent-session-timeline/provider-turn-message-ordinals.ts +++ b/src/main/codex/codex-turn-ordinals.ts @@ -1,11 +1,18 @@ -import { boundPayload, digestPayload } from '../agent-session-journal/journal-payload-bounds' +import { + boundPayload, + digestPayload +} from '../native-chat/agent-session-journal/journal-payload-bounds' + +/** Codex records the user message first in a turn, so a restored submission is + * ordinal 0 of `(threadId, turnId)`. */ +export const CODEX_USER_MESSAGE_ORDINAL = 0 /** Maximum forgotten turn keys retained for late-frame reconciliation. */ -export const MAX_PROVIDER_TURN_ORDINAL_ENTRIES = 256 -export const MAX_PROVIDER_TURN_ORDINAL_BYTES = 512 * 1024 +export const MAX_CODEX_TURN_ORDINAL_ENTRIES = 256 +export const MAX_CODEX_TURN_ORDINAL_BYTES = 512 * 1024 /** Assigns stable message ordinals while retaining a bounded late-frame window. */ -export class ProviderTurnMessageOrdinals { +export class CodexTurnOrdinals { private readonly turns = new Map< string, { assigned: Map; next: number; active: boolean } @@ -39,7 +46,7 @@ export class ProviderTurnMessageOrdinals { } private trimForgotten(): void { - while (this.forgottenTurnCount > MAX_PROVIDER_TURN_ORDINAL_ENTRIES) { + while (this.forgottenTurnCount > MAX_CODEX_TURN_ORDINAL_ENTRIES) { const oldest = this.forgottenTurns.values().next().value if (!oldest) { break @@ -60,7 +67,7 @@ export class ProviderTurnMessageOrdinals { private trimBytes(currentTurnKey: string): void { this.trimForgotten() - while (this.retainedBytes > MAX_PROVIDER_TURN_ORDINAL_BYTES) { + while (this.retainedBytes > MAX_CODEX_TURN_ORDINAL_BYTES) { const forgotten = this.forgottenTurns.values().next().value const oldest = forgotten ?? this.turns.keys().next().value if (typeof oldest !== 'string') { diff --git a/src/main/native-chat/agent-session-journal/journal-item-fold.ts b/src/main/native-chat/agent-session-journal/journal-item-fold.ts index aa1a02e4a70a..10aee4a99c41 100644 --- a/src/main/native-chat/agent-session-journal/journal-item-fold.ts +++ b/src/main/native-chat/agent-session-journal/journal-item-fold.ts @@ -39,9 +39,6 @@ export function upsertJournalItem( return } state.derivedTurnScope.observe(itemId, isRootAgentJournalItem(next), existing?.body, next.body) - if (next.providerItemRef !== undefined && existing?.providerItemRef === undefined) { - state.providerItemRefs.set(next.providerItemRef, itemId) - } if (!existing) { state.items.set(itemId, next) state.itemFences.set(itemId, fence) @@ -68,9 +65,7 @@ export function upsertJournalItem( sequence: existing.sequence, ...(existing.sequenceIndex !== undefined ? { sequenceIndex: existing.sequenceIndex } : {}), observedAt: existing.observedAt, - turnScope: existing.turnScope ?? next.turnScope, - // The creating write's, as the scope is: a revision never re-points the row at another item. - ...(existing.providerItemRef === undefined ? {} : { providerItemRef: existing.providerItemRef }) + turnScope: existing.turnScope ?? next.turnScope }) state.tombstones.delete(itemId) } @@ -97,9 +92,6 @@ export function removeJournalItem( ) } state.tombstones.set(itemId, revision) - if (existing?.providerItemRef !== undefined) { - state.providerItemRefs.delete(existing.providerItemRef) - } state.items.delete(itemId) state.itemFences.delete(itemId) } diff --git a/src/main/native-chat/agent-session-journal/journal-provider-item-ref.test.ts b/src/main/native-chat/agent-session-journal/journal-provider-item-ref.test.ts deleted file mode 100644 index d80d5a539965..000000000000 --- a/src/main/native-chat/agent-session-journal/journal-provider-item-ref.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, describe, expect, it } from 'vitest' -import type { AgentJournalItemIdentity } from '../../../shared/agent-session-journal-types' -import { createTrackedJournalOpener } from './journal-host-database-test-support' - -const journals = createTrackedJournalOpener() -const roots: string[] = [] - -afterEach(async () => { - await journals.closeAll() - for (const root of roots.splice(0)) { - await rm(root, { recursive: true, force: true }) - } -}) - -async function open(root?: string) { - const directory = root ?? (await mkdtemp(join(tmpdir(), 'orca-provider-item-ref-'))) - if (!root) { - roots.push(directory) - } - const journal = await journals.open({ - identity: { - sessionId: 'session-ref', - workspaceId: 'workspace-1', - hostId: 'local', - agent: 'codex', - providerHandle: { kind: 'codex', threadId: 'root' } - }, - stateDirectory: directory, - now: () => 1_000 - }) - return { journal, directory } -} - -const row: AgentJournalItemIdentity = { - provider: 'codex', - threadId: 'root', - turnId: 't1', - ordinal: 0 -} -const text = (value: string) => ({ - kind: 'message' as const, - role: 'assistant' as const, - blocks: [{ type: 'text' as const, text: value }] -}) -const scope = { kind: 'turn', turnItemId: 'turn:t1' } as const - -describe('a row’s provider item reference', () => { - it('is the creating write’s, kept by revisions and indexed for its producer', async () => { - const { journal } = await open() - await journal.appendItem(row, text('a'), { fence: 1, turnScope: scope, providerItemRef: 'm0' }) - await journal.appendItem(row, text('ab'), { fence: 1, turnScope: scope, providerItemRef: 'm9' }) - await journal.appendItem(row, text('abc'), { fence: 1, turnScope: scope }) - - expect(journal.item('codex:root:t1:0')).toMatchObject({ - providerItemRef: 'm0', - body: text('abc') - }) - expect(journal.itemIdForProviderItemRef('m0')).toBe('codex:root:t1:0') - expect(journal.itemIdForProviderItemRef('m9')).toBeNull() - }) - - it('survives a reopen, and leaves the index with its row', async () => { - const { journal, directory } = await open() - await journal.appendItem(row, text('a'), { fence: 1, turnScope: scope, providerItemRef: 'm0' }) - await journals.closeAll() - - const { journal: reopened } = await open(directory) - expect(reopened.itemIdForProviderItemRef('m0')).toBe('codex:root:t1:0') - await reopened.appendTombstone(row, { fence: 1 }) - expect(reopened.itemIdForProviderItemRef('m0')).toBeNull() - }) -}) diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index c4e892de8038..e2d1b1ef3ca3 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -56,8 +56,6 @@ export type JournalReducerState = { /** Provider item id → the submission slot that adopted it. Stops an accepted * echo from appending a second copy of the user's own message. */ aliases: Map - /** `providerItemRef` → the row that carries it, so a producer finds that row again in O(1). */ - providerItemRefs: Map appliedSettlementIds: Set /** Scope for rows stored without one; rebuilt by replay, never persisted. */ derivedTurnScope: JournalDerivedTurnScope @@ -82,7 +80,6 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou submissions: new Map(), receipts: new Map(), aliases: new Map(), - providerItemRefs: new Map(), appliedSettlementIds: new Set(), derivedTurnScope: new JournalDerivedTurnScope(), latestPersonTurnSequence: 0, diff --git a/src/main/native-chat/agent-session-journal/journal-render-item.ts b/src/main/native-chat/agent-session-journal/journal-render-item.ts index 76a7abf054ec..fd58b2c25e4e 100644 --- a/src/main/native-chat/agent-session-journal/journal-render-item.ts +++ b/src/main/native-chat/agent-session-journal/journal-render-item.ts @@ -35,7 +35,6 @@ export function journalRenderItem( ...(row.recovered ? { recoveredAt: row.ts } : {}), ...(row.recovered ? { recovered: row.recovered } : {}), turnScope, - ...(row.providerItemRef === undefined ? {} : { providerItemRef: row.providerItemRef }), ...agentJournalLinkageFields(producer) } } diff --git a/src/main/native-chat/agent-session-journal/journal-row-builders.ts b/src/main/native-chat/agent-session-journal/journal-row-builders.ts index 16bf1960e400..0865df3a41fa 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-builders.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-builders.ts @@ -49,8 +49,7 @@ export function journalItemRowBuilder( ts: options.observedAt ?? ts, recovered: options.recovered, linkage: options, - turnScope: options.turnScope, - ...(options.providerItemRef === undefined ? {} : { providerItemRef: options.providerItemRef }) + turnScope: options.turnScope }) } @@ -250,7 +249,6 @@ export function buildJournalItemRow(input: { recovered?: true linkage?: AgentJournalProducerLinkage turnScope: AgentJournalTurnScope - providerItemRef?: string }): JournalItemRow { const itemId = agentJournalItemKey(input.identity) const resolved = input.state.aliases.get(itemId) ?? itemId @@ -270,7 +268,6 @@ export function buildJournalItemRow(input: { ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts, [body]), ...(input.recovered ? { recovered: input.recovered } : {}), turnScope: input.turnScope, - ...(input.providerItemRef === undefined ? {} : { providerItemRef: input.providerItemRef }), ...agentJournalLinkageFields(input.linkage) } } diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.ts index e8d1746ceb8d..9276b1a8c05f 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.ts @@ -20,7 +20,6 @@ import { type AgentJournalItemBody, type AgentJournalMessageItem, type AgentJournalProducerLinkage, - type AgentJournalProviderItemRef, type AgentJournalTurnScope, type AgentSessionProviderHandle } from '../../../shared/agent-session-journal-types' @@ -52,9 +51,6 @@ type JournalRowBase = AgentJournalProducerLinkage & { * for the reason linkage does. Absent on rows from hosts that predate it: the reducer * derives one for them on read. */ turnScope?: AgentJournalTurnScope - /** On an item row: the provider's reference for the item it creates. Rides the base for the - * reason linkage does. */ - providerItemRef?: AgentJournalProviderItemRef } /** First row of every epoch: binds the epoch to a provider handle and records why it opened. */ @@ -253,13 +249,7 @@ export function parseJournalRow(line: string): JournalRowParse { * survives is a real one: the reader scopes on PRESENCE, so `''` or a * non-string left in place would hide the row from its own author for good. */ function dropUnusableProducerLinkage(record: Record): void { - for (const field of [ - 'agentId', - 'parentAgentId', - 'providerParentRef', - 'producerKind', - 'providerItemRef' - ]) { + for (const field of ['agentId', 'parentAgentId', 'providerParentRef', 'producerKind']) { const value = record[field] if (value !== undefined && (typeof value !== 'string' || value.length === 0)) { delete record[field] diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index b5a960360368..a85b3adc3172 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -62,10 +62,7 @@ export type JournalItemLinkageVisitor = ( itemId: string, sequence: number, body: AgentJournalItemBody, - attribution: AgentJournalProducerLinkage & { - turnScope?: AgentJournalTurnScope - providerItemRef?: string - } + attribution: AgentJournalProducerLinkage & { turnScope?: AgentJournalTurnScope } ) => void export type JournalLifecycleBatchInput = { diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 19a8b61a62cd..d12257447cfd 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -230,8 +230,6 @@ export class AgentSessionJournal { /** One reduced item with its attribution, for a writer that needs the turn a row joined. */ item = (itemId: string): AgentJournalRenderItem | null => this.state.items.get(itemId) ?? null - /** The row that carries this `providerItemRef`, if any. */ - itemIdForProviderItemRef = (ref: string) => this.state.providerItemRefs.get(ref) ?? null /** Visits reduced items with the producer that wrote each, for a producer re-deriving what an * earlier run of this session left. */ diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-identity.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-identity.ts deleted file mode 100644 index 77286d029055..000000000000 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-identity.ts +++ /dev/null @@ -1,138 +0,0 @@ -// How the assembler's addresses become persisted journal identities. -// -// The assembler decides WHICH rows are the same row: it resolves every provider join to an -// address — the provider session's namespace, the key, the thread and turn the item belongs to, -// the item's class, and its place among its turn's messages — or mints a key when the provider -// names nothing. A scheme only spells an address as an `AgentJournalItemIdentity`. It must be pure, -// and it is the one place a provider's persisted identity shape lives, so moving a provider to a -// new identity arm is a change here. - -import type { - AgentJournalItemBody, - AgentJournalItemIdentity, - AgentType -} from '../../../shared/agent-session-journal-types' -import { boundPayload, digestPayload } from '../agent-session-journal/journal-payload-bounds' - -/** A key the provider vouched for, or one the assembler minted because it named nothing. - * A minted value is unique per acquisition, so it never needs a namespace. */ -export type ProviderTimelineKey = { source: 'provider' | 'minted'; value: string } - -export type ProviderTimelineTurnAddress = { - /** The provider session whose ids the key belongs to. */ - namespace: string - key: ProviderTimelineKey -} - -/** Which family an item belongs to; families never share a key space. Streamed text and full - * item snapshots are one family, so a provider-named message is one row however it arrives. */ -export type ProviderTimelineItemFamily = 'item' | 'request' | 'frame' - -/** What kind of thing an item is, derived from what it carries: `message` is user or assistant - * prose; every other class is its row kind (`reasoning` text included). */ -export type ProviderTimelineItemClass = - | 'message' - | 'reasoning' - | Exclude - | 'frame' - -export type ProviderTimelineItemAddress = { - namespace: string - family: ProviderTimelineItemFamily - key: ProviderTimelineKey - /** The provider thread it came from; null when the provider named none. */ - thread: string | null - /** The provider turn it belongs to; null for a row outside any turn. */ - turn: ProviderTimelineKey | null - itemClass: ProviderTimelineItemClass - /** Its place among its (thread, turn)'s `message` items, by first write; null for any other, and - * for every item of a scheme without `ordinalMessages`. */ - messageOrdinal: number | null - /** 1 for the first request under a key; a key reused after it settled is the next one. */ - incarnation: number -} - -export type ProviderTimelineMessageSlot = { place: string; ordinal: number } - -export type ProviderTimelineIdentityScheme = { - /** Messages inside a turn are keyed by their place among the turn's messages, not by their key. - * Such an identity must spell only (thread, turn, ordinal), and its row carries the key as - * `providerItemRef`, since nothing else could find it again. */ - ordinalMessages: boolean - /** Reads an ordinal message identity back: its (thread, turn) place, spelled by the scheme, and - * its ordinal; null for any other identity. Required with `ordinalMessages`. */ - messageSlot?(identity: AgentJournalItemIdentity): ProviderTimelineMessageSlot | null - turn(address: ProviderTimelineTurnAddress): AgentJournalItemIdentity - /** The id the turn row carries and a client's Stop names. */ - turnId(address: ProviderTimelineTurnAddress): string - /** The user item a turn names when no send of Orca's opened it; absent ⇒ the turn row itself. */ - turnOpener?(address: ProviderTimelineTurnAddress): string - /** With a null `messageOrdinal` it must spell only (namespace, family, key, thread, incarnation), - * so a row is found again from its key alone. */ - item(address: ProviderTimelineItemAddress): AgentJournalItemIdentity -} - -export function providerTimelineItemClass(body: AgentJournalItemBody): ProviderTimelineItemClass { - if (body.kind !== 'message') { - return body.kind - } - return body.role === 'user' || body.role === 'assistant' ? 'message' : 'reasoning' -} - -const MAX_KEY_PART_BYTES = 256 - -/** A provider id as a bounded identity part: escaped so `:` cannot forge another part, and - * digest-suffixed when long so two long ids never share a prefix-only spelling. */ -export function providerTimelineKeyPart(value: string): string { - const encoded = encodeURIComponent(value) - if (Buffer.byteLength(encoded, 'utf8') <= MAX_KEY_PART_BYTES) { - return encoded - } - const suffix = `#${digestPayload(value).slice(0, 24)}` - const head = boundPayload(encoded, { - inlineHeadBytes: MAX_KEY_PART_BYTES - Buffer.byteLength(suffix, 'utf8') - }).head - return `${head}${suffix}` -} - -/** A key as one bounded string inside its namespace (and thread, for a per-thread key). */ -export function spellProviderTimelineKey( - namespace: string, - key: ProviderTimelineKey, - thread: string | null = null -): string { - // A provider's item ids are its own per thread, so a subagent thread's ids are spelled apart. - const scope = thread === null ? '' : `${providerTimelineKeyPart(thread)}/` - return key.source === 'provider' - ? `p:${providerTimelineKeyPart(namespace)}:${scope}${providerTimelineKeyPart(key.value)}` - : `m:${providerTimelineKeyPart(key.value)}` -} - -/** - * The scheme for a provider with no identity arm of its own: the existing `legacy` arm, so no row - * shape changes. Turn rows keep the `turn-lifecycle:` record prefix the other lanes write. - * Provider keys are spelled inside their namespace, and apart from minted ones. - */ -export function createLegacyProviderTimelineIdentityScheme(input: { - agent: AgentType - sessionId: string -}): ProviderTimelineIdentityScheme { - const identity = (recordId: string): AgentJournalItemIdentity => ({ - provider: 'legacy', - agent: input.agent, - sessionId: input.sessionId, - recordId - }) - return { - ordinalMessages: false, - turn: (address) => - identity(`turn-lifecycle:${spellProviderTimelineKey(address.namespace, address.key)}`), - turnId: (address) => spellProviderTimelineKey(address.namespace, address.key), - item: (address) => - identity( - `${address.family}:${spellProviderTimelineKey(address.namespace, address.key, address.family === 'request' ? null : address.thread)}${ - address.incarnation > 1 ? `#${address.incarnation}` : '' - }` - ) - } -} diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts deleted file mode 100644 index 6a3f1e8d9d3f..000000000000 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-joins.test.ts +++ /dev/null @@ -1,231 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, describe, expect, it } from 'vitest' -import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' -import { - AGENT_JOURNAL_THREAD_SCOPE, - type AgentJournalApprovalItem, - type AgentJournalItemIdentity -} from '../../../shared/agent-session-journal-types' -import { createCodexProviderTimelineIdentityScheme } from '../../codex/codex-provider-timeline-identity' -import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { createLegacyProviderTimelineIdentityScheme } from './provider-timeline-identity' -import { ProviderTimelineJoins, type ProviderTimelineItemJoin } from './provider-timeline-joins' - -const journals = createTrackedJournalOpener() -const roots: string[] = [] - -afterEach(async () => { - await journals.closeAll() - for (const root of roots.splice(0)) { - await rm(root, { recursive: true, force: true }) - } -}) - -async function openJournal(): Promise { - const root = await mkdtemp(join(tmpdir(), 'orca-provider-timeline-joins-')) - roots.push(root) - return journals.open({ - identity: { - sessionId: 'session-joins', - workspaceId: 'workspace-1', - hostId: 'local', - agent: 'codex', - providerHandle: { kind: 'codex', threadId: 'root' } - }, - stateDirectory: root, - now: () => 1_000 - }) -} - -const codexJoins = () => - new ProviderTimelineJoins({ - scheme: createCodexProviderTimelineIdentityScheme({ - sessionId: 'session-joins', - primaryThreadId: () => 'root' - }), - generation: 'gen-1', - namespace: 'ns' - }) - -const message = (id: string): ProviderTimelineItemJoin => ({ - family: 'item', - key: { source: 'provider', value: id }, - thread: 'root' -}) - -const inTurn = (turn: string) => ({ - thread: 'root', - turn: { source: 'provider', value: turn } as const, - scope: { kind: 'turn', turnItemId: `turn:${turn}` } as const -}) - -const codexSlot = (turn: string, ordinal: number, threadId = 'root'): AgentJournalItemIdentity => ({ - provider: 'codex', - threadId, - turnId: turn, - ordinal -}) - -async function write( - journal: AgentSessionJournal, - row: { identity: AgentJournalItemIdentity; ref?: string }, - text: string -) { - await journal.appendItem( - row.identity, - { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text }] }, - { - fence: 1, - turnScope: { kind: 'turn', turnItemId: 'turn:t1' }, - ...(row.ref === undefined ? {} : { providerItemRef: row.ref }) - } - ) -} - -describe('provider timeline joins', () => { - it('finds an ordinal message again from the reference its row carries, after a restart', async () => { - const journal = await openJournal() - const before = codexJoins() - const m0 = before.place(message('m0'), 'message', inTurn('t1'), journal) - await write(journal, m0, 'first') - const m1 = before.place(message('m1'), 'message', inTurn('t1'), journal) - await write(journal, m1, 'second') - expect([m0.itemId, m1.itemId]).toEqual(['codex:root:t1:0', 'codex:root:t1:1']) - - const after = codexJoins() - expect(after.find(message('m1'), journal)).toMatchObject({ - itemId: 'codex:root:t1:1', - scope: { kind: 'turn', turnItemId: 'turn:t1' } - }) - // A new message takes the next free place, past every row the turn holds. - expect(after.place(message('m2'), 'message', inTurn('t1'), journal).itemId).toBe( - 'codex:root:t1:2' - ) - }) - - it('skips a place an echoed send holds through its submission alias', async () => { - const journal = await openJournal() - await journal.appendSubmission({ - clientMessageId: 'send-1', - payloadFingerprint: 'fp', - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }, - fence: 1 - }) - await journal.resolveDispatch({ - clientMessageId: 'send-1', - state: 'accepted', - providerIdentity: { provider: 'codex', threadId: 'root', turnId: 't1', ordinal: 0 }, - fence: 1 - }) - expect(codexJoins().place(message('reply'), 'message', inTurn('t1'), journal).itemId).toBe( - 'codex:root:t1:1' - ) - }) - - it('continues past the highest ordinal the journal holds, never into a gap below it', async () => { - const journal = await openJournal() - await write(journal, { identity: codexSlot('t1', 2), ref: 'item:old' }, 'old') - // Another turn and another thread keep their own places. - await write(journal, { identity: codexSlot('t2', 7) }, 'other turn') - await write(journal, { identity: codexSlot('t1', 9, 'sub') }, 'subagent') - const joins = codexJoins() - expect(joins.place(message('new'), 'message', inTurn('t1'), journal).itemId).toBe( - 'codex:root:t1:3' - ) - expect(joins.place(message('next'), 'message', inTurn('t1'), journal).itemId).toBe( - 'codex:root:t1:4' - ) - }) - - it('continues past an echoed send above the highest row', async () => { - const journal = await openJournal() - await write(journal, { identity: codexSlot('t1', 0) }, 'first') - await journal.appendSubmission({ - clientMessageId: 'send-1', - payloadFingerprint: 'fp', - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }, - fence: 1 - }) - await journal.resolveDispatch({ - clientMessageId: 'send-1', - state: 'accepted', - providerIdentity: codexSlot('t1', 4), - fence: 1 - }) - expect(codexJoins().place(message('reply'), 'message', inTurn('t1'), journal).itemId).toBe( - 'codex:root:t1:5' - ) - }) - - it('forgets what it read from an epoch the journal replaced', async () => { - const journal = await openJournal() - const joins = codexJoins() - const m0 = joins.place(message('m0'), 'message', inTurn('t1'), journal) - await write(journal, m0, 'first') - expect(joins.find(message('m0'), journal)?.itemId).toBe('codex:root:t1:0') - await journal.replaceEpochItems('handle_forked', 1, []) - expect(joins.find(message('m0'), journal)).toBeNull() - expect(joins.place(message('m1'), 'message', inTurn('t1'), journal).itemId).toBe( - 'codex:root:t1:0' - ) - }) - - it('finds a row whose identity spells its key without any reference', async () => { - const journal = await openJournal() - const scheme = createLegacyProviderTimelineIdentityScheme({ - agent: 'grok', - sessionId: 'session-joins' - }) - const joins = () => new ProviderTimelineJoins({ scheme, generation: 'g', namespace: 'ns' }) - const tool = joins().place(message('tool'), 'tool-call', inTurn('t1'), journal) - expect(tool.ref).toBeUndefined() - await write(journal, tool, 'x') - expect(joins().find(message('tool'), journal)?.itemId).toBe(tool.itemId) - expect(joins().find({ ...message('tool'), thread: 'other' }, journal)).toBeNull() - }) - - it('reads the current request incarnation back from the rows', async () => { - const journal = await openJournal() - const scheme = createLegacyProviderTimelineIdentityScheme({ - agent: 'grok', - sessionId: 'session-joins' - }) - const joins = () => new ProviderTimelineJoins({ scheme, generation: 'g', namespace: 'ns' }) - const key = { source: 'provider', value: 'p1' } as const - const thread = { thread: null, turn: null, scope: AGENT_JOURNAL_THREAD_SCOPE } - const approval: AgentJournalApprovalItem = { - kind: 'approval', - title: 'Run?', - detail: null, - options: [], - resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } - } - const before = joins() - for (let incarnation = 1; incarnation <= 2; incarnation += 1) { - const row = before.nextRequest(key, 'approval', thread, journal) - expect(row.incarnation).toBe(incarnation) - await journal.appendItem(row.identity, approval, { - fence: 1, - turnScope: AGENT_JOURNAL_THREAD_SCOPE - }) - } - expect(joins().request(key, journal)).toMatchObject({ - incarnation: 2, - itemId: agentJournalItemKey( - scheme.item({ - namespace: 'ns', - family: 'request', - key, - thread: null, - turn: null, - itemClass: 'approval', - messageOrdinal: null, - incarnation: 2 - }) - ) - }) - }) -}) diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-joins.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-joins.ts deleted file mode 100644 index 7cb48073933d..000000000000 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-joins.ts +++ /dev/null @@ -1,306 +0,0 @@ -// Provider joins → journal rows: the assembler's canonical join index. -// -// Every row the assembler writes is found again through here, and every entry is a cache of a -// fact the journal holds. A row whose identity spells its provider key is found by recomputing -// that identity; a row whose identity does not (a message keyed by its place among its turn's -// messages) carries the key as `providerItemRef`, which the journal indexes. So a miss — after a -// restart or an eviction — recovers the original row; it never re-places it. Message ordinals and -// request incarnations are read back from the rows that hold them. Only transition resolvers call -// the allocating members, so allocation happens in journal order and a refused event allocates -// nothing. - -import { - agentJournalItemKey, - parseAgentJournalItemKey -} from '../../../shared/agent-session-journal-item-key' -import { - AGENT_JOURNAL_THREAD_SCOPE, - type AgentJournalItemIdentity, - type AgentJournalRenderItem, - type AgentJournalTurnScope -} from '../../../shared/agent-session-journal-types' -import { BoundedMap } from '../../../shared/bounded-map' -import type { StructuredAgentSessionTransitionJournal } from '../agent-session-wire/structured-agent-session-transition' -import { - spellProviderTimelineKey, - type ProviderTimelineIdentityScheme, - type ProviderTimelineItemAddress, - type ProviderTimelineItemClass, - type ProviderTimelineItemFamily, - type ProviderTimelineKey, - type ProviderTimelineTurnAddress -} from './provider-timeline-identity' -import { ProviderTimelineMessagePlaces } from './provider-timeline-message-places' - -const MAX_JOINED_ROWS = 1_024 -const MAX_JOINED_ROW_BYTES = 1024 * 1024 - -/** One provider item as the provider names it. Item keys are the provider's per thread. */ -export type ProviderTimelineItemJoin = { - family: ProviderTimelineItemFamily - key: ProviderTimelineKey - thread: string | null -} - -export type ProviderTimelineRow = { - identity: AgentJournalItemIdentity - itemId: string - scope: AgentJournalTurnScope - /** The join's reference, on a row whose identity does not spell its key. */ - ref?: string - /** Which request under its key, 1 for the first. */ - incarnation: number -} - -export type ProviderTimelineTurnRef = { - address: ProviderTimelineTurnAddress - identity: AgentJournalItemIdentity - itemId: string - turnId: string -} - -/** Where a new row goes: the provider thread and turn, and the journal turn scope. */ -export type ProviderTimelinePlacement = { - thread: string | null - turn: ProviderTimelineKey | null - scope: AgentJournalTurnScope -} - -type Journal = StructuredAgentSessionTransitionJournal | null - -function bytes(value: string): number { - return Buffer.byteLength(value, 'utf8') -} - -export class ProviderTimelineJoins { - private readonly rows = new BoundedMap({ - maxEntries: MAX_JOINED_ROWS, - maxBytes: MAX_JOINED_ROW_BYTES, - sizeOf: (row, key) => bytes(key) + 2 * bytes(row.itemId) + 64 - }) - private readonly places: ProviderTimelineMessagePlaces - /** The journal epoch every cache entry was read from; a replaced epoch holds other rows. */ - private epoch: string | null = null - private serial = 0 - - constructor( - private readonly deps: { - scheme: ProviderTimelineIdentityScheme - generation: string - namespace: string - } - ) { - this.places = new ProviderTimelineMessagePlaces({ - scheme: deps.scheme, - namespace: () => this.deps.namespace - }) - } - - get namespace(): string { - return this.deps.namespace - } - - /** A new provider session: its ids name new rows. Minted keys stay unique as they are. */ - reset(namespace: string): void { - this.deps.namespace = namespace - this.rows.clear() - this.places.clear() - } - - /** Drops every cache read from an epoch the journal has since replaced (a rewind, an import). */ - private sync(journal: Journal): void { - if (!journal || journal.epoch === this.epoch) { - return - } - if (this.epoch !== null) { - this.rows.clear() - this.places.clear() - } - this.epoch = journal.epoch - } - - /** A key unique to this acquisition, for something the provider names nothing. */ - mint(kind: string): ProviderTimelineKey { - this.serial += 1 - return { source: 'minted', value: `${this.deps.generation}:${kind}${this.serial}` } - } - - /** A turn's row. `namespace` defaults to the current one; a planner names the one it expects. */ - turn(key: ProviderTimelineKey, namespace = this.deps.namespace): ProviderTimelineTurnRef { - const address = { namespace, key } - const identity = this.deps.scheme.turn(address) - return { - address, - identity, - itemId: agentJournalItemKey(identity), - turnId: this.deps.scheme.turnId(address) - } - } - - /** The user item a turn names when no send of Orca's opened it. */ - turnOpener(turn: ProviderTimelineTurnRef): string { - return this.deps.scheme.turnOpener?.(turn.address) ?? turn.itemId - } - - /** The join as one bounded string: what rows carry as `providerItemRef` and caches key by. */ - reference(join: ProviderTimelineItemJoin, namespace = this.deps.namespace): string { - const thread = join.family === 'request' ? null : join.thread - return `${join.family}:${spellProviderTimelineKey(namespace, join.key, thread)}` - } - - /** The row the join already names, from memory or the fold; null when the journal holds none. */ - find( - join: ProviderTimelineItemJoin, - journal: Journal, - namespace = this.deps.namespace - ): ProviderTimelineRow | null { - this.sync(journal) - const ref = this.reference(join, namespace) - const cached = this.rows.get(ref) - // A planner ahead of a reset asks about a namespace whose rows this index does not spell yet. - if (cached || !journal || namespace !== this.deps.namespace) { - return cached ?? null - } - const itemId = - journal.itemIdForProviderItemRef(ref) ?? agentJournalItemKey(this.spelled(join, 1)) - const row = rowOf(journal.item(itemId), 1) - if (row) { - this.rows.set(ref, row) - } - return row - } - - /** A new row for a join no row names yet. Allocates its message ordinal: resolvers only. */ - place( - join: ProviderTimelineItemJoin, - itemClass: ProviderTimelineItemClass, - placement: ProviderTimelinePlacement, - journal: Journal - ): ProviderTimelineRow { - this.sync(journal) - const ordinal = - this.deps.scheme.ordinalMessages && itemClass === 'message' && placement.turn - ? this.places.next(placement.thread, placement.turn, journal) - : null - const identity = this.deps.scheme.item({ - ...this.address(join, 1), - turn: placement.turn, - itemClass, - messageOrdinal: ordinal - }) - const ref = this.reference(join) - const row: ProviderTimelineRow = { - identity, - itemId: agentJournalItemKey(identity), - scope: placement.scope, - ...(ordinal === null ? {} : { ref }), - incarnation: 1 - } - this.rows.set(ref, row) - return row - } - - /** An echoed send takes its message's place without a row (its send is the bubble). */ - reserveEcho( - join: ProviderTimelineItemJoin, - placement: ProviderTimelinePlacement, - journal: Journal - ) { - if (!this.deps.scheme.ordinalMessages || !placement.turn || this.find(join, journal)) { - return - } - const row = this.place(join, 'message', placement, journal) - this.places.reserve(row.itemId) - } - - /** The request under `key` the journal holds last: the highest incarnation with a row. */ - request( - key: ProviderTimelineKey, - journal: Journal, - namespace = this.deps.namespace - ): ProviderTimelineRow | null { - this.sync(journal) - const join = { family: 'request' as const, key, thread: null } - const ref = this.reference(join, namespace) - const cached = this.rows.get(ref) - if (cached || !journal || namespace !== this.deps.namespace) { - return cached ?? null - } - let current: ProviderTimelineRow | null = null - for (let incarnation = 1; ; incarnation += 1) { - const itemId = agentJournalItemKey(this.spelled(join, incarnation)) - // A scheme that spells no incarnation names one row for every request under the key. - const row = itemId === current?.itemId ? null : rowOf(journal.item(itemId), incarnation) - if (!row) { - break - } - current = row - } - if (current) { - this.rows.set(ref, current) - } - return current - } - - /** The next request under `key`: resolvers only. */ - nextRequest( - key: ProviderTimelineKey, - itemClass: ProviderTimelineItemClass, - placement: ProviderTimelinePlacement, - journal: Journal - ): ProviderTimelineRow { - const join = { family: 'request' as const, key, thread: null } - const incarnation = (this.request(key, journal)?.incarnation ?? 0) + 1 - const identity = this.deps.scheme.item({ - ...this.address(join, incarnation), - thread: placement.thread, - turn: placement.turn, - itemClass - }) - const row = { - identity, - itemId: agentJournalItemKey(identity), - scope: placement.scope, - incarnation - } - this.rows.set(this.reference(join), row) - return row - } - - private address( - join: ProviderTimelineItemJoin, - incarnation: number - ): ProviderTimelineItemAddress { - return { - namespace: this.deps.namespace, - family: join.family, - key: join.key, - thread: join.thread, - turn: null, - itemClass: 'status', - messageOrdinal: null, - incarnation - } - } - - private spelled(join: ProviderTimelineItemJoin, incarnation: number): AgentJournalItemIdentity { - return this.deps.scheme.item(this.address(join, incarnation)) - } -} - -function rowOf( - item: AgentJournalRenderItem | null, - incarnation: number -): ProviderTimelineRow | null { - const identity = item ? parseAgentJournalItemKey(item.itemId) : null - if (!item || !identity) { - return null - } - return { - identity, - itemId: item.itemId, - scope: item.turnScope ?? AGENT_JOURNAL_THREAD_SCOPE, - ...(item.providerItemRef === undefined ? {} : { ref: item.providerItemRef }), - incarnation - } -} diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-message-places.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-message-places.ts deleted file mode 100644 index e5350c39c93a..000000000000 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-message-places.ts +++ /dev/null @@ -1,122 +0,0 @@ -// Where a new message goes among its (thread, turn)'s messages, for a scheme that keys messages -// by that place. Read back from the journal on a miss, past every place a row or an echoed send -// holds, so a restart continues the sequence the journal already has. - -import { - agentJournalItemKey, - parseAgentJournalItemKey -} from '../../../shared/agent-session-journal-item-key' -import type { AgentJournalItemIdentity } from '../../../shared/agent-session-journal-types' -import { BoundedMap } from '../../../shared/bounded-map' -import type { StructuredAgentSessionTransitionJournal } from '../agent-session-wire/structured-agent-session-transition' -import { - providerTimelineKeyPart, - spellProviderTimelineKey, - type ProviderTimelineIdentityScheme, - type ProviderTimelineKey -} from './provider-timeline-identity' - -const MAX_TURN_ORDINALS = 256 -/** Echoed sends whose slot no row holds yet; a lane records each on its send soon after. */ -const MAX_RESERVED_ECHOES = 64 - -type Journal = StructuredAgentSessionTransitionJournal | null - -function bytes(value: string): number { - return Buffer.byteLength(value, 'utf8') -} - -export class ProviderTimelineMessagePlaces { - /** The next ordinal per place. */ - private readonly ordinals = new BoundedMap({ - maxEntries: MAX_TURN_ORDINALS, - maxBytes: MAX_TURN_ORDINALS * 1024, - sizeOf: (_next, place) => bytes(place) + 8 - }) - private readonly echoes = new BoundedMap({ - maxEntries: MAX_RESERVED_ECHOES, - maxBytes: MAX_RESERVED_ECHOES * 1024, - sizeOf: (_held, itemId) => bytes(itemId) + 8 - }) - - constructor( - private readonly deps: { - scheme: ProviderTimelineIdentityScheme - namespace: () => string - } - ) {} - - clear(): void { - this.ordinals.clear() - this.echoes.clear() - } - - /** Holds a place no row takes (an echoed send's). */ - reserve(itemId: string): void { - this.echoes.set(itemId, true) - } - - /** The next place in (thread, turn): allocating, so resolvers only. */ - next(thread: string | null, turn: ProviderTimelineKey, journal: Journal): number { - const place = `${providerTimelineKeyPart(thread ?? '')}\u001f${spellProviderTimelineKey( - this.deps.namespace(), - turn - )}` - let ordinal = this.ordinals.get(place) ?? this.highWater(thread, turn, journal) - // Another writer of the same turn may have taken places since this one cached its mark. - while (this.taken(agentJournalItemKey(this.slot(thread, turn, ordinal)), journal)) { - ordinal += 1 - } - this.ordinals.set(place, ordinal + 1) - return ordinal - } - - /** One past the highest ordinal the journal holds in the place, like the ordinal counter it - * continues: a gap below it (a removed or never-imported row) is never filled. */ - private highWater(thread: string | null, turn: ProviderTimelineKey, journal: Journal): number { - const read = this.deps.scheme.messageSlot - const place = read?.(this.slot(thread, turn, 0))?.place - if (!read || place === undefined || !journal) { - return 0 - } - let next = 0 - const consider = (itemId: string | null) => { - const identity = itemId === null ? null : parseAgentJournalItemKey(itemId) - const slot = identity ? read(identity) : null - if (slot?.place === place) { - next = Math.max(next, slot.ordinal + 1) - } - } - journal.visitItems((itemId) => consider(itemId)) - // An echoed send holds its place through its submission alias, with no row of its own. - for (const submission of journal.submissions()) { - consider(submission.providerItemId) - } - return next - } - - private slot( - thread: string | null, - turn: ProviderTimelineKey, - ordinal: number - ): AgentJournalItemIdentity { - return this.deps.scheme.item({ - namespace: this.deps.namespace(), - family: 'item', - key: turn, - thread, - turn, - itemClass: 'message', - messageOrdinal: ordinal, - incarnation: 1 - }) - } - - private taken(itemId: string, journal: Journal): boolean { - return ( - this.echoes.has(itemId) || - (journal !== null && - (journal.itemBody(itemId) !== null || journal.canonicalItemId(itemId) !== itemId)) - ) - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts index 5cdabdce713d..8e997bf8477f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts @@ -51,8 +51,6 @@ export type StructuredAgentSessionPublishOptions = Pick< /** An item write states which turn its row belongs to; the write that creates the row decides. */ export type StructuredAgentSessionItemAppendOptions = StructuredAgentSessionAppendOptions & { turnScope: AgentJournalTurnScope - /** The provider's reference for the item, on the write that creates its row. */ - providerItemRef?: string } export type StructuredAgentSessionLifecycleJournal = Pick< diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts index 57bb482f7c2b..e7ce667ed2b6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts @@ -16,7 +16,6 @@ export function structuredAgentSessionJournalAppendOptions( fence, ...(options.observedAt === undefined ? {} : { observedAt: options.observedAt }), turnScope: options.turnScope, - ...(options.providerItemRef === undefined ? {} : { providerItemRef: options.providerItemRef }), ...agentJournalLinkageFields(options) } } diff --git a/src/shared/agent-session-journal-types.ts b/src/shared/agent-session-journal-types.ts index 271053b1b214..e5f8125a5c0f 100644 --- a/src/shared/agent-session-journal-types.ts +++ b/src/shared/agent-session-journal-types.ts @@ -360,16 +360,8 @@ export const AGENT_JOURNAL_THREAD_SCOPE: AgentJournalTurnScope = { kind: 'thread /** Who produced a row and which turn it belongs to: what every item write states. */ export type AgentJournalRowAttribution = AgentJournalProducerLinkage & { turnScope: AgentJournalTurnScope - providerItemRef?: AgentJournalProviderItemRef } -/** The provider's own reference for the item a row is, as its producer joined it, on a row whose - * identity does not spell it (a Codex message keys by its turn ordinal, and Codex renumbers its - * item ids on resume). Set by the write that creates the row and kept by every revision, like the - * turn scope. Only its producer reads it back, to find the row again after a restart; no client - * depends on it. */ -export type AgentJournalProviderItemRef = string - /** Where the journal placed an item: the sequence of the row that created it, * then its place among that row's writes. The timeline's only ordering key. */ export type AgentJournalPosition = { @@ -395,7 +387,6 @@ export type AgentJournalRenderItem = AgentJournalProducerLinkage & { recoveredAt?: number /** Absent only from a host that predates it. */ turnScope?: AgentJournalTurnScope - providerItemRef?: AgentJournalProviderItemRef } // ─── Submissions ──────────────────────────────────────────────────────────── diff --git a/tests/e2e/cross-version-wire/agent-session-provider-item-ref-downgrade.unit.test.ts b/tests/e2e/cross-version-wire/agent-session-provider-item-ref-downgrade.unit.test.ts deleted file mode 100644 index ff0ecc5fed33..000000000000 --- a/tests/e2e/cross-version-wire/agent-session-provider-item-ref-downgrade.unit.test.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { expect, test } from 'vitest' -import type { AgentJournalItemIdentity } from '../../../src/shared/agent-session-journal-types' -import { createTrackedJournalOpener } from '../../../src/main/native-chat/agent-session-journal/journal-host-database-test-support' -import type { JournalRow } from '../../../src/main/native-chat/agent-session-journal/journal-row-schema' -import { importReleaseCheckoutModule, materializeReleaseCheckout } from './release-checkout' - -// A release that predates `providerItemRef` on journal rows and render items. -const BASELINE_REF = 'v1.4.218' - -const SESSION = 'session-provider-item-ref' - -function message(ordinal: number): AgentJournalItemIdentity { - return { provider: 'codex', threadId: 'thread-1', turnId: 'turn-1', ordinal } -} - -function releaseExport(module: Record, name: string): T { - const value = module[name] - if (value === undefined) { - throw new Error(`the pinned release exports no ${name}`) - } - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: an export the pinned release has; each caller names the shape it uses, and a changed one fails the test. - return value as T -} - -type OldRenderItem = { itemId: string; [field: string]: unknown } - -// Loads a real old build, including cold extraction and transforms. -test('an older host and client render a row carrying a provider item reference as if it had none', async () => { - const directory = mkdtempSync(join(tmpdir(), 'orca-provider-item-ref-downgrade-')) - const journals = createTrackedJournalOpener() - try { - // This build: the same message twice, the first carrying the reference. - const journal = await journals.open({ - identity: { - sessionId: SESSION, - workspaceId: 'ws-1', - hostId: 'host-1', - agent: 'codex', - providerHandle: { kind: 'codex', threadId: 'thread-1' } - }, - stateDirectory: directory, - now: () => 1_000 - }) - const body = { - kind: 'message' as const, - role: 'assistant' as const, - blocks: [{ type: 'text' as const, text: 'Hello' }] - } - const scope = { kind: 'thread' as const } - await journal.appendItem(message(0), body, { - fence: 1, - turnScope: scope, - providerItemRef: 'item:p:thread-1/item-1' - }) - await journal.appendItem(message(1), body, { fence: 1, turnScope: scope }) - const since = journal.readSince({ epoch: journal.epoch, sequence: 0 }) - if (!since.ok) { - throw new Error(`expected rows, got reset ${since.reset}`) - } - const rows: JournalRow[] = since.rows - const current = journal.snapshot().items - expect(current[0]?.providerItemRef).toBe('item:p:thread-1/item-1') - - const checkout = await materializeReleaseCheckout(BASELINE_REF) - const [reducer, schemas, projection] = await Promise.all( - [ - 'src/main/native-chat/agent-session-journal/journal-reducer.ts', - 'src/shared/agent-session-journal-schemas.ts', - 'src/shared/structured-agent-session-projection.ts' - ].map((path) => importReleaseCheckoutModule(checkout, path)) - ) - const createState = releaseExport<(sessionId: string, epoch: string) => unknown>( - reducer, - 'createJournalReducerState' - ) - const applyRow = releaseExport<(state: unknown, row: JournalRow) => void>( - reducer, - 'applyJournalRow' - ) - const render = releaseExport<(state: unknown) => { items: OldRenderItem[] }>( - reducer, - 'renderJournalState' - ) - const renderItemSchema = releaseExport<{ parse: (value: unknown) => OldRenderItem }>( - schemas, - 'AgentJournalRenderItemSchema' - ) - const project = releaseExport<(item: unknown) => unknown>( - projection, - 'projectStructuredItemToNativeChat' - ) - - // The older host, after a downgrade, folds the rows as it always did. - const state = createState(SESSION, journal.epoch) - for (const row of rows) { - applyRow(state, row) - } - const folded = render(state).items - expect(folded.map((item) => item.itemId)).toEqual([ - 'codex:thread-1:turn-1:0', - 'codex:thread-1:turn-1:1' - ]) - const [first, second] = folded - if (!first || !second) { - throw new Error('the older host folded both rows') - } - expect(first).not.toHaveProperty('providerItemRef') - const same = ({ itemId: _id, sequence: _seq, ...rest }: OldRenderItem) => rest - expect(same(first)).toEqual(same(second)) - - // The older client reads this host's render item without the field, and renders it the same. - const [withRef, withoutRef] = current.map((item) => renderItemSchema.parse(item)) - if (!withRef || !withoutRef) { - throw new Error('the older client read both items') - } - expect(withRef).not.toHaveProperty('providerItemRef') - expect(project(withRef)).toEqual( - project({ ...withoutRef, itemId: withRef.itemId, sequence: withRef.sequence }) - ) - } finally { - await journals.closeAll() - rmSync(directory, { recursive: true, force: true }) - } -}) From 7d048776c85f0ef254cd9f674e1c51844e0c0103 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:13:40 -0700 Subject: [PATCH 07/11] Write a resolved settlement in one transaction through enqueueRows A settlement too large for one row now commits all its rows or none, through the journal's existing all-or-nothing write, instead of a row-by-row writer. Every row is built before any commits, so a settlement naming one item twice is refused before anything is written. --- .../journal-lifecycle-batch-appender.ts | 37 ++++++++--- .../journal-row-writer.ts | 61 +++++++------------ .../journal-store-collaborators.ts | 1 - 3 files changed, 52 insertions(+), 47 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts index 9f1e0cce218f..cc300a7b7497 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts @@ -1,13 +1,16 @@ +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { JournalReducerState } from './journal-reducer' import { partitionJournalLifecycleMutations } from './journal-lifecycle-batch-partition' -import { journalLifecycleBatchRowBuilder } from './journal-row-builders' +import { + journalLifecycleBatchRowBuilder, + type JournalLifecycleMutationInput +} from './journal-row-builders' import type { JournalLifecycleBatchInput, JournalResolvedLifecycleBatchInput } from './journal-store-contracts' import type { JournalRow } from './journal-row-schema' -import type { JournalRowWriter } from './journal-row-writer' import { journalQueuedRejectionRowBuilders } from './journal-pending-submission-recovery' const SETTLEMENT_ALREADY_APPLIED = new Error('journal_settlement_already_applied') @@ -18,8 +21,9 @@ export class JournalLifecycleBatchAppender { state: () => JournalReducerState cursor: () => AgentJournalCursor enqueue: (build: (seq: number, ts: number) => JournalRow) => Promise - enqueueEach: JournalRowWriter['enqueueEach'] - enqueueRows: JournalRowWriter['enqueueRows'] + enqueueRows: ( + plan: () => readonly ((seq: number, ts: number) => JournalRow)[] + ) => Promise } ) {} @@ -75,11 +79,15 @@ export class JournalLifecycleBatchAppender { } /** Chooses the mutations at its turn in the queue; one too large for a row becomes consecutive - * rows that nothing else lands between, in the order resolved. */ + * rows, committed in one transaction, in the order resolved. */ appendResolved(input: JournalResolvedLifecycleBatchInput): Promise { return this.deps - .enqueueEach(() => - partitionJournalLifecycleMutations(input.settlementId, input.resolve()) + .enqueueRows(() => { + const mutations = input.resolve() + // Every chunk is built before any commits, so a second chunk naming the same item would + // reuse the first chunk's revision. + this.assertDistinctItems(mutations) + return partitionJournalLifecycleMutations(input.settlementId, mutations) .filter((chunk) => !this.wasApplied(chunk.settlementId)) .map((chunk) => journalLifecycleBatchRowBuilder( @@ -89,13 +97,26 @@ export class JournalLifecycleBatchAppender { input ) ) - ) + }) .then((rows) => { const last = rows.at(-1) return last ? { epoch: last.epoch, sequence: last.seq } : null }) } + private assertDistinctItems(mutations: readonly JournalLifecycleMutationInput[]): void { + const { aliases } = this.deps.state() + const seen = new Set() + for (const mutation of mutations) { + const itemId = agentJournalItemKey(mutation.identity) + const resolved = aliases.get(itemId) ?? itemId + if (seen.has(resolved)) { + throw new Error('journal_resolved_lifecycle_batch_names_item_twice') + } + seen.add(resolved) + } + } + private wasApplied(settlementId: string): boolean { return this.deps.state().appliedSettlementIds.has(settlementId) } diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.ts index 312635c60a77..dde1a66741d1 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.ts @@ -46,44 +46,29 @@ export class JournalRowWriter { hook?: JournalRowTransactionHook, receipt?: JournalOperationReceipt ): Promise { - return this.deps.serialize(() => this.write(build, hook, receipt)) - } - - /** Rows chosen at this write's turn in the queue and written back to back, so no other write - * lands between them; each commits alone and is built after the one before it is adopted, so a - * later row may revise an earlier one. `enqueueRows` is the all-or-nothing form. */ - enqueueEach( - resolve: () => readonly ((seq: number, ts: number) => JournalRow)[] - ): Promise { - return this.deps.serialize(() => resolve().map((build) => this.write(build))) - } - - private write( - build: (seq: number, ts: number) => JournalRow, - hook?: JournalRowTransactionHook, - receipt?: JournalOperationReceipt - ): JournalRow { - assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) - const row = build(this.deps.nextSequence(), this.deps.now()) - assertJournalFence(row.fence, this.deps.highestFence()) - try { - // One INSERT: the chat's epoch pointer moves only when the epoch does. - this.deps.database().transaction((db) => { - insertJournalRow(db, this.deps.sessionId, row) - hook?.(db, row) - receipt?.write(db) - this.runBookkeeping(db, row) - }) - } catch (error) { - this.deps.rolledBack?.() - throw error - } - // COMMIT landed, so the row is durable: adopt it before anything that can - // fail. Rejecting here instead would leave the next append reusing a - // sequence the table already holds. The ledger first: it cannot throw, the fold can. - receipt?.committed() - this.deps.commit(row) - return row + return this.deps.serialize(() => { + assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) + const row = build(this.deps.nextSequence(), this.deps.now()) + assertJournalFence(row.fence, this.deps.highestFence()) + try { + // One INSERT: the chat's epoch pointer moves only when the epoch does. + this.deps.database().transaction((db) => { + insertJournalRow(db, this.deps.sessionId, row) + hook?.(db, row) + receipt?.write(db) + this.runBookkeeping(db, row) + }) + } catch (error) { + this.deps.rolledBack?.() + throw error + } + // COMMIT landed, so the row is durable: adopt it before anything that can + // fail. Rejecting here instead would leave the next append reusing a + // sequence the table already holds. The ledger first: it cannot throw, the fold can. + receipt?.committed() + this.deps.commit(row) + return row + }) } /** Several rows in ONE transaction, in order, planned once the lane is this append's: none is diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 6a77b8001032..ba8796964692 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -125,7 +125,6 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal state: host.state, cursor: host.cursor, enqueue: host.enqueue, - enqueueEach: (resolve) => rowWriter.enqueueEach(resolve), enqueueRows: (plan) => rowWriter.enqueueRows(plan) }) } From e1c3ccdaa3c93c99c08d95764b979948f3d162b0 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:13:40 -0700 Subject: [PATCH 08/11] Drop the transition's landing report; keep the turn-row write fire-and-forget Nothing reads which steps of a transition wrote. A failed step fails the sink, leaving the steps before it written; the header says so, and tests cover it plus a settlement whose second row fails inside the transaction. writeAgentJournalTurnRow returns nothing again, as on main. --- .../agent-journal-turn-row-revision.ts | 9 +- ...tructured-agent-session-transition.test.ts | 119 +++++++++++++++--- .../structured-agent-session-transition.ts | 24 +--- 3 files changed, 108 insertions(+), 44 deletions(-) diff --git a/src/main/native-chat/agent-session-timeline/agent-journal-turn-row-revision.ts b/src/main/native-chat/agent-session-timeline/agent-journal-turn-row-revision.ts index 69356c406784..dfede8c46f7c 100644 --- a/src/main/native-chat/agent-session-timeline/agent-journal-turn-row-revision.ts +++ b/src/main/native-chat/agent-session-timeline/agent-journal-turn-row-revision.ts @@ -24,8 +24,7 @@ import { estimateStructuredAgentSessionItemBytes } from '../agent-session-wire/s import type { StructuredAgentSessionEventSink, StructuredAgentSessionRevisionJournal, - StructuredAgentSessionRevisionOptions, - StructuredAgentSessionSinkAdmission + StructuredAgentSessionRevisionOptions } from '../agent-session-wire/structured-agent-session-event-sink' /** The row a write revises: a known one, or the newest turn in the journal. */ @@ -203,7 +202,7 @@ export function writeAgentJournalTurnRow( target: AgentJournalTurnRowTarget, write: AgentJournalTurnRowWrite, { publish, options: delivery = {} }: AgentJournalTurnRowDelivery -): StructuredAgentSessionSinkAdmission { +): void { // A turn record belongs to no turn. const options = { ...delivery, turnScope: AGENT_JOURNAL_THREAD_SCOPE } const revise = publish ? sink.tryReviseResolvedItemAndPublish : sink.tryReviseResolvedItem @@ -214,10 +213,10 @@ export function writeAgentJournalTurnRow( sink.publish() } } - return { accepted: true } + return } const reservedBytes = agentJournalTurnRowReservedBytes(target, write) - return revise.call( + revise.call( sink, reservedBytes, (journal) => resolveAgentJournalTurnRowWrite(journal, target, write, reservedBytes), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts index 1b242dd8d983..f752a2147afc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts @@ -8,7 +8,11 @@ import { type AgentJournalItemIdentity, type AgentJournalToolCallItem } from '../../../shared/agent-session-journal-types' -import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' +import { + createTrackedJournalOpener, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { createAgentSessionDeltaCoalescer } from './agent-session-delta-coalescer' import { @@ -38,6 +42,15 @@ function tool(name: string, state: AgentJournalToolCallItem['state']): AgentJour return { kind: 'tool-call', name, input: { name }, state } } +function failedTool(id: AgentJournalItemIdentity): JournalLifecycleMutationInput { + return { + kind: 'item', + identity: id, + body: tool('read', 'failed'), + turnScope: AGENT_JOURNAL_THREAD_SCOPE + } +} + function itemStep( resolve: Extract['resolve'] ): StructuredAgentSessionTransitionStep { @@ -69,7 +82,7 @@ async function rig(watermarks: Partial = { watermarks }) const bind = () => deferred.bind({ journal, fence: 1, publish: () => publishes.push(1) }) - return { journal, deferred, sink: deferred.sink, publishes, bind } + return { root, journal, deferred, sink: deferred.sink, publishes, bind } } describe('structured agent-session transitions', () => { @@ -108,10 +121,9 @@ describe('structured agent-session transitions', () => { expect(publishes).toHaveLength(1) }) - it('places a row where its resolver says, and hears every landing', async () => { + it('places a row where its resolver says', async () => { const { journal, deferred, sink, bind } = await rig() bind() - const landed: (readonly boolean[])[] = [] const scope = { kind: 'turn', turnItemId: 'turn-row' } as const sink.tryAppendTransition?.({ lifecycle: false, @@ -120,19 +132,44 @@ describe('structured agent-session transitions', () => { itemStep(() => ({ identity: identity('placed'), body: tool('read', 'running'), - options: { turnScope: scope, providerItemRef: 'item:read' } + options: { turnScope: scope } })), itemStep(() => null) - ], - landed: (wrote) => landed.push(wrote) + ] }) await deferred.drained() expect(journal.item(agentJournalItemKey(identity('placed')))).toMatchObject({ - turnScope: scope, - providerItemRef: 'item:read' + turnScope: scope }) - expect(landed).toEqual([[true, false]]) + }) + + it('admitted whole is not executed whole: a failed step keeps the ones before it and fails the sink', async () => { + const { journal, deferred, sink, publishes, bind } = await rig() + bind() + sink.tryAppendTransition?.({ + lifecycle: false, + publish: true, + steps: [ + itemStep(() => ({ identity: identity('kept'), body: tool('read', 'running') })), + itemStep(() => { + throw new Error('resolver failed') + }) + ] + }) + + await expect(deferred.drained()).resolves.toMatchObject({ ok: false }) + expect(journal.snapshot().items.map((item) => item.itemId)).toEqual([ + agentJournalItemKey(identity('kept')) + ]) + expect(publishes).toEqual([]) + expect( + sink.tryAppendTransition?.({ + lifecycle: false, + publish: false, + steps: [itemStep(() => null)] + }) + ).toEqual({ accepted: false, reason: 'failed' }) }) it('refuses a transition whole, so none of its steps ever lands', async () => { @@ -165,14 +202,10 @@ describe('structured agent-session transitions', () => { sink.tryAppendItem?.(identity('done'), tool('read', 'completed'), { turnScope: AGENT_JOURNAL_THREAD_SCOPE }) - let wrote: readonly boolean[] = [] expect( sink.tryAppendTransition?.({ lifecycle: true, publish: true, - landed: (result) => { - wrote = result - }, steps: [ { kind: 'settlement', @@ -213,20 +246,50 @@ describe('structured agent-session transitions', () => { ).toMatchObject({ state: 'completed' }) // Two batch rows, back to back, between the last append before it and the first after it. expect(sequenceOf('after') - sequenceOf('done')).toBe(3) - expect(wrote).toEqual([true]) expect(publishes).toHaveLength(1) }) + it('leaves no row of a settlement durable when a later one of its rows fails to write', async () => { + const { root, journal, deferred, sink, bind } = await rig() + bind() + const running = Array.from({ length: 250 }, (_, index) => identity(`tool-${index}`)) + for (const id of running) { + sink.tryAppendItem?.(id, tool('read', 'running'), { turnScope: AGENT_JOURNAL_THREAD_SCOPE }) + } + await deferred.drained() + const before = journal.cursor().sequence + // The settlement needs two rows; the second one's insert aborts inside the transaction. + openTestJournalHostDatabase(root).db.exec(`CREATE TEMP TRIGGER fail_second_chunk +BEFORE INSERT ON main.journal_rows WHEN NEW.seq = ${before + 2} +BEGIN SELECT RAISE(ABORT, 'second chunk refused'); END`) + sink.tryAppendTransition?.({ + lifecycle: true, + publish: true, + steps: [ + { + kind: 'settlement', + settlementId: 'settle-all', + reservedBytes: 1, + resolve: () => running.map(failedTool) + } + ] + }) + + await expect(deferred.drained()).resolves.toMatchObject({ ok: false }) + expect(journal.cursor().sequence).toBe(before) + expect( + journal + .snapshot() + .items.filter((item) => item.body.kind === 'tool-call' && item.body.state === 'failed') + ).toEqual([]) + }) + it('writes and announces nothing when a step resolves to nothing', async () => { const { journal, deferred, sink, publishes, bind } = await rig() bind() - let wrote: readonly boolean[] = [] sink.tryAppendTransition?.({ lifecycle: true, publish: true, - landed: (result) => { - wrote = result - }, steps: [ itemStep(() => null), { kind: 'settlement', settlementId: 'none', reservedBytes: 1, resolve: () => [] } @@ -235,11 +298,27 @@ describe('structured agent-session transitions', () => { await deferred.drained() expect(journal.snapshot().items).toEqual([]) - expect(wrote).toEqual([false, false]) expect(publishes).toEqual([]) }) }) +describe('resolved lifecycle batches', () => { + it('refuses, before writing anything, a settlement that names one item twice', async () => { + const { journal } = await rig() + const before = journal.cursor().sequence + const twice = identity('twice') + + await expect( + journal.appendResolvedLifecycleBatch({ + settlementId: 'twice', + fence: 1, + resolve: () => [failedTool(identity('once')), failedTool(twice), failedTool(twice)] + }) + ).rejects.toThrow('journal_resolved_lifecycle_batch_names_item_twice') + expect(journal.cursor().sequence).toBe(before) + }) +}) + describe('coalescer text a caller writes itself', () => { it('reports unwritten streams and owes nothing once the caller marks them written', () => { const emitted: string[] = [] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts index c67dab93edef..a185695d5ea5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts @@ -6,6 +6,8 @@ // all. At execution its steps are issued in the same tick, so they sit together in the journal's // write queue, and each step resolves against the fold with every earlier write landed — the // steps before it included — so what a step writes is decided by the journal, not by memory. +// Admitted whole is not executed whole: a step that fails leaves the steps before it written, and +// fails the sink, so nothing more is admitted after it. import type { AgentJournalItemBody, @@ -21,18 +23,10 @@ import type { import type { StructuredAgentSessionSinkQueue } from './structured-agent-session-event-sink-queue' import { structuredAgentSessionJournalAppendOptions } from './structured-agent-session-journal-append-options' -/** What a transition step reads: rows by key or provider reference, every row, their turns, and - * the sends whose echo holds a provider item's place. */ +/** What a transition step reads: rows by key, every row, and the turns they joined. */ export type StructuredAgentSessionTransitionJournal = Pick< AgentSessionJournal, - | 'epoch' - | 'submissions' - | 'visitItems' - | 'itemBody' - | 'item' - | 'itemIdForProviderItemRef' - | 'canonicalItemId' - | 'visitItemsWithLinkage' + 'epoch' | 'visitItems' | 'itemBody' | 'item' | 'visitItemsWithLinkage' > export type StructuredAgentSessionTransitionStep = @@ -69,8 +63,6 @@ export type StructuredAgentSessionTransition = { lifecycle: boolean /** Announce the writes once they land, when any step wrote. */ publish: boolean - /** Which steps wrote a row, once every step has landed or one failed (then none counts). */ - landed?: (wrote: readonly boolean[]) => void } const STEP_OVERFLOW = 'structured agent-session transition step exceeded its reserved size' @@ -143,13 +135,7 @@ function transitionAppend( }) .then((landed) => landed !== null) ) - let wrote: boolean[] = transition.steps.map(() => false) - try { - wrote = await Promise.all(writes) - } finally { - // Heard even when a write failed, so a writer's speculation about it always ends. - transition.landed?.(wrote) - } + const wrote = await Promise.all(writes) if (transition.publish && wrote.some(Boolean)) { bound.publish() } From a4df9df2f38d307492628a3d441b5c1657140da6 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:32:46 -0700 Subject: [PATCH 09/11] Run a transition's steps as a prefix; drop the paced flag and resolved options A failed step no longer lets the steps after it write: each step checks, at its own turn in the journal's queue, whether the write handed over just ahead of it completed, using the queue's count of completed write bodies (a promise would report the failure only after the next step ran). The sink fails only once every step has had its turn. The item step's `paced` size bypass and the resolver's replacement `options` are removed; nothing planned uses them. --- .../journal-item-appender.ts | 9 +- .../agent-session-journal/journal-store.ts | 5 + .../journal-write-queue.ts | 16 ++- ...tructured-agent-session-transition.test.ts | 106 +++++++++++++----- .../structured-agent-session-transition.ts | 78 ++++++++----- 5 files changed, 152 insertions(+), 62 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-item-appender.ts b/src/main/native-chat/agent-session-journal/journal-item-appender.ts index 479f0bc04a9d..146078d830c9 100644 --- a/src/main/native-chat/agent-session-journal/journal-item-appender.ts +++ b/src/main/native-chat/agent-session-journal/journal-item-appender.ts @@ -10,12 +10,7 @@ import type { JournalRow } from './journal-row-schema' const NOTHING_RESOLVED = new Error('journal_item_resolved_to_nothing') -export type JournalResolvedItem = { - identity: AgentJournalItemIdentity - body: AgentJournalItemBody - /** Replaces the call's options when only the fold could say where the row belongs. */ - options?: JournalItemAppendOptions -} +export type JournalResolvedItem = { identity: AgentJournalItemIdentity; body: AgentJournalItemBody } export class JournalItemAppender { constructor( @@ -53,7 +48,7 @@ export class JournalItemAppender { this.deps.state, resolved.identity, resolved.body, - resolved.options ?? options + options )(seq, ts) }) .then( diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index d12257447cfd..3cabb7bd53ef 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -208,6 +208,11 @@ export class AgentSessionJournal { return this.queue.owing } + /** Write bodies that returned so far; see `JournalWriteQueue.completedWrites`. */ + get completedWrites(): number { + return this.queue.completedWrites + } + cursor = (): AgentJournalCursor => ({ epoch: this.state.epoch, sequence: this.state.lastSequence diff --git a/src/main/native-chat/agent-session-journal/journal-write-queue.ts b/src/main/native-chat/agent-session-journal/journal-write-queue.ts index 74ef33472bd4..ae4f5c96d893 100644 --- a/src/main/native-chat/agent-session-journal/journal-write-queue.ts +++ b/src/main/native-chat/agent-session-journal/journal-write-queue.ts @@ -30,6 +30,7 @@ export class JournalWriteQueue { private closed = false /** Runs before the next write, and stays owed until it succeeds. */ private owed: (() => Promise) | null = null + private completed = 0 constructor(private readonly sessionId: string) {} @@ -41,9 +42,20 @@ export class JournalWriteQueue { if (this.closed) { return Promise.reject(this.closedError()) } + const counted = (): JournalWriteResult => { + const result = run() + this.completed++ + return result + } return this.owed !== null || this.lineBusy - ? this.join(run, this.owed !== null) - : this.runNow(run) + ? this.join(counted, this.owed !== null) + : this.runNow(counted) + } + + /** Write bodies that returned without throwing. Bodies run one at a time, so a body that reads + * this learns synchronously whether the write just ahead of it landed. */ + get completedWrites(): number { + return this.completed } /** Runs `read` after every write admitted before it, whatever each one's outcome, and ahead of any diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts index f752a2147afc..b4232642dee0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import { AGENT_JOURNAL_THREAD_SCOPE, @@ -52,11 +52,12 @@ function failedTool(id: AgentJournalItemIdentity): JournalLifecycleMutationInput } function itemStep( - resolve: Extract['resolve'] + resolve: Extract['resolve'], + reservedBytes = 4096 ): StructuredAgentSessionTransitionStep { return { kind: 'item', - reservedBytes: 4096, + reservedBytes, resolve, options: { turnScope: AGENT_JOURNAL_THREAD_SCOPE } } @@ -121,29 +122,6 @@ describe('structured agent-session transitions', () => { expect(publishes).toHaveLength(1) }) - it('places a row where its resolver says', async () => { - const { journal, deferred, sink, bind } = await rig() - bind() - const scope = { kind: 'turn', turnItemId: 'turn-row' } as const - sink.tryAppendTransition?.({ - lifecycle: false, - publish: true, - steps: [ - itemStep(() => ({ - identity: identity('placed'), - body: tool('read', 'running'), - options: { turnScope: scope } - })), - itemStep(() => null) - ] - }) - await deferred.drained() - - expect(journal.item(agentJournalItemKey(identity('placed')))).toMatchObject({ - turnScope: scope - }) - }) - it('admitted whole is not executed whole: a failed step keeps the ones before it and fails the sink', async () => { const { journal, deferred, sink, publishes, bind } = await rig() bind() @@ -172,6 +150,82 @@ describe('structured agent-session transitions', () => { ).toEqual({ accepted: false, reason: 'failed' }) }) + it('writes nothing after a step that overflows its reservation', async () => { + const { journal, deferred, sink, publishes, bind } = await rig() + bind() + const after = vi.fn(() => ({ identity: identity('third'), body: tool('read', 'running') })) + sink.tryAppendTransition?.({ + lifecycle: false, + publish: true, + steps: [ + itemStep(() => ({ identity: identity('first'), body: tool('read', 'running') })), + itemStep( + () => ({ identity: identity('second'), body: tool('x'.repeat(10_000), 'running') }), + 64 + ), + itemStep(after) + ] + }) + + await expect(deferred.drained()).resolves.toMatchObject({ ok: false }) + expect(journal.snapshot().items.map((item) => item.itemId)).toEqual([ + agentJournalItemKey(identity('first')) + ]) + expect(after).not.toHaveBeenCalled() + expect(publishes).toEqual([]) + }) + + it('opens no next-turn work after a settlement the journal refused', async () => { + const { journal, deferred, sink, bind } = await rig() + bind() + const old = identity('old-tool') + sink.tryAppendItem?.(old, tool('read', 'running'), { turnScope: AGENT_JOURNAL_THREAD_SCOPE }) + await deferred.drained() + sink.tryAppendTransition?.({ + lifecycle: true, + publish: true, + steps: [ + itemStep(() => ({ identity: identity('before'), body: tool('read', 'completed') })), + // Names one item twice, which the journal refuses. + { + kind: 'settlement', + settlementId: 'settle', + reservedBytes: 1, + resolve: () => [failedTool(old), failedTool(old)] + }, + itemStep(() => ({ identity: identity('next-turn-work'), body: tool('read', 'running') })) + ] + }) + + await expect(deferred.drained()).resolves.toMatchObject({ ok: false }) + expect(journal.snapshot().items.map((item) => item.itemId)).toEqual([ + agentJournalItemKey(old), + agentJournalItemKey(identity('before')) + ]) + expect(journal.item(agentJournalItemKey(old))?.body).toMatchObject({ state: 'running' }) + }) + + it('writes nothing after a step whose row the database refused', async () => { + const { root, journal, deferred, sink, bind } = await rig() + bind() + // Refuses only the second step's row, so the third would land in its place. + openTestJournalHostDatabase(root).db.exec(`CREATE TEMP TRIGGER fail_second_step +BEFORE INSERT ON main.journal_rows WHEN instr(NEW.row_json, 'refused-step') > 0 +BEGIN SELECT RAISE(ABORT, 'second step refused'); END`) + const step = (recordId: string) => + itemStep(() => ({ identity: identity(recordId), body: tool(recordId, 'running') })) + sink.tryAppendTransition?.({ + lifecycle: false, + publish: true, + steps: [step('first'), step('refused-step'), step('third')] + }) + + await expect(deferred.drained()).resolves.toMatchObject({ ok: false }) + expect(journal.snapshot().items.map((item) => item.itemId)).toEqual([ + agentJournalItemKey(identity('first')) + ]) + }) + it('refuses a transition whole, so none of its steps ever lands', async () => { const { journal, deferred, sink, bind } = await rig({ maxQueuedOperations: 1 }) const step = (recordId: string) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts index a185695d5ea5..57c0c7d0f5e8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts @@ -6,8 +6,8 @@ // all. At execution its steps are issued in the same tick, so they sit together in the journal's // write queue, and each step resolves against the fold with every earlier write landed — the // steps before it included — so what a step writes is decided by the journal, not by memory. -// Admitted whole is not executed whole: a step that fails leaves the steps before it written, and -// fails the sink, so nothing more is admitted after it. +// Admitted whole, executed as a prefix: once a step fails, the steps after it write nothing, the +// steps before it stay written, and the sink fails, so nothing more is admitted after it. import type { AgentJournalItemBody, @@ -32,16 +32,12 @@ export type StructuredAgentSessionTransitionJournal = Pick< export type StructuredAgentSessionTransitionStep = | { kind: 'item' - /** Bounds what `resolve` may write; a larger write fails the sink, unless `paced`. */ + /** Bounds what `resolve` may write; a larger write fails the sink. */ reservedBytes: number - /** The size only paces the queue: the row grows by text the fold holds (a resumed message). */ - paced?: true - /** The row and its whole body; null writes nothing. Resolved `options` replace the planned - * ones, for a writer that learns the row's turn or producer only from the fold. */ + /** The row and its whole body; null writes nothing. */ resolve: (journal: StructuredAgentSessionTransitionJournal) => { identity: AgentJournalItemIdentity body: AgentJournalItemBody - options?: StructuredAgentSessionItemAppendOptions } | null options: StructuredAgentSessionItemAppendOptions } @@ -94,35 +90,25 @@ function transitionAppend( lifecycle: transition.lifecycle, run: async (bound) => { const { journal, fence } = bound + const gate = createStepGate(journal) // Issued in one tick, so no write submitted after this transition lands between its steps. - const writes = transition.steps.map((step) => + const writes = transition.steps.map((step, index) => step.kind === 'item' ? journal .appendResolvedItem( () => { - const resolved = step.resolve(journal) + const resolved = gate.open(index) ? step.resolve(journal) : null if (!resolved) { + gate.wroteNothing() return null } if ( - !step.paced && estimateStructuredAgentSessionItemBytes(resolved.identity, resolved.body) > - step.reservedBytes + step.reservedBytes ) { throw new Error(STEP_OVERFLOW) } - return { - identity: resolved.identity, - body: resolved.body, - ...(resolved.options - ? { - options: structuredAgentSessionJournalAppendOptions( - fence, - resolved.options - ) - } - : {}) - } + return resolved }, structuredAgentSessionJournalAppendOptions(fence, step.options) ) @@ -130,15 +116,53 @@ function transitionAppend( : journal .appendResolvedLifecycleBatch({ settlementId: step.settlementId, - resolve: () => step.resolve(journal), + resolve: () => (gate.open(index) ? step.resolve(journal) : []), fence }) .then((landed) => landed !== null) ) - const wrote = await Promise.all(writes) - if (transition.publish && wrote.some(Boolean)) { + // Every step, the ones a failure turned into no-ops included, has run before this settles. + const outcomes = await Promise.allSettled(writes) + const failure = outcomes.find( + (outcome): outcome is PromiseRejectedResult => outcome.status === 'rejected' + ) + if (failure) { + throw failure.reason + } + if ( + transition.publish && + outcomes.some((outcome) => outcome.status === 'fulfilled' && outcome.value) + ) { bound.publish() } } }) } + +/** Lets a step write only when every step before it landed. Steps run in queue order, so each + * step's write learns synchronously, from the journal's count of completed writes, whether the + * write handed over just ahead of it landed; a promise would say so only after it ran. */ +function createStepGate(journal: Pick): { + /** First thing step `index` does at its turn; false once an earlier step failed. */ + open: (index: number) => boolean + /** The step writes nothing, so whether its write completes says nothing about a failure. */ + wroteNothing: () => void +} { + let failed = false + let reached = 0 + let handedAt: number | null = null + return { + open: (index) => { + // A step that never reached its resolve failed before it: a refused import, a read-only journal. + if (reached !== index || (handedAt !== null && journal.completedWrites === handedAt)) { + failed = true + } + reached = index + 1 + handedAt = failed ? null : journal.completedWrites + return !failed + }, + wroteNothing: () => { + handedAt = null + } + } +} From fc8541f5dd1f1cd1fd293e7085413d0097747ef4 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:49:41 -0700 Subject: [PATCH 10/11] Run a transition's steps in one queued write that loops over them The steps of one event now share one turn in the journal's write queue: a loop writes each in its own transaction through the row writer's synchronous writeRows (split out of enqueueRows) and stops at the first throw. Prefix semantics and "nothing lands between the steps" now hold by construction, so the completed-write counter on the queue, the step gate and the allSettled barrier are gone; the queue is back to main's bytes. --- .../journal-lifecycle-batch-appender.ts | 38 ++--- .../journal-row-writer.ts | 55 ++++---- .../journal-step-writer.ts | 58 ++++++++ .../journal-store-collaborators.ts | 17 ++- .../agent-session-journal/journal-store.ts | 14 +- .../journal-write-queue.ts | 16 +-- ...tructured-agent-session-transition.test.ts | 132 +++++++++++++++++- .../structured-agent-session-transition.ts | 94 ++++--------- 8 files changed, 274 insertions(+), 150 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-step-writer.ts diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts index cc300a7b7497..3cac8d03c211 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts @@ -78,30 +78,20 @@ export class JournalLifecycleBatchAppender { }) } - /** Chooses the mutations at its turn in the queue; one too large for a row becomes consecutive - * rows, committed in one transaction, in the order resolved. */ - appendResolved(input: JournalResolvedLifecycleBatchInput): Promise { - return this.deps - .enqueueRows(() => { - const mutations = input.resolve() - // Every chunk is built before any commits, so a second chunk naming the same item would - // reuse the first chunk's revision. - this.assertDistinctItems(mutations) - return partitionJournalLifecycleMutations(input.settlementId, mutations) - .filter((chunk) => !this.wasApplied(chunk.settlementId)) - .map((chunk) => - journalLifecycleBatchRowBuilder( - this.deps.state, - chunk.settlementId, - chunk.mutations, - input - ) - ) - }) - .then((rows) => { - const last = rows.at(-1) - return last ? { epoch: last.epoch, sequence: last.seq } : null - }) + /** The rows a resolved settlement writes, planned at its own turn in the queue: its mutations, + * chosen then, in as many consecutive rows as they need, minus any already applied. */ + planResolved( + input: JournalResolvedLifecycleBatchInput + ): ((seq: number, ts: number) => JournalRow)[] { + const mutations = input.resolve() + // Every chunk is built before any commits, so a second chunk naming the same item would + // reuse the first chunk's revision. + this.assertDistinctItems(mutations) + return partitionJournalLifecycleMutations(input.settlementId, mutations) + .filter((chunk) => !this.wasApplied(chunk.settlementId)) + .map((chunk) => + journalLifecycleBatchRowBuilder(this.deps.state, chunk.settlementId, chunk.mutations, input) + ) } private assertDistinctItems(mutations: readonly JournalLifecycleMutationInput[]): void { diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.ts index dde1a66741d1..60db6e0cbd1e 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.ts @@ -76,33 +76,36 @@ export class JournalRowWriter { enqueueRows( plan: () => readonly ((seq: number, ts: number) => JournalRow)[] ): Promise { - return this.deps.serialize(() => { - assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) - const first = this.deps.nextSequence() - const ts = this.deps.now() - const rows = plan().map((build, index) => build(first + index, ts)) - if (rows.length === 0) { - return rows - } - for (const row of rows) { - assertJournalFence(row.fence, this.deps.highestFence()) - } - try { - this.deps.database().transaction((db) => { - for (const row of rows) { - insertJournalRow(db, this.deps.sessionId, row) - this.runBookkeeping(db, row) - } - }) - } catch (error) { - this.deps.rolledBack?.() - throw error - } - for (const row of rows) { - this.deps.commit(row) - } + return this.deps.serialize(() => this.writeRows(plan)) + } + + /** `enqueueRows`' write, for a caller already running at its own turn in the queue. */ + writeRows(plan: () => readonly ((seq: number, ts: number) => JournalRow)[]): JournalRow[] { + assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) + const first = this.deps.nextSequence() + const ts = this.deps.now() + const rows = plan().map((build, index) => build(first + index, ts)) + if (rows.length === 0) { return rows - }) + } + for (const row of rows) { + assertJournalFence(row.fence, this.deps.highestFence()) + } + try { + this.deps.database().transaction((db) => { + for (const row of rows) { + insertJournalRow(db, this.deps.sessionId, row) + this.runBookkeeping(db, row) + } + }) + } catch (error) { + this.deps.rolledBack?.() + throw error + } + for (const row of rows) { + this.deps.commit(row) + } + return rows } /** Assign the next sequence, make the row durable, and fold it through the SAME reducer diff --git a/src/main/native-chat/agent-session-journal/journal-step-writer.ts b/src/main/native-chat/agent-session-journal/journal-step-writer.ts new file mode 100644 index 000000000000..c0b62ce50c4c --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-step-writer.ts @@ -0,0 +1,58 @@ +// Several writes run as ONE turn in the chat's write queue, one after another. +// +// Each step is planned from the fold with every step before it landed, and commits in its own +// transaction. The steps share one queue body, so no other write lands between them, and the +// first step that throws ends the run: the steps before it stay written and none after it runs. + +import type { + JournalItemAppendOptions, + JournalResolvedLifecycleBatchInput +} from './journal-store-contracts' +import type { JournalResolvedItem } from './journal-item-appender' +import type { JournalLifecycleBatchAppender } from './journal-lifecycle-batch-appender' +import type { JournalReducerState } from './journal-reducer' +import { journalItemRowBuilder } from './journal-row-builders' +import type { JournalRow } from './journal-row-schema' +import type { JournalRowWriter } from './journal-row-writer' +import type { JournalWriteBody } from './journal-write-queue' + +export type JournalStep = + | { + kind: 'item' + /** Read at the step's turn; null writes nothing. */ + resolve: () => JournalResolvedItem | null + options: JournalItemAppendOptions + } + | { kind: 'settlement'; batch: JournalResolvedLifecycleBatchInput } + +export class JournalStepWriter { + constructor( + private readonly deps: { + serialize: (run: JournalWriteBody) => Promise + state: () => JournalReducerState + writeRows: JournalRowWriter['writeRows'] + planSettlement: JournalLifecycleBatchAppender['planResolved'] + } + ) {} + + /** Whether each step wrote; rejects with the error of the step that threw. */ + append(steps: readonly JournalStep[]): Promise { + return this.deps.serialize(() => { + const wrote: boolean[] = [] + for (const step of steps) { + wrote.push(this.deps.writeRows(() => this.plan(step)).length > 0) + } + return wrote + }) + } + + private plan(step: JournalStep): ((seq: number, ts: number) => JournalRow)[] { + if (step.kind === 'settlement') { + return this.deps.planSettlement(step.batch) + } + const resolved = step.resolve() + return resolved === null + ? [] + : [journalItemRowBuilder(this.deps.state, resolved.identity, resolved.body, step.options)] + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index ba8796964692..e5f521e49aea 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -17,6 +17,7 @@ import { JournalStopMarks } from './journal-stop-marks' import { journalQueuePauseRestatement } from './queued-message-pause' import type { JournalReducerState } from './journal-reducer' import { JournalRowWriter } from './journal-row-writer' +import { JournalStepWriter } from './journal-step-writer' import { restoreJournalStore } from './journal-store-restore' import type { JournalRow } from './journal-row-schema' import type { AgentSessionJournal } from './journal-store' @@ -57,6 +58,7 @@ export type JournalStoreCollaborators = { epochController: JournalEpochController itemAppender: JournalItemAppender lifecycleBatchAppender: JournalLifecycleBatchAppender + stepWriter: JournalStepWriter queuedMessages: JournalQueuedMessages stopMarks: JournalStopMarks /** Restores the store's state from disk. Owned here because it needs the same @@ -106,6 +108,12 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal inTransaction: (db, row) => queuedMessages.onRowInTransaction(db, row), rolledBack: () => queuedMessages.invalidate() }) + const lifecycleBatchAppender = new JournalLifecycleBatchAppender({ + state: host.state, + cursor: host.cursor, + enqueue: host.enqueue, + enqueueRows: (plan) => rowWriter.enqueueRows(plan) + }) return { epochController, queuedMessages, @@ -121,11 +129,12 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal state: host.state, enqueue: host.enqueue }), - lifecycleBatchAppender: new JournalLifecycleBatchAppender({ + lifecycleBatchAppender, + stepWriter: new JournalStepWriter({ + serialize: host.serialize, state: host.state, - cursor: host.cursor, - enqueue: host.enqueue, - enqueueRows: (plan) => rowWriter.enqueueRows(plan) + writeRows: (plan) => rowWriter.writeRows(plan), + planSettlement: (batch) => lifecycleBatchAppender.planResolved(batch) }) } } diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 3cabb7bd53ef..3edfbf0c1a75 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -71,6 +71,7 @@ import { createJournalStoreCollaborators } from './journal-store-collaborators' import { journalStoreLoadedFields } from './journal-store-open' import type { JournalItemAppender } from './journal-item-appender' import type { JournalLifecycleBatchAppender } from './journal-lifecycle-batch-appender' +import type { JournalStepWriter } from './journal-step-writer' import type { JournalStopMarks } from './journal-stop-marks' export { AgentSessionJournalError } from './journal-write-guards' @@ -92,6 +93,7 @@ export class AgentSessionJournal { private readonly epochController: JournalEpochController private readonly itemAppender: JournalItemAppender private readonly lifecycleBatchAppender: JournalLifecycleBatchAppender + private readonly stepWriter: JournalStepWriter private readonly restore: () => Promise /** Draft rows queued while the agent works; never reducer input or owed work. */ readonly queuedMessages: JournalQueuedMessages @@ -143,6 +145,7 @@ export class AgentSessionJournal { this.epochController = collaborators.epochController this.itemAppender = collaborators.itemAppender this.lifecycleBatchAppender = collaborators.lifecycleBatchAppender + this.stepWriter = collaborators.stepWriter this.queuedMessages = collaborators.queuedMessages this.stopMarks = collaborators.stopMarks this.restore = collaborators.restore @@ -208,11 +211,6 @@ export class AgentSessionJournal { return this.queue.owing } - /** Write bodies that returned so far; see `JournalWriteQueue.completedWrites`. */ - get completedWrites(): number { - return this.queue.completedWrites - } - cursor = (): AgentJournalCursor => ({ epoch: this.state.epoch, sequence: this.state.lastSequence @@ -336,10 +334,8 @@ export class AgentSessionJournal { return this.lifecycleBatchAppender.append(input) } - /** A lifecycle batch whose mutations are chosen from the fold at its own turn in the queue; - * null when none are. */ - appendResolvedLifecycleBatch: JournalLifecycleBatchAppender['appendResolved'] = (input) => - this.lifecycleBatchAppender.appendResolved(input) + /** Several writes as one turn in the queue; see `JournalStepWriter`. */ + appendSteps: JournalStepWriter['append'] = (steps) => this.stepWriter.append(steps) /** * Write-ahead submission row. It is durable before the caller dispatches diff --git a/src/main/native-chat/agent-session-journal/journal-write-queue.ts b/src/main/native-chat/agent-session-journal/journal-write-queue.ts index ae4f5c96d893..74ef33472bd4 100644 --- a/src/main/native-chat/agent-session-journal/journal-write-queue.ts +++ b/src/main/native-chat/agent-session-journal/journal-write-queue.ts @@ -30,7 +30,6 @@ export class JournalWriteQueue { private closed = false /** Runs before the next write, and stays owed until it succeeds. */ private owed: (() => Promise) | null = null - private completed = 0 constructor(private readonly sessionId: string) {} @@ -42,20 +41,9 @@ export class JournalWriteQueue { if (this.closed) { return Promise.reject(this.closedError()) } - const counted = (): JournalWriteResult => { - const result = run() - this.completed++ - return result - } return this.owed !== null || this.lineBusy - ? this.join(counted, this.owed !== null) - : this.runNow(counted) - } - - /** Write bodies that returned without throwing. Bodies run one at a time, so a body that reads - * this learns synchronously whether the write just ahead of it landed. */ - get completedWrites(): number { - return this.completed + ? this.join(run, this.owed !== null) + : this.runNow(run) } /** Runs `read` after every write admitted before it, whatever each one's outcome, and ahead of any diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts index b4232642dee0..6698b4cef5ec 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts @@ -13,6 +13,7 @@ import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' +import { openJournalOwingImport } from '../agent-session-journal/journal-owed-import-test-support' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { createAgentSessionDeltaCoalescer } from './agent-session-delta-coalescer' import { @@ -86,6 +87,36 @@ async function rig(watermarks: Partial = { return { root, journal, deferred, sink: deferred.sink, publishes, bind } } +/** A bound sink over a chat whose copy into the host's database is still owed, so every write, + * the transition's included, waits in the journal's queue until the first one pays it. */ +async function owedRig() { + const root = await mkdtemp(join(tmpdir(), 'orca-transition-owed-')) + roots.push(root) + const { journal } = await openJournalOwingImport({ + stateDirectory: root, + identity: { + sessionId: SESSION, + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'grok', + providerHandle: { kind: 'opaque', agent: 'grok', value: 'provider-session-1' } + }, + now: () => 1_000 + }) + const failures: unknown[] = [] + const deferred = createDeferredStructuredAgentSessionEventSink({ + ...testEventSinkLogging(SESSION), + onFailed: (error) => failures.push(error) + }) + deferred.bind({ journal, fence: 1, publish: () => undefined }) + const ownKeys = () => + journal + .snapshot() + .items.map((item) => item.itemId) + .filter((itemId) => itemId.includes(SESSION)) + return { journal, deferred, sink: deferred.sink, failures, ownKeys, close: () => journal.close() } +} + describe('structured agent-session transitions', () => { it('lands its steps back to back, each resolved after the one before it', async () => { const { journal, deferred, sink, publishes, bind } = await rig() @@ -354,6 +385,92 @@ BEGIN SELECT RAISE(ABORT, 'second chunk refused'); END`) expect(journal.snapshot().items).toEqual([]) expect(publishes).toEqual([]) }) + + it('lands a write a step issues while it runs after the whole transition', async () => { + const { journal, deferred, sink, bind } = await rig() + bind() + sink.tryAppendTransition?.({ + lifecycle: false, + publish: false, + steps: [ + itemStep(() => { + // Another writer, reached from inside the step: it waits for the transition's turn to end. + sink.tryAppendItem?.(identity('nested'), tool('nested', 'running'), { + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) + return { identity: identity('first'), body: tool('first', 'running') } + }), + itemStep(() => ({ identity: identity('second'), body: tool('second', 'running') })) + ] + }) + await deferred.drained() + + expect(journal.snapshot().items.map((item) => item.itemId)).toEqual( + ['first', 'second', 'nested'].map((recordId) => agentJournalItemKey(identity(recordId))) + ) + }) + + it('runs its steps back to back after an owed import, ahead of a write issued after it', async () => { + const { journal, deferred, sink, ownKeys, close } = await owedRig() + try { + sink.tryAppendTransition?.({ + lifecycle: false, + publish: false, + steps: [ + itemStep(() => ({ identity: identity('first'), body: tool('read', 'running') })), + itemStep((view) => + view.itemBody(agentJournalItemKey(identity('first'))) + ? { identity: identity('second'), body: tool('read', 'running') } + : null + ) + ] + }) + sink.tryAppendItem?.(identity('later'), tool('later', 'running'), { + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) + expect(journal.importPending).toBe(true) + await expect(deferred.drained()).resolves.toEqual({ ok: true }) + + expect(journal.importPending).toBe(false) + expect(ownKeys()).toEqual( + ['first', 'second', 'later'].map((recordId) => agentJournalItemKey(identity(recordId))) + ) + } finally { + await close() + } + }) + + it('stops at a failed middle step after an owed import; a write accepted before the failure still lands', async () => { + const { deferred, sink, failures, ownKeys, close } = await owedRig() + try { + const third = vi.fn(() => ({ identity: identity('third'), body: tool('read', 'running') })) + sink.tryAppendTransition?.({ + lifecycle: false, + publish: false, + steps: [ + itemStep(() => ({ identity: identity('first'), body: tool('read', 'running') })), + itemStep( + () => ({ identity: identity('second'), body: tool('x'.repeat(10_000), 'running') }), + 64 + ), + itemStep(third) + ] + }) + // Accepted before the failure is known: the sink's existing rule lets it land. + sink.tryAppendItem?.(identity('later'), tool('later', 'running'), { + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) + await expect(deferred.drained()).resolves.toMatchObject({ ok: false }) + + expect(ownKeys()).toEqual( + ['first', 'later'].map((recordId) => agentJournalItemKey(identity(recordId))) + ) + expect(third).not.toHaveBeenCalled() + expect(failures).toHaveLength(1) + } finally { + await close() + } + }) }) describe('resolved lifecycle batches', () => { @@ -363,11 +480,16 @@ describe('resolved lifecycle batches', () => { const twice = identity('twice') await expect( - journal.appendResolvedLifecycleBatch({ - settlementId: 'twice', - fence: 1, - resolve: () => [failedTool(identity('once')), failedTool(twice), failedTool(twice)] - }) + journal.appendSteps([ + { + kind: 'settlement', + batch: { + settlementId: 'twice', + fence: 1, + resolve: () => [failedTool(identity('once')), failedTool(twice), failedTool(twice)] + } + } + ]) ).rejects.toThrow('journal_resolved_lifecycle_batch_names_item_twice') expect(journal.cursor().sequence).toBe(before) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts index 57c0c7d0f5e8..f13b897f12bd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts @@ -3,11 +3,10 @@ // A producer that keeps state about what it wrote must change that state only for writes the sink // took; when an event needs several rows, a refusal of the third after the first two were taken // would leave the producer and the journal disagreeing. A transition is admitted whole or not at -// all. At execution its steps are issued in the same tick, so they sit together in the journal's -// write queue, and each step resolves against the fold with every earlier write landed — the -// steps before it included — so what a step writes is decided by the journal, not by memory. -// Admitted whole, executed as a prefix: once a step fails, the steps after it write nothing, the -// steps before it stay written, and the sink fails, so nothing more is admitted after it. +// all. At execution its steps run as one turn in the journal's write queue, each resolved against +// the fold with every earlier write landed (the steps before it included), so what a step writes +// is decided by the journal, not by memory. Admitted whole, executed as a prefix: once a step +// fails, the steps after it never run, the steps before it stay written, and the sink fails. import type { AgentJournalItemBody, @@ -15,6 +14,7 @@ import type { } from '../../../shared/agent-session-journal-types' import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { JournalStep } from '../agent-session-journal/journal-step-writer' import { estimateStructuredAgentSessionItemBytes } from './structured-agent-session-event-sink-estimate' import type { StructuredAgentSessionItemAppendOptions, @@ -90,79 +90,37 @@ function transitionAppend( lifecycle: transition.lifecycle, run: async (bound) => { const { journal, fence } = bound - const gate = createStepGate(journal) - // Issued in one tick, so no write submitted after this transition lands between its steps. - const writes = transition.steps.map((step, index) => - step.kind === 'item' - ? journal - .appendResolvedItem( - () => { - const resolved = gate.open(index) ? step.resolve(journal) : null - if (!resolved) { - gate.wroteNothing() - return null - } + const wrote = await journal.appendSteps( + transition.steps.map((step): JournalStep => + step.kind === 'item' + ? { + kind: 'item', + resolve: () => { + const resolved = step.resolve(journal) if ( + resolved && estimateStructuredAgentSessionItemBytes(resolved.identity, resolved.body) > - step.reservedBytes + step.reservedBytes ) { throw new Error(STEP_OVERFLOW) } return resolved }, - structuredAgentSessionJournalAppendOptions(fence, step.options) - ) - .then((landed) => landed !== null) - : journal - .appendResolvedLifecycleBatch({ - settlementId: step.settlementId, - resolve: () => (gate.open(index) ? step.resolve(journal) : []), - fence - }) - .then((landed) => landed !== null) - ) - // Every step, the ones a failure turned into no-ops included, has run before this settles. - const outcomes = await Promise.allSettled(writes) - const failure = outcomes.find( - (outcome): outcome is PromiseRejectedResult => outcome.status === 'rejected' + options: structuredAgentSessionJournalAppendOptions(fence, step.options) + } + : { + kind: 'settlement', + batch: { + settlementId: step.settlementId, + fence, + resolve: () => step.resolve(journal) + } + } + ) ) - if (failure) { - throw failure.reason - } - if ( - transition.publish && - outcomes.some((outcome) => outcome.status === 'fulfilled' && outcome.value) - ) { + if (transition.publish && wrote.includes(true)) { bound.publish() } } }) } - -/** Lets a step write only when every step before it landed. Steps run in queue order, so each - * step's write learns synchronously, from the journal's count of completed writes, whether the - * write handed over just ahead of it landed; a promise would say so only after it ran. */ -function createStepGate(journal: Pick): { - /** First thing step `index` does at its turn; false once an earlier step failed. */ - open: (index: number) => boolean - /** The step writes nothing, so whether its write completes says nothing about a failure. */ - wroteNothing: () => void -} { - let failed = false - let reached = 0 - let handedAt: number | null = null - return { - open: (index) => { - // A step that never reached its resolve failed before it: a refused import, a read-only journal. - if (reached !== index || (handedAt !== null && journal.completedWrites === handedAt)) { - failed = true - } - reached = index + 1 - handedAt = failed ? null : journal.completedWrites - return !failed - }, - wroteNothing: () => { - handedAt = null - } - } -} From eba27a3c784ea7e4c0a4eb3d6d6020d7962faf47 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:26:46 -0700 Subject: [PATCH 11/11] Use current provider handles in transition tests --- .../structured-agent-session-transition.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts index 6698b4cef5ec..d9a152d0e6ed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.test.ts @@ -73,7 +73,7 @@ async function rig(watermarks: Partial = { workspaceId: 'workspace-1', hostId: 'local', agent: 'grok', - providerHandle: { kind: 'opaque', agent: 'grok', value: 'provider-session-1' } + providerHandle: { transport: 'acp', agent: 'grok', nativeId: 'provider-session-1' } }, stateDirectory: root, now: () => 1_000 @@ -99,7 +99,7 @@ async function owedRig() { workspaceId: 'workspace-1', hostId: 'local', agent: 'grok', - providerHandle: { kind: 'opaque', agent: 'grok', value: 'provider-session-1' } + providerHandle: { transport: 'acp', agent: 'grok', nativeId: 'provider-session-1' } }, now: () => 1_000 })