diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d6af22878..eb74a172b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,11 @@ ### Fixed +- Fixed Codex task imports for valid rollout JSONL files larger than 64 MiB by + parsing a fixed file snapshot incrementally under source, record, converted-byte, + and message-count limits. Source read/conversion failures now reach Desktop as + an actionable error instead of an unknown commit outcome; the Runtime Host + compatibility epoch moves to 110 (#4642). - Fixed a renderer crash dialog reporting React error #185 ("Maximum update depth exceeded") coming from the composer's prompt-history inline completion (#4117): the offer engine the 0.1.11 composer fed could flip-flop its announcement state on diff --git a/apps/desktop/src/main/__tests__/import-tasks-settings-page.test.ts b/apps/desktop/src/main/__tests__/import-tasks-settings-page.test.ts index a62994ecaf..e0ea4a4343 100644 --- a/apps/desktop/src/main/__tests__/import-tasks-settings-page.test.ts +++ b/apps/desktop/src/main/__tests__/import-tasks-settings-page.test.ts @@ -25,6 +25,7 @@ import { createRoot, type Root } from 'react-dom/client'; import { AstryxLocaleProvider, LocaleProvider } from '@maka/ui'; import type { DesktopRuntimeHostRef } from '../../preload/bridge-contract.js'; import type { DesktopExternalSessionCatalogItem } from '../../preload/external-session-catalog.js'; +import type { ExternalSessionImportFailureReason } from '../../preload/external-session-import-result.js'; import { ImportTasksSettingsPage } from '../../renderer/settings/import-tasks-settings-page.js'; import { RuntimeHostSettingsTarget } from '../../renderer/settings/runtime-host-settings-target.js'; @@ -108,6 +109,31 @@ describe('ImportTasksSettingsPage durable import state', () => { await act(async () => harness.root.unmount()); }); + it('shows a source-unreadable banner without starting unknown-outcome recovery', async () => { + const harness = await renderPage({ + catalog: catalog(externalSession()), + importResult: { ok: false, reason: 'source_unreadable' }, + }); + + const importButton = buttonWithText(harness.container, 'Import'); + assert.ok(importButton); + await act(async () => { + importButton.click(); + await Promise.resolve(); + await Promise.resolve(); + }); + + assert.match(harness.container.textContent, /could not be read or converted/); + assert.doesNotMatch(harness.container.textContent, /Check the import result/); + assert.deepEqual(harness.hostCalls(), [ + { operation: 'listSources', host: TEST_RUNTIME_HOST }, + { operation: 'list', host: TEST_RUNTIME_HOST }, + { operation: 'import', host: TEST_RUNTIME_HOST }, + ]); + + await act(async () => harness.root.unmount()); + }); + it('uses catalog in-flight state after remount to disable the source row', async () => { const harness = await renderPage({ catalog: { @@ -1202,14 +1228,15 @@ async function renderPage(options: { adapterIds?: string[]; bySource?: Record>>; importResult?: - | { ok: false; reason: 'commit_outcome_unknown' } - | Promise<{ ok: false; reason: 'commit_outcome_unknown' }>; + | { ok: false; reason: ExternalSessionImportFailureReason } + | Promise<{ ok: false; reason: ExternalSessionImportFailureReason }>; /** * Per-source answers for a batch: `ok` lands, `unknown` is the Host not - * answering, `throw` is a rejection. Keyed by source session id, because a - * batch is exactly the case where the ids must not share one answer. + * answering, `source_unreadable` is a definite pre-commit failure, and + * `throw` is a rejection. Keyed by source session id, because a batch is + * exactly the case where the ids must not share one answer. */ - importBySource?: Record; + importBySource?: Record; onOpenImported?: (sessionId: string) => void; locale?: 'en' | 'zh'; }): Promise<{ @@ -1288,6 +1315,7 @@ async function renderPage(options: { const perSource = options.importBySource?.[sourceSessionId]; if (perSource === 'throw') throw new Error(`import-failed:${sourceSessionId}`); if (perSource === 'unknown') return { ok: false, reason: 'commit_outcome_unknown' }; + if (perSource === 'source_unreadable') return { ok: false, reason: 'source_unreadable' }; if (perSource === 'ok') { return { ok: true, session: { id: `imported-${sourceSessionId}` } }; } @@ -1498,6 +1526,29 @@ describe('ImportTasksSettingsPage batch import', () => { assert.match(text, /unconfirmed|Unconfirmed|outcome/i); }); + it('counts an unreadable source as a definite failure without offering recovery', async () => { + const { container } = await renderPage({ + catalog: { sessions: [externalSession({ id: 'unreadable' })], nextCursor: null }, + importBySource: { unreadable: 'source_unreadable' }, + }); + + await tick(masterBox(container), true); + const run = buttonWithText(container, 'Import selected'); + assert.ok(run); + await act(async () => { + run.click(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }); + + const text = container.textContent ?? ''; + assert.match(text, /No conversation was imported/); + assert.match(text, /1 more could not be imported/); + assert.doesNotMatch(text, /Check the import result/); + assert.equal(buttonWithText(container, 'Retry'), undefined); + }); + it('spins only the conversion in flight, not every queued row', async () => { // A spinner claims something is happening now. Marking every selected row // would put one on rows the batch has not reached, and on rows it already diff --git a/apps/desktop/src/main/__tests__/runtime-host-external-sessions-ipc-main.test.ts b/apps/desktop/src/main/__tests__/runtime-host-external-sessions-ipc-main.test.ts index d4d8fa388c..f7e4f3e02d 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-external-sessions-ipc-main.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-external-sessions-ipc-main.test.ts @@ -134,6 +134,35 @@ test('an uncertain commit still asks the shell to re-read the catalog', async () assert.deepEqual(events, [{ reason: 'created', sessionId: undefined }]); }); +test('maps a pre-commit source failure to a distinct non-recovering reason', async () => { + const events: unknown[] = []; + const ipc = ipcHarness(); + registerRuntimeHostExternalSessionsIpc( + { + client: clientFixture({ + importExternalSession: async () => { + throw new RuntimeHostOperationError( + 'external-session.import', + 'source_unreadable', + 'External Session could not be read or converted', + ); + }, + }), + emitSessionsChanged: (reason, sessionId) => events.push({ reason, sessionId }), + }, + ipc, + ); + + assert.deepEqual( + await ipc.invoke('external-sessions:import', { + adapterId: 'codex', + sourceSessionId: 'source-1', + }), + { ok: false, reason: 'source_unreadable' }, + ); + assert.deepEqual(events, []); +}); + test('rejects malformed renderer requests before they reach the Host client', async () => { let calls = 0; const ipc = ipcHarness(); diff --git a/apps/desktop/src/main/runtime-host-external-sessions-ipc-main.ts b/apps/desktop/src/main/runtime-host-external-sessions-ipc-main.ts index 91e871f800..5b8b1bd0a4 100644 --- a/apps/desktop/src/main/runtime-host-external-sessions-ipc-main.ts +++ b/apps/desktop/src/main/runtime-host-external-sessions-ipc-main.ts @@ -85,20 +85,27 @@ export function registerRuntimeHostExternalSessionsIpc( } catch (error) { if ( error instanceof RuntimeHostOperationError && - error.operation === 'external-session.import' && - error.code === 'commit_outcome_unknown' + error.operation === 'external-session.import' ) { - // "Unknown" means the task may well be in the catalog, so tell the - // shell to read it again. Without this, the only trace of a maybe- - // committed import is the banner on the page, and the page is gone the - // moment the user leaves Settings -- which is exactly when they come - // back and import the same conversation a second time. No id: the - // whole point is that we do not know which task, if any, landed. - deps.emitSessionsChanged('created'); - return { - ok: false, - reason: 'commit_outcome_unknown', - } satisfies ExternalSessionImportIpcResult; + if (error.code === 'commit_outcome_unknown') { + // "Unknown" means the task may well be in the catalog, so tell the + // shell to read it again. Without this, the only trace of a maybe- + // committed import is the banner on the page, and the page is gone the + // moment the user leaves Settings -- which is exactly when they come + // back and import the same conversation a second time. No id: the + // whole point is that we do not know which task, if any, landed. + deps.emitSessionsChanged('created'); + return { + ok: false, + reason: 'commit_outcome_unknown', + } satisfies ExternalSessionImportIpcResult; + } + if (error.code === 'source_unreadable') { + return { + ok: false, + reason: 'source_unreadable', + } satisfies ExternalSessionImportIpcResult; + } } throw error; } diff --git a/apps/desktop/src/preload/external-session-import-result.ts b/apps/desktop/src/preload/external-session-import-result.ts index 5be40aae2b..38c7c40a1a 100644 --- a/apps/desktop/src/preload/external-session-import-result.ts +++ b/apps/desktop/src/preload/external-session-import-result.ts @@ -19,7 +19,11 @@ import type { SessionSummary } from '@maka/core/session'; -/** Stable Desktop IPC result for the one import failure that must not be retried blindly. */ +export type ExternalSessionImportFailureReason = + | 'commit_outcome_unknown' + | 'source_unreadable'; + +/** Stable Desktop IPC result for import failures that require distinct handling. */ export type ExternalSessionImportIpcResult = | { readonly ok: true; readonly session: T } - | { readonly ok: false; readonly reason: 'commit_outcome_unknown' }; + | { readonly ok: false; readonly reason: ExternalSessionImportFailureReason }; diff --git a/apps/desktop/src/renderer/locales/external-session-import-copy.ts b/apps/desktop/src/renderer/locales/external-session-import-copy.ts index 76e6265918..c9913e813a 100644 --- a/apps/desktop/src/renderer/locales/external-session-import-copy.ts +++ b/apps/desktop/src/renderer/locales/external-session-import-copy.ts @@ -68,6 +68,7 @@ type ExternalSessionImportCopy = { importInProgressDescription: (name: string) => string; importFailedTitle: string; importFailedFallback: string; + importFailedSourceUnreadable: string; importRecoveredTitle: string; importRecoveredDescription: (name: string) => string; importNotRecordedTitle: string; @@ -133,6 +134,8 @@ const COPY = { importInProgressDescription: (name) => `正在导入「${name}」,完成后会直接打开这个任务。`, importFailedTitle: '导入失败', importFailedFallback: '该对话无法转换或保存。请检查来源后重试。', + importFailedSourceUnreadable: + '无法读取或转换该对话。来源可能已损坏,或内容超过安全导入限制。请检查来源后重试。', importRecoveredTitle: '已确认导入', importRecoveredDescription: (name) => `「${name}」导入的任务现已可用。`, importNotRecordedTitle: '没有发现新任务', @@ -186,6 +189,8 @@ const COPY = { `Importing “${name}”. Maka opens the task as soon as it lands.`, importFailedTitle: 'Import failed', importFailedFallback: 'This conversation could not be converted or saved. Check the source and try again.', + importFailedSourceUnreadable: + 'This conversation could not be read or converted. The source may be malformed or exceed a safe import limit.', importRecoveredTitle: 'Import confirmed', importRecoveredDescription: (name) => `The imported task is available now for “${name}”.`, diff --git a/apps/desktop/src/renderer/settings/import-tasks-settings-page.tsx b/apps/desktop/src/renderer/settings/import-tasks-settings-page.tsx index 39fc454fd3..98193b99bb 100644 --- a/apps/desktop/src/renderer/settings/import-tasks-settings-page.tsx +++ b/apps/desktop/src/renderer/settings/import-tasks-settings-page.tsx @@ -694,7 +694,14 @@ export function ImportTasksSettingsPage(props: { // the user has left steering the shell somewhere they did not ask for. if (!mountedRef.current) return; if (!outcome.ok) { - await recoverUnknownImport(attempt); + if (outcome.reason === 'commit_outcome_unknown') { + await recoverUnknownImport(attempt); + } else if (outcome.reason === 'source_unreadable') { + setImportError(copy.importFailedSourceUnreadable); + } else { + const _exhaustive: never = outcome.reason; + return _exhaustive; + } return; } props.onImported(outcome.session); @@ -773,30 +780,45 @@ export function ImportTasksSettingsPage(props: { try { const result = await requestImport(attempt.adapterId, attempt.sourceSessionId); if (!mountedRef.current) return; - outcome = recordImportBatchResult( - outcome, - session.id, - // Not `failed`: the call did not answer, and only a catalog read - // settles whether the conversion landed. Calling it a failure is - // what invites the retry that makes a second copy. - result.ok ? (wasImported ? 'duplicated' : 'imported') : 'unknown', - ); - if (!result.ok) { - // Recorded, not recovered. A single import recovers inline, but - // recovery re-reads the whole catalog window per attempt, and doing - // that between conversions would interleave N full reads with the - // batch and race the writes it is making. The unconfirmed banner - // names every one of these and its 重试 resolves them a press at a - // time, removing each as it settles. - setUncertainImports((current) => - current.some( - (entry) => - entry.adapterId === attempt.adapterId && - entry.sourceSessionId === attempt.sourceSessionId, - ) - ? current - : [...current, attempt], + if (result.ok) { + outcome = recordImportBatchResult( + outcome, + session.id, + wasImported ? 'duplicated' : 'imported', ); + } else { + switch (result.reason) { + case 'commit_outcome_unknown': + // Not `failed`: the call did not answer, and only a catalog read + // settles whether the conversion landed. Calling it a failure is + // what invites the retry that makes a second copy. + outcome = recordImportBatchResult(outcome, session.id, 'unknown'); + // Recorded, not recovered. A single import recovers inline, but + // recovery re-reads the whole catalog window per attempt, and doing + // that between conversions would interleave N full reads with the + // batch and race the writes it is making. The unconfirmed banner + // names every one of these and its 重试 resolves them a press at a + // time, removing each as it settles. + setUncertainImports((current) => + current.some( + (entry) => + entry.adapterId === attempt.adapterId && + entry.sourceSessionId === attempt.sourceSessionId, + ) + ? current + : [...current, attempt], + ); + break; + case 'source_unreadable': + // Conversion failed before persistence, so this one definitely + // did not land and must never enter unknown-outcome recovery. + outcome = recordImportBatchResult(outcome, session.id, 'failed'); + break; + default: { + const _exhaustive: never = result.reason; + return _exhaustive; + } + } } } catch { if (!mountedRef.current) return; diff --git a/packages/runtime-host/src/__tests__/external-session-coordinator.test.ts b/packages/runtime-host/src/__tests__/external-session-coordinator.test.ts index 6a7826628e..fbc13624df 100644 --- a/packages/runtime-host/src/__tests__/external-session-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/external-session-coordinator.test.ts @@ -345,7 +345,10 @@ test('reports conversion errors before persistence and store uncertainty after e ), { ok: false, - error: { code: 'invalid_request', message: 'External Session could not be converted' }, + error: { + code: 'source_unreadable', + message: 'External Session could not be read or converted', + }, }, ); assert.equal(createAttempts, 0); diff --git a/packages/runtime-host/src/__tests__/protocol.test.ts b/packages/runtime-host/src/__tests__/protocol.test.ts index 6f8210429e..c0d323f90b 100644 --- a/packages/runtime-host/src/__tests__/protocol.test.ts +++ b/packages/runtime-host/src/__tests__/protocol.test.ts @@ -437,6 +437,10 @@ describe('Runtime Host bootstrap protocol', () => { assert.ok(RUNTIME_HOST_COMPATIBILITY_EPOCH > 99); }); + test('publishes a new compatibility epoch for external Session source failures', () => { + assert.ok(RUNTIME_HOST_COMPATIBILITY_EPOCH > 109); + }); + test('selects the highest mutually supported protocol and rejects a gap', () => { assert.equal(negotiateProtocol({ min: 0, max: 0 }, { min: 0, max: 0 }), 0); assert.equal(negotiateProtocol({ min: 1, max: 3 }, { min: 2, max: 4 }), 3); diff --git a/packages/runtime-host/src/protocol/external-session.ts b/packages/runtime-host/src/protocol/external-session.ts index 7087db0a36..3a8e683156 100644 --- a/packages/runtime-host/src/protocol/external-session.ts +++ b/packages/runtime-host/src/protocol/external-session.ts @@ -57,6 +57,7 @@ const IMPORT_ERRORS = [ 'not_found', 'operation_conflict', 'commit_outcome_unknown', + 'source_unreadable', ] as const; export type ExternalSessionSourceQueryInput = Record; diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index b929c378be..a3d0abc5cd 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -100,7 +100,9 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const; export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 110 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 111 as const; +// 111: external Session import can report `source_unreadable`, allowing clients +// to distinguish a definite source conversion failure from an uncertain commit. // 110: Runtime Host is the sole schema-migration authority for its State Root. // Epoch 109 Desktop builds could migrate the event-only AgentRun schema while // an older service Host still held the root, leaving that Host querying a diff --git a/packages/runtime-host/src/protocol/operation-spec.ts b/packages/runtime-host/src/protocol/operation-spec.ts index 7b85f061a4..736be7ad60 100644 --- a/packages/runtime-host/src/protocol/operation-spec.ts +++ b/packages/runtime-host/src/protocol/operation-spec.ts @@ -31,6 +31,7 @@ export type HostOperationErrorCode = | 'operation_conflict' | 'capability_unavailable' | 'invalid_request' + | 'source_unreadable' | 'projection_incomplete' | 'stale_cursor' | 'persistence_failed' diff --git a/packages/runtime-host/src/server/external-session-coordinator.ts b/packages/runtime-host/src/server/external-session-coordinator.ts index 18cad24d7f..028734a2f7 100644 --- a/packages/runtime-host/src/server/external-session-coordinator.ts +++ b/packages/runtime-host/src/server/external-session-coordinator.ts @@ -269,10 +269,10 @@ export class HostExternalSessionCoordinator { } catch (error) { if (!commitAttempted) { return importFailure( - isSourceSessionNotFound(error) ? 'not_found' : 'invalid_request', + isSourceSessionNotFound(error) ? 'not_found' : 'source_unreadable', isSourceSessionNotFound(error) ? 'External Session does not exist' - : 'External Session could not be converted', + : 'External Session could not be read or converted', ); } this.#requestDrain(); diff --git a/packages/storage/src/__tests__/codex-session-adapter.test.ts b/packages/storage/src/__tests__/codex-session-adapter.test.ts index dfc65b4761..132ad73120 100644 --- a/packages/storage/src/__tests__/codex-session-adapter.test.ts +++ b/packages/storage/src/__tests__/codex-session-adapter.test.ts @@ -18,10 +18,10 @@ */ import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { appendFile, mkdir, mkdtemp, open, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { describe, test } from 'node:test'; +import { describe, mock, test } from 'node:test'; import { fileURLToPath } from 'node:url'; import { decodeCanonicalMessage } from '@maka/core/session'; import { CodexSessionAdapter } from '../codex-session-adapter.js'; @@ -413,7 +413,7 @@ describe('CodexSessionAdapter', () => { }); }); - test('rejects corrupt interior records, tolerates a torn tail, and bounds full reads', async () => { + test('rejects corrupt interior records, tolerates a torn tail, and bounds scanned bytes', async () => { await withCodexHome(async (codexHome) => { const fixture = await readFile(CURRENT_FIXTURE, 'utf8'); const corruptId = 'codex-corrupt'; @@ -443,6 +443,237 @@ describe('CodexSessionAdapter', () => { }); }); + test('parses a UTF-8 JSONL record split across read buffers', async () => { + await withCodexHome(async (codexHome) => { + const sessionId = 'codex-cross-buffer-utf8'; + const meta = `${JSON.stringify({ + timestamp: '2026-08-08T00:00:00.000Z', + type: 'session_meta', + payload: { + session_id: sessionId, + id: sessionId, + cwd: '/workspace/utf8', + source: 'cli', + }, + })}\n`; + const prefixBytes = Buffer.byteLength(meta, 'utf8'); + const eventTemplate = JSON.stringify({ + timestamp: '2026-08-08T00:00:01.000Z', + type: 'event_msg', + payload: { type: 'user_message', message: '__MESSAGE__' }, + }); + const [eventPrefix, eventSuffix] = eventTemplate.split('__MESSAGE__'); + assert.ok(eventPrefix !== undefined && eventSuffix !== undefined); + const paddingBytes = 64 * 1024 - prefixBytes - Buffer.byteLength(eventPrefix, 'utf8') - 1; + assert.ok(paddingBytes > 0); + const content = `${meta}${eventPrefix}${'x'.repeat(paddingBytes)}你${eventSuffix}\n`; + await seedRawRollout(codexHome, sessionId, content); + + const session = await new CodexSessionAdapter({ codexHome }).readSession(sessionId); + assert.equal(session.messages[0]?.type, 'user'); + assert.equal( + session.messages[0]?.type === 'user' ? session.messages[0].text : undefined, + `${'x'.repeat(paddingBytes)}你`, + ); + }); + }); + + test('rejects a short read before the fixed rollout snapshot is complete', async () => { + await withCodexHome(async (codexHome) => { + const sessionId = 'codex-truncated-during-read'; + const rolloutPath = await seedRawRollout( + codexHome, + sessionId, + `${minimalRollout(sessionId, '/workspace', 'Keep this message')}${JSON.stringify({ + timestamp: '2026-08-08T00:00:02.000Z', + type: 'world_state', + payload: { padding: 'x'.repeat(128 * 1024) }, + })}\n`, + ); + await seedStateDatabase(codexHome, [ + { + id: sessionId, + rolloutPath, + cwd: '/workspace', + name: 'Truncated during read', + createdAtMs: 1_000, + updatedAtMs: 2_000, + archived: false, + source: 'cli', + }, + ]); + + const probe = await open(rolloutPath, 'r'); + type PositionalRead = ( + buffer: Buffer, + offset: number, + length: number, + position: number, + ) => Promise<{ bytesRead: number; buffer: Buffer }>; + const fileHandlePrototype = Object.getPrototypeOf(probe) as { read: PositionalRead }; + const originalRead = fileHandlePrototype.read; + await probe.close(); + let readCalls = 0; + const readMock = mock.method( + fileHandlePrototype, + 'read', + async function ( + this: typeof probe, + buffer: Buffer, + offset: number, + length: number, + position: number, + ) { + readCalls += 1; + return readCalls === 2 + ? { bytesRead: 0, buffer } + : originalRead.call(this, buffer, offset, length, position); + }, + ); + try { + await assert.rejects( + new CodexSessionAdapter({ codexHome }).readSession(sessionId), + /changed while being read/, + ); + } finally { + readMock.mock.restore(); + } + }); + }); + + test('does not follow records appended after the rollout snapshot is opened', async () => { + await withCodexHome(async (codexHome) => { + const sessionId = 'codex-appended-during-read'; + const rolloutPath = await seedRawRollout( + codexHome, + sessionId, + minimalRollout(sessionId, '/workspace', 'Keep this message'), + ); + await seedStateDatabase(codexHome, [ + { + id: sessionId, + rolloutPath, + cwd: '/workspace', + name: 'Appended during read', + createdAtMs: 1_000, + updatedAtMs: 2_000, + archived: false, + source: 'cli', + }, + ]); + + const probe = await open(rolloutPath, 'r'); + type PositionalRead = ( + buffer: Buffer, + offset: number, + length: number, + position: number, + ) => Promise<{ bytesRead: number; buffer: Buffer }>; + const fileHandlePrototype = Object.getPrototypeOf(probe) as { read: PositionalRead }; + const originalRead = fileHandlePrototype.read; + await probe.close(); + let appended = false; + const readMock = mock.method( + fileHandlePrototype, + 'read', + async function ( + this: typeof probe, + buffer: Buffer, + offset: number, + length: number, + position: number, + ) { + const result = await originalRead.call(this, buffer, offset, length, position); + if (!appended) { + appended = true; + await appendFile(rolloutPath, 'not-json\n'); + } + return result; + }, + ); + try { + const session = await new CodexSessionAdapter({ codexHome }).readSession(sessionId); + assert.equal(session.messages[0]?.type, 'user'); + assert.equal( + session.messages[0]?.type === 'user' ? session.messages[0].text : undefined, + 'Keep this message', + ); + } finally { + readMock.mock.restore(); + } + }); + }); + + test('rejects an oversized JSONL record without buffering the complete rollout', async () => { + await withCodexHome(async (codexHome) => { + const sessionId = 'codex-record-limit'; + await seedMinimalRollout(codexHome, sessionId, false, '/workspace', 'hello'); + const adapter = new CodexSessionAdapter({ codexHome, maxRecordBytes: 100 }); + + await assert.rejects(adapter.readSession(sessionId), /record at line 1 exceeds 100 bytes/); + }); + }); + + test('rejects converted histories that exceed message count or byte budgets', async () => { + await withCodexHome(async (codexHome) => { + const sessionId = 'codex-converted-limits'; + await seedRawRollout( + codexHome, + sessionId, + `${minimalRollout(sessionId, '/workspace', 'hello')}${JSON.stringify({ + timestamp: '2026-08-08T00:00:02.000Z', + type: 'event_msg', + payload: { type: 'agent_message', message: 'world' }, + })}\n`, + ); + + await assert.rejects( + new CodexSessionAdapter({ codexHome, maxMessages: 1 }).readSession(sessionId), + /more than 1 messages/, + ); + await assert.rejects( + new CodexSessionAdapter({ codexHome, maxConvertedBytes: 10 }).readSession(sessionId), + /more than 10 bytes/, + ); + }); + }); + + test('streams valid rollouts larger than the legacy 64 MiB whole-file limit', async () => { + await withCodexHome(async (codexHome) => { + const sessionId = 'codex-large-streamed'; + const rolloutPath = await seedMinimalRollout( + codexHome, + sessionId, + false, + '/workspace/large', + 'Keep this message', + ); + const ignoredRecord = `${JSON.stringify({ + timestamp: '2026-08-08T00:00:02.000Z', + type: 'world_state', + payload: { padding: 'x'.repeat(1024 * 1024) }, + })}\n`; + const handle = await open(rolloutPath, 'a'); + try { + for (let index = 0; index < 65; index += 1) await handle.write(ignoredRecord); + } finally { + await handle.close(); + } + assert.ok((await stat(rolloutPath)).size > 64 * 1024 * 1024); + + const session = await new CodexSessionAdapter({ codexHome }).readSession(sessionId); + assert.deepEqual(session.messages, [ + { + type: 'user', + id: `codex-${sessionId}-user-2`, + turnId: `codex-${sessionId}-turn-2`, + ts: Date.parse('2026-08-08T00:00:01.000Z'), + text: 'Keep this message', + }, + ]); + }); + }); + test('never follows a state database rollout path outside CODEX_HOME', async () => { const outside = await mkdtemp(join(tmpdir(), 'maka-codex-outside-')); try { diff --git a/packages/storage/src/codex-session-adapter.ts b/packages/storage/src/codex-session-adapter.ts index e9aa2e982c..32a4b3700b 100644 --- a/packages/storage/src/codex-session-adapter.ts +++ b/packages/storage/src/codex-session-adapter.ts @@ -32,9 +32,13 @@ import type { } from '@maka/core/external-session'; export const CODEX_SESSION_ADAPTER_ID = 'codex'; -export const CODEX_ROLLOUT_MAX_BYTES = 64 * 1024 * 1024; +export const CODEX_ROLLOUT_MAX_BYTES = 2 * 1024 * 1024 * 1024; const CODEX_ROLLOUT_HEAD_BYTES = 512 * 1024; +const CODEX_ROLLOUT_READ_BYTES = 64 * 1024; +const CODEX_ROLLOUT_MAX_RECORD_BYTES = 64 * 1024 * 1024; +const CODEX_ROLLOUT_MAX_CONVERTED_BYTES = 256 * 1024 * 1024; +const CODEX_ROLLOUT_MAX_MESSAGES = 250_000; const CODEX_SESSION_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; const CODEX_UNSAFE_PATH_CHARS = /[\u0000-\u001F\u007F\u0080-\u009F\u061C\u200B-\u200F\u202A-\u202E\u2060-\u2064\u2066-\u2069\uFEFF]/; @@ -42,8 +46,14 @@ const CODEX_UNSAFE_PATH_CHARS = export interface CodexSessionAdapterOptions { /** Codex's state root. Defaults to `$CODEX_HOME`, then `~/.codex`. */ codexHome?: string; - /** Test/host override for the bounded transcript read. */ + /** Maximum source bytes scanned from one fixed rollout snapshot. */ maxRolloutBytes?: number; + /** Maximum bytes buffered for one JSONL record. */ + maxRecordBytes?: number; + /** Maximum serialized bytes retained across converted messages. */ + maxConvertedBytes?: number; + /** Maximum number of converted messages retained in memory. */ + maxMessages?: number; } interface CodexCatalogEntry extends ExternalSessionSummary { @@ -83,15 +93,22 @@ export class CodexSessionAdapter implements ExternalSessionAdapter { private readonly codexHome: string; private readonly maxRolloutBytes: number; + private readonly maxRecordBytes: number; + private readonly maxConvertedBytes: number; + private readonly maxMessages: number; constructor(options: CodexSessionAdapterOptions = {}) { this.codexHome = resolve( options.codexHome ?? process.env.CODEX_HOME ?? join(homedir(), '.codex'), ); this.maxRolloutBytes = options.maxRolloutBytes ?? CODEX_ROLLOUT_MAX_BYTES; - if (!Number.isSafeInteger(this.maxRolloutBytes) || this.maxRolloutBytes <= 0) { - throw new Error('Codex rollout byte limit must be a positive safe integer'); - } + this.maxRecordBytes = options.maxRecordBytes ?? CODEX_ROLLOUT_MAX_RECORD_BYTES; + this.maxConvertedBytes = options.maxConvertedBytes ?? CODEX_ROLLOUT_MAX_CONVERTED_BYTES; + this.maxMessages = options.maxMessages ?? CODEX_ROLLOUT_MAX_MESSAGES; + assertPositiveSafeInteger(this.maxRolloutBytes, 'Codex rollout byte limit'); + assertPositiveSafeInteger(this.maxRecordBytes, 'Codex rollout record byte limit'); + assertPositiveSafeInteger(this.maxConvertedBytes, 'Codex converted message byte limit'); + assertPositiveSafeInteger(this.maxMessages, 'Codex converted message count limit'); } async detect(): Promise { @@ -114,8 +131,18 @@ export class CodexSessionAdapter implements ExternalSessionAdapter { const rolloutPath = await this.resolveRolloutPath(catalogEntry.rolloutPath, sessionId); if (!rolloutPath) throw new Error(`Codex rollout is unavailable: ${sessionId}`); - const text = await readBoundedUtf8File(rolloutPath, this.maxRolloutBytes); - const converted = convertCodexRollout(text, sessionId, catalogEntry.name, catalogEntry.cwd); + const converted = await convertCodexRollout( + rolloutPath, + sessionId, + catalogEntry.name, + catalogEntry.cwd, + { + maxRolloutBytes: this.maxRolloutBytes, + maxRecordBytes: this.maxRecordBytes, + maxConvertedBytes: this.maxConvertedBytes, + maxMessages: this.maxMessages, + }, + ); return { sourceSessionId: sessionId, @@ -239,49 +266,75 @@ interface RolloutCandidate { archived: boolean; } -function convertCodexRollout( - text: string, +interface CodexRolloutLimits { + maxRolloutBytes: number; + maxRecordBytes: number; + maxConvertedBytes: number; + maxMessages: number; +} + +interface ParsedRolloutRecord { + line: number; + value: JsonRecord; +} + +async function convertCodexRollout( + path: string, expectedSessionId: string, fallbackName: string, fallbackCwd: string, -): ExternalMakaSession { - const records = parseRolloutRecords(text, expectedSessionId); - const sessionMeta = records.find((record) => record.value.type === 'session_meta')?.value; - const metaPayload = asRecord(sessionMeta?.payload); - const actualSessionId = stringField(metaPayload, 'session_id') ?? stringField(metaPayload, 'id'); - if (actualSessionId !== expectedSessionId) { - throw new Error(`Codex rollout Session id mismatch: expected ${expectedSessionId}`); + limits: CodexRolloutLimits, +): Promise { + const converter = new CodexRolloutConverter(expectedSessionId, fallbackName, fallbackCwd, limits); + for await (const record of readCodexRolloutRecords(path, expectedSessionId, limits)) { + converter.accept(record); } + return converter.finish(); +} - const metaCwd = safeCodexCwd(metaPayload?.cwd); - const messages: StoredMessage[] = []; - let activeTurnId: string | undefined; - let activeTurnIsExplicit = false; - let activeModel = stringField(metaPayload, 'model_provider') ?? 'codex'; - let lastTimestamp = normalizeEpochMs(sessionMeta?.timestamp) ?? 0; - let firstUserText: string | undefined; - const failedTurnIds = new Set(); - - const timestampFor = (record: ParsedRolloutRecord): number => { - const parsed = normalizeEpochMs(record.value.timestamp); - if (parsed !== undefined) lastTimestamp = Math.max(lastTimestamp, parsed); - else lastTimestamp += 1; - return parsed ?? lastTimestamp; - }; - const ensureTurnId = (line: number): string => { - activeTurnId ??= generatedCodexId(expectedSessionId, 'turn', line); - return activeTurnId; - }; - - for (const record of records) { +class CodexRolloutConverter { + private readonly messages: StoredMessage[] = []; + private readonly failedTurnIds = new Set(); + private activeTurnId: string | undefined; + private activeTurnIsExplicit = false; + private activeModel = 'codex'; + private lastTimestamp = 0; + private firstUserText: string | undefined; + private metaCwd = ''; + private hasSessionMeta = false; + private convertedBytes = 0; + + constructor( + private readonly expectedSessionId: string, + private readonly fallbackName: string, + private readonly fallbackCwd: string, + private readonly limits: CodexRolloutLimits, + ) {} + + accept(record: ParsedRolloutRecord): void { const envelope = record.value; + if (envelope.type === 'session_meta' && !this.hasSessionMeta) { + this.hasSessionMeta = true; + const metaPayload = asRecord(envelope.payload); + const actualSessionId = + stringField(metaPayload, 'session_id') ?? stringField(metaPayload, 'id'); + if (actualSessionId !== this.expectedSessionId) { + throw new Error(`Codex rollout Session id mismatch: expected ${this.expectedSessionId}`); + } + this.metaCwd = safeCodexCwd(metaPayload?.cwd); + this.activeModel = stringField(metaPayload, 'model_provider') ?? this.activeModel; + const timestamp = normalizeEpochMs(envelope.timestamp); + if (timestamp !== undefined) this.lastTimestamp = Math.max(this.lastTimestamp, timestamp); + return; + } + const payload = asRecord(envelope.payload); - if (!payload) continue; + if (!payload) return; if (envelope.type === 'turn_context') { - activeTurnId = stringField(payload, 'turn_id') ?? activeTurnId; - activeModel = stringField(payload, 'model') ?? activeModel; - continue; + this.activeTurnId = stringField(payload, 'turn_id') ?? this.activeTurnId; + this.activeModel = stringField(payload, 'model') ?? this.activeModel; + return; } if (envelope.type === 'event_msg') { @@ -289,10 +342,10 @@ function convertCodexRollout( if (eventType === 'task_started' || eventType === 'turn_started') { const turnId = stringField(payload, 'turn_id'); if (turnId) { - activeTurnId = turnId; - activeTurnIsExplicit = true; + this.activeTurnId = turnId; + this.activeTurnIsExplicit = true; } - continue; + return; } if (eventType === 'item_completed') { @@ -300,173 +353,173 @@ function convertCodexRollout( const itemType = stringField(item, 'type')?.toLowerCase(); const eventTurnId = stringField(payload, 'turn_id'); if (eventTurnId) { - activeTurnId = eventTurnId; - activeTurnIsExplicit = true; + this.activeTurnId = eventTurnId; + this.activeTurnIsExplicit = true; } if (itemType === 'usermessage') { - if (!activeTurnIsExplicit) { - activeTurnId = generatedCodexId(expectedSessionId, 'turn', record.line); + if (!this.activeTurnIsExplicit) { + this.activeTurnId = generatedCodexId(this.expectedSessionId, 'turn', record.line); } const text = codexCompletedItemText(item) || codexCompletedItemMediaText(item); - if (text.length === 0) continue; - firstUserText ??= text; - messages.push({ + if (text.length === 0) return; + this.firstUserText ??= text; + this.append({ type: 'user', id: stringField(item, 'client_id') ?? stringField(item, 'id') ?? - generatedCodexId(expectedSessionId, 'user', record.line), - turnId: ensureTurnId(record.line), - ts: timestampFor(record), + generatedCodexId(this.expectedSessionId, 'user', record.line), + turnId: this.ensureTurnId(record.line), + ts: this.timestampFor(record), text, }); - continue; + return; } if (itemType === 'agentmessage') { const text = codexCompletedItemText(item); - if (text.length === 0) continue; + if (text.length === 0) return; const providerOptions = codexAssistantProviderOptions(item); - messages.push({ + this.append({ type: 'assistant', id: stringField(item, 'id') ?? - generatedCodexId(expectedSessionId, 'assistant', record.line), - turnId: ensureTurnId(record.line), - ts: timestampFor(record), + generatedCodexId(this.expectedSessionId, 'assistant', record.line), + turnId: this.ensureTurnId(record.line), + ts: this.timestampFor(record), text, ...(providerOptions !== undefined ? { providerOptions } : {}), - modelId: activeModel, + modelId: this.activeModel, contentOrder: ['text'], }); - continue; + return; } if (itemType === 'reasoning') { const reasoning = codexCompletedReasoningText(item); - if (reasoning.length === 0) continue; - messages.push({ + if (reasoning.length === 0) return; + this.append({ type: 'assistant', id: stringField(item, 'id') ?? - generatedCodexId(expectedSessionId, 'reasoning', record.line), - turnId: ensureTurnId(record.line), - ts: timestampFor(record), + generatedCodexId(this.expectedSessionId, 'reasoning', record.line), + turnId: this.ensureTurnId(record.line), + ts: this.timestampFor(record), text: '', thinking: { text: reasoning }, contentOrder: ['thinking'], - modelId: activeModel, + modelId: this.activeModel, }); - continue; + return; } } if (eventType === 'user_message') { - if (!activeTurnIsExplicit) { - activeTurnId = generatedCodexId(expectedSessionId, 'turn', record.line); + if (!this.activeTurnIsExplicit) { + this.activeTurnId = generatedCodexId(this.expectedSessionId, 'turn', record.line); } const text = stringField(payload, 'message') ?? mediaOnlyUserText(payload); - if (text.length === 0) continue; - firstUserText ??= text; - const turnId = ensureTurnId(record.line); - messages.push({ + if (text.length === 0) return; + this.firstUserText ??= text; + const turnId = this.ensureTurnId(record.line); + this.append({ type: 'user', id: stringField(payload, 'client_id') ?? - generatedCodexId(expectedSessionId, 'user', record.line), + generatedCodexId(this.expectedSessionId, 'user', record.line), turnId, - ts: timestampFor(record), + ts: this.timestampFor(record), text, }); - continue; + return; } if (eventType === 'agent_message') { const text = stringField(payload, 'message'); - if (!text) continue; + if (!text) return; const providerOptions = codexAssistantProviderOptions(payload); - messages.push({ + this.append({ type: 'assistant', - id: generatedCodexId(expectedSessionId, 'assistant', record.line), - turnId: ensureTurnId(record.line), - ts: timestampFor(record), + id: generatedCodexId(this.expectedSessionId, 'assistant', record.line), + turnId: this.ensureTurnId(record.line), + ts: this.timestampFor(record), text, ...(providerOptions !== undefined ? { providerOptions } : {}), - modelId: activeModel, + modelId: this.activeModel, contentOrder: ['text'], }); - continue; + return; } if (eventType === 'agent_reasoning') { const reasoning = stringField(payload, 'text'); - if (!reasoning) continue; - messages.push({ + if (!reasoning) return; + this.append({ type: 'assistant', - id: generatedCodexId(expectedSessionId, 'reasoning', record.line), - turnId: ensureTurnId(record.line), - ts: timestampFor(record), + id: generatedCodexId(this.expectedSessionId, 'reasoning', record.line), + turnId: this.ensureTurnId(record.line), + ts: this.timestampFor(record), text: '', thinking: { text: reasoning }, contentOrder: ['thinking'], - modelId: activeModel, + modelId: this.activeModel, }); - continue; + return; } if (eventType === 'context_compacted') { - messages.push({ + this.append({ type: 'system_note', - id: generatedCodexId(expectedSessionId, 'compact', record.line), - turnId: activeTurnId, - ts: timestampFor(record), + id: generatedCodexId(this.expectedSessionId, 'compact', record.line), + turnId: this.activeTurnId, + ts: this.timestampFor(record), kind: 'context_compacted', }); - continue; + return; } if (eventType === 'error') { - if (activeTurnId && codexErrorAffectsTurnStatus(payload)) { - failedTurnIds.add(activeTurnId); + if (this.activeTurnId && codexErrorAffectsTurnStatus(payload)) { + this.failedTurnIds.add(this.activeTurnId); } - messages.push({ + this.append({ type: 'system_note', - id: generatedCodexId(expectedSessionId, 'error', record.line), - turnId: activeTurnId, - ts: timestampFor(record), + id: generatedCodexId(this.expectedSessionId, 'error', record.line), + turnId: this.activeTurnId, + ts: this.timestampFor(record), kind: 'error', data: JSON.parse(JSON.stringify(payload)) as unknown, }); - continue; + return; } if (eventType === 'task_complete' || eventType === 'turn_complete') { - const turnId = stringField(payload, 'turn_id') ?? ensureTurnId(record.line); - const failed = failedTurnIds.has(turnId) || payload.error != null; - messages.push({ + const turnId = stringField(payload, 'turn_id') ?? this.ensureTurnId(record.line); + const failed = this.failedTurnIds.has(turnId) || payload.error != null; + this.append({ type: 'turn_state', - id: generatedCodexId(expectedSessionId, 'turn-state', record.line), + id: generatedCodexId(this.expectedSessionId, 'turn-state', record.line), turnId, - ts: timestampFor(record), + ts: this.timestampFor(record), status: failed ? 'failed' : 'completed', ...(failed ? { errorClass: 'codex_error' } : {}), partialOutputRetained: true, }); - failedTurnIds.delete(turnId); - if (activeTurnId === turnId) { - activeTurnId = undefined; - activeTurnIsExplicit = false; + this.failedTurnIds.delete(turnId); + if (this.activeTurnId === turnId) { + this.activeTurnId = undefined; + this.activeTurnIsExplicit = false; } - continue; + return; } if (eventType === 'turn_aborted') { - const turnId = stringField(payload, 'turn_id') ?? ensureTurnId(record.line); - const ts = timestampFor(record); - messages.push({ + const turnId = stringField(payload, 'turn_id') ?? this.ensureTurnId(record.line); + const ts = this.timestampFor(record); + this.append({ type: 'turn_state', - id: generatedCodexId(expectedSessionId, 'turn-state', record.line), + id: generatedCodexId(this.expectedSessionId, 'turn-state', record.line), turnId, ts, status: 'aborted', @@ -474,45 +527,45 @@ function convertCodexRollout( abortSource: stringField(payload, 'reason') ?? 'codex', partialOutputRetained: true, }); - if (activeTurnId === turnId) { - activeTurnId = undefined; - activeTurnIsExplicit = false; + if (this.activeTurnId === turnId) { + this.activeTurnId = undefined; + this.activeTurnIsExplicit = false; } - continue; + return; } } - if (envelope.type !== 'response_item') continue; + if (envelope.type !== 'response_item') return; const itemType = stringField(payload, 'type'); if (itemType === 'function_call' || itemType === 'custom_tool_call') { const callId = stringField(payload, 'call_id'); const toolName = namespacedToolName(payload); - if (!callId || !toolName) continue; + if (!callId || !toolName) return; const rawArgs = itemType === 'function_call' ? stringField(payload, 'arguments') : stringField(payload, 'input'); - messages.push({ + this.append({ type: 'tool_call', id: callId, - turnId: ensureTurnId(record.line), - ts: timestampFor(record), + turnId: this.ensureTurnId(record.line), + ts: this.timestampFor(record), toolName, args: parseJsonString(rawArgs), }); - continue; + return; } if (itemType === 'function_call_output' || itemType === 'custom_tool_call_output') { const callId = stringField(payload, 'call_id'); - if (!callId) continue; - messages.push({ + if (!callId) return; + this.append({ type: 'tool_result', id: stringField(payload, 'id') ?? - generatedCodexId(expectedSessionId, 'tool-result', record.line), - turnId: ensureTurnId(record.line), - ts: timestampFor(record), + generatedCodexId(this.expectedSessionId, 'tool-result', record.line), + turnId: this.ensureTurnId(record.line), + ts: this.timestampFor(record), toolUseId: callId, // Codex persists the output body but not FunctionCallOutputPayload.success. // Preserve the raw body and avoid guessing failure from its text. @@ -522,39 +575,146 @@ function convertCodexRollout( } } - const name = - sanitizeForeignTitle(fallbackName) || sanitizeForeignTitle(firstUserText) || expectedSessionId; - return { - sourceSessionId: expectedSessionId, - metadata: { name, cwd: metaCwd || fallbackCwd }, - messages, - }; -} + finish(): ExternalMakaSession { + if (!this.hasSessionMeta) { + throw new Error(`Codex rollout Session id mismatch: expected ${this.expectedSessionId}`); + } + const name = + sanitizeForeignTitle(this.fallbackName) || + sanitizeForeignTitle(this.firstUserText) || + this.expectedSessionId; + return { + sourceSessionId: this.expectedSessionId, + metadata: { name, cwd: this.metaCwd || this.fallbackCwd }, + messages: this.messages, + }; + } -interface ParsedRolloutRecord { - line: number; - value: JsonRecord; + private timestampFor(record: ParsedRolloutRecord): number { + const parsed = normalizeEpochMs(record.value.timestamp); + if (parsed !== undefined) this.lastTimestamp = Math.max(this.lastTimestamp, parsed); + else this.lastTimestamp += 1; + return parsed ?? this.lastTimestamp; + } + + private ensureTurnId(line: number): string { + this.activeTurnId ??= generatedCodexId(this.expectedSessionId, 'turn', line); + return this.activeTurnId; + } + + private append(message: StoredMessage): void { + if (this.messages.length >= this.limits.maxMessages) { + throw new Error(`Codex rollout converts to more than ${this.limits.maxMessages} messages`); + } + const encodedBytes = Buffer.byteLength(JSON.stringify(message), 'utf8'); + if (encodedBytes > this.limits.maxConvertedBytes - this.convertedBytes) { + throw new Error(`Codex rollout converts to more than ${this.limits.maxConvertedBytes} bytes`); + } + this.convertedBytes += encodedBytes; + this.messages.push(message); + } } -function parseRolloutRecords(text: string, sessionId: string): ParsedRolloutRecord[] { - const endsWithNewline = text.endsWith('\n'); - const lines = text.split('\n'); - if (endsWithNewline) lines.pop(); - const records: ParsedRolloutRecord[] = []; - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]!; - if (line.trim().length === 0) continue; - try { - const value = JSON.parse(line) as unknown; - if (!isRecord(value)) throw new Error('record is not an object'); - records.push({ line: index + 1, value }); - } catch (error) { - if (!endsWithNewline && index === lines.length - 1) break; - const detail = error instanceof Error ? error.message : String(error); - throw new Error(`Invalid Codex rollout ${sessionId} at line ${index + 1}: ${detail}`); +async function* readCodexRolloutRecords( + path: string, + sessionId: string, + limits: CodexRolloutLimits, +): AsyncGenerator { + const handle = await open(path, 'r'); + try { + const metadata = await handle.stat(); + if (!metadata.isFile()) throw new Error('Codex rollout is not a regular file'); + if (metadata.size > limits.maxRolloutBytes) { + throw new Error(`Codex rollout exceeds ${limits.maxRolloutBytes} bytes`); + } + + const snapshotBytes = metadata.size; + const pending: Buffer[] = []; + let pendingBytes = 0; + let position = 0; + let line = 0; + while (position < snapshotBytes) { + const buffer = Buffer.allocUnsafe( + Math.min(CODEX_ROLLOUT_READ_BYTES, snapshotBytes - position), + ); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, position); + if (bytesRead === 0) { + throw new Error('Codex rollout changed while being read'); + } + position += bytesRead; + const chunk = buffer.subarray(0, bytesRead); + let start = 0; + for (;;) { + const newline = chunk.indexOf(0x0a, start); + if (newline === -1) break; + const segment = chunk.subarray(start, newline); + assertCodexRecordSize(pendingBytes + segment.byteLength, limits.maxRecordBytes, line + 1); + line += 1; + const record = parseCodexRolloutLine( + pending.length === 0 + ? segment + : Buffer.concat([...pending, segment], pendingBytes + segment.byteLength), + sessionId, + line, + false, + ); + if (record) yield record; + pending.length = 0; + pendingBytes = 0; + start = newline + 1; + } + if (start < chunk.byteLength) { + const segment = chunk.subarray(start); + assertCodexRecordSize(pendingBytes + segment.byteLength, limits.maxRecordBytes, line + 1); + pending.push(segment); + pendingBytes += segment.byteLength; + } } + + if (pendingBytes > 0) { + line += 1; + const record = parseCodexRolloutLine( + pending.length === 1 ? pending[0]! : Buffer.concat(pending, pendingBytes), + sessionId, + line, + true, + ); + if (record) yield record; + } + } finally { + await handle.close(); + } +} + +function parseCodexRolloutLine( + bytes: Buffer, + sessionId: string, + line: number, + tolerateTornTail: boolean, +): ParsedRolloutRecord | undefined { + const text = bytes.toString('utf8'); + if (text.trim().length === 0) return undefined; + try { + const value = JSON.parse(text) as unknown; + if (!isRecord(value)) throw new Error('record is not an object'); + return { line, value }; + } catch (error) { + if (tolerateTornTail) return undefined; + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`Invalid Codex rollout ${sessionId} at line ${line}: ${detail}`); + } +} + +function assertCodexRecordSize(actualBytes: number, maxBytes: number, line: number): void { + if (actualBytes > maxBytes) { + throw new Error(`Codex rollout record at line ${line} exceeds ${maxBytes} bytes`); + } +} + +function assertPositiveSafeInteger(value: number, label: string): void { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${label} must be a positive safe integer`); } - return records; } function catalogEntryFromRolloutHead( @@ -723,35 +883,6 @@ async function walkRolloutFiles(root: string, archived: boolean): Promise { - const handle = await open(path, 'r'); - try { - const metadata = await handle.stat(); - if (!metadata.isFile()) throw new Error('Codex rollout is not a regular file'); - if (metadata.size > maxBytes) throw new Error(`Codex rollout exceeds ${maxBytes} bytes`); - const chunks: Buffer[] = []; - let total = 0; - for (;;) { - const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, maxBytes + 1 - total)); - const { bytesRead } = await handle.read(buffer, 0, buffer.length, total); - if (bytesRead === 0) break; - total += bytesRead; - if (total > maxBytes) throw new Error(`Codex rollout exceeds ${maxBytes} bytes`); - chunks.push(buffer.subarray(0, bytesRead)); - if (total === maxBytes) { - const probe = Buffer.allocUnsafe(1); - if ((await handle.read(probe, 0, 1, total)).bytesRead > 0) { - throw new Error(`Codex rollout exceeds ${maxBytes} bytes`); - } - break; - } - } - return Buffer.concat(chunks, total).toString('utf8'); - } finally { - await handle.close(); - } -} - async function readUtf8Prefix(path: string, maxBytes: number): Promise { const handle = await open(path, 'r'); try {