From 3cd26c2deef6e7f61793104fba5f16e24fb29dad Mon Sep 17 00:00:00 2001 From: Greg Pazo Date: Mon, 26 Oct 2026 20:21:16 -0700 Subject: [PATCH 1/5] Support offline Bedrock cold checkpoint verification --- docs/agents-api.md | 6 +- runtime/codex/README.md | 8 +-- src/runner/codex-cold-verification.ts | 25 ++++++-- tests/runner/codex-cold-verification.test.ts | 64 ++++++++++++++++---- 4 files changed, 77 insertions(+), 26 deletions(-) diff --git a/docs/agents-api.md b/docs/agents-api.md index 8588e37..e213a00 100644 --- a/docs/agents-api.md +++ b/docs/agents-api.md @@ -410,9 +410,9 @@ resume. Verification has no network or credentials and cannot run model work or tools. Extra coordination details omitted by Codex's public history remain in the host journal. This does not verify Codex's hidden mailbox state. -Codex cold checkpoints currently support the built-in OpenAI provider and custom -Responses API providers. Bedrock, Ollama and LM Studio are refused before the -running process stops because their cold recovery has not been validated. This +Codex cold checkpoints support the built-in OpenAI and Amazon Bedrock providers +and custom Responses API providers. Ollama and LM Studio are refused before the +running process stops because their cold recovery is unsupported. This restriction does not prevent ordinary Sessions or Pi Durable execution. Checkpointing refuses unsupported startup integrations, external native storage, diff --git a/runtime/codex/README.md b/runtime/codex/README.md index 9d31b52..daef5e1 100644 --- a/runtime/codex/README.md +++ b/runtime/codex/README.md @@ -70,10 +70,10 @@ verification must fail the handoff; it must not silently reconstruct a prompt or start a new native thread. The opt-in AWS checkpoint harness remains necessary before claiming cloud worker replacement acceptance. -The cold verifier supports the built-in OpenAI provider and custom Responses API -providers. It rejects Bedrock, Ollama and LM Studio before stopping the live -process until their recovery paths are validated; ordinary Sessions and Pi -Durable execution are unaffected. +The cold verifier supports the built-in OpenAI and Amazon Bedrock providers and +custom Responses API providers. It rejects Ollama and LM Studio before stopping +the live process because their recovery paths are unsupported; ordinary Sessions +and Pi Durable execution are unaffected. ## Upgrade and rollback diff --git a/src/runner/codex-cold-verification.ts b/src/runner/codex-cold-verification.ts index 0d42bed..dee58af 100644 --- a/src/runner/codex-cold-verification.ts +++ b/src/runner/codex-cold-verification.ts @@ -33,7 +33,7 @@ export function assertCheckpointStartupSafe(value: unknown, sessionConfig: unkno export async function prepareCodexColdVerification(options: { request: CodexAppServerRequest; stateRoot: string; manifest: NativeCheckpointManifest }): Promise { const { request, stateRoot, manifest } = options; if (manifest.threads.some(thread => thread.cwd !== '/workspace' || typeof thread.modelProvider !== 'string' || !/^[a-zA-Z0-9_-]{1,100}$/.test(thread.modelProvider))) throw new Error('Cold native checkpoint thread paths or providers are unsupported'); - if (manifest.threads.some(thread => ['amazon-bedrock', 'amazon-bedrock-runtime', 'ollama', 'lmstudio'].includes(String(thread.modelProvider)))) throw new Error('Cold native checkpoint provider is not yet supported'); + if (manifest.threads.some(thread => ['ollama', 'lmstudio'].includes(String(thread.modelProvider)))) throw new Error('Cold native checkpoint provider is not yet supported'); const uid = request.identity?.uid; const gid = request.identity?.gid; const home = request.environment.CODEX_HOME; @@ -68,7 +68,7 @@ export async function prepareCodexColdVerification(options: { request: CodexAppS await mkdir(disposable, { mode: 0o711 }); await ownedDirectory(join(disposable, 'home'), identity); await ownedDirectory(join(disposable, 'workspace'), identity); - const args = namespaceArguments({ binary, home, sqlite, workspace: request.workspace, disposable, identity }); + const args = namespaceArguments({ binary, bwrap, home, sqlite, workspace: request.workspace, disposable, identity }); // Check namespace support before sealing durable authority or stopping Codex. await checkedProcess(bwrap, [...args, '--', '/usr/bin/true'], identity); return { @@ -92,13 +92,16 @@ export async function prepareCodexColdVerification(options: { request: CodexAppS await writeFile(join(disposable, 'home', 'config.toml'), verifierConfiguration(manifest), { mode: 0o600 }); await chown(join(disposable, 'home', 'config.toml'), identity.uid, identity.gid); let activity = false; + let configurationFailure: Error | undefined; rpc = new CodexRpcClient({ binary: bwrap, binaryArguments: [...args, '--', binary, 'app-server'], cwd: '/', environment: {}, identity, onEvent: event => { + if (event.method === 'configWarning') configurationFailure ??= new Error('Cold native verifier configuration is invalid'); if (event.requestId !== undefined || /^(turn\/(started|completed)|item\/|error$)/.test(event.method)) activity = true; }, onServerRequest: async () => { activity = true; throw new Error('Cold native verifier cannot perform tools'); }, }); try { await rpc.initialize(); + if (configurationFailure) throw configurationFailure; for (const expected of manifest.threads) { const result = await rpc.call('thread/read', { threadId: expected.id, includeTurns: true }); assertNativeHistory(expected, record(result) ? result.thread : undefined, String(expected.id)); @@ -112,25 +115,30 @@ export async function prepareCodexColdVerification(options: { request: CodexAppS } await rpc.closeCheckpoint(); await stopCheckpointWriters(identity.uid); + if (configurationFailure) throw configurationFailure; if (activity) throw new Error('Cold native verification attempted activity'); if (await treeDigest(candidate) !== candidateDigest) throw new Error('Cold native candidate changed during verification'); return nativeCheckpointEvidence(manifest); + } catch (error) { + // A fallback warning can arrive before a native request also fails to load config. + throw configurationFailure ?? error; } finally { await rpc.close(); await stopCheckpointWriters(identity.uid); } }, }; } catch (error) { await close(); throw error; } } -function namespaceArguments(options: { binary: string; home: string; sqlite: string | undefined; workspace: string; disposable: string; identity: { uid: number; gid: number } }): string[] { - const { binary, home, sqlite, workspace, disposable } = options; +function namespaceArguments(options: { binary: string; bwrap: string; home: string; sqlite: string | undefined; workspace: string; disposable: string; identity: { uid: number; gid: number } }): string[] { + const { binary, bwrap, home, sqlite, workspace, disposable } = options; return ['--unshare-all', '--die-with-parent', '--new-session', '--cap-drop', 'ALL', '--clearenv', '--ro-bind', '/usr', '/usr', '--symlink', 'usr/bin', '/bin', '--symlink', 'usr/lib', '/lib', '--symlink', 'usr/lib64', '/lib64', '--proc', '/proc', '--dev', '/dev', '--tmpfs', '/tmp', '--dir', '/home/agent', '--ro-bind', binary, binary, + '--dir', '/rat-cold-bin', '--ro-bind', bwrap, '/rat-cold-bin/bwrap', '--bind', join(disposable, 'home'), home, '--ro-bind', join(disposable, 'workspace'), workspace, ...(workspace === '/workspace' ? [] : ['--ro-bind', join(disposable, 'workspace'), '/workspace']), - '--setenv', 'PATH', '/usr/bin:/bin', '--setenv', 'HOME', '/home/agent', '--setenv', 'CODEX_HOME', home, + '--setenv', 'PATH', '/rat-cold-bin:/usr/bin:/bin', '--setenv', 'HOME', '/home/agent', '--setenv', 'CODEX_HOME', home, ...(sqlite ? ['--setenv', 'CODEX_SQLITE_HOME', sqlite] : []), '--setenv', 'AWS_EC2_METADATA_DISABLED', 'true', '--setenv', 'AWS_CONFIG_FILE', '/dev/null', '--setenv', 'AWS_SHARED_CREDENTIALS_FILE', '/dev/null', '--setenv', 'LOG_FORMAT', 'json', '--setenv', 'RUST_LOG', 'warn', '--chdir', '/workspace']; @@ -138,7 +146,12 @@ function namespaceArguments(options: { binary: string; home: string; sqlite: str function verifierConfiguration(manifest: NativeCheckpointManifest): string { // Keep built-in OpenAI intact. Its prewarm has no credentials and can only // reach a closed loopback address in the private network namespace. - const providers = [...new Set(manifest.threads.map(thread => String(thread.modelProvider)))].filter(provider => provider !== 'openai').map(provider => `\n[model_providers.${JSON.stringify(provider)}]\nname = "Offline checkpoint verifier"\nbase_url = "http://127.0.0.1:1"\nwire_api = "responses"\nrequires_openai_auth = false\nsupports_websockets = false\nrequest_max_retries = 0\nstream_max_retries = 0\n`).join(''); + // Built-in Bedrock permits endpoint overrides, not generic provider replacement. + // Its auth prewarm may fail without credentials; native read/resume remains local. + const providers = [...new Set(manifest.threads.map(thread => String(thread.modelProvider)))].filter(provider => provider !== 'openai').map(provider => + ['amazon-bedrock', 'amazon-bedrock-runtime'].includes(provider) + ? `\n[model_providers.${JSON.stringify(provider)}]\nbase_url = "http://127.0.0.1:1"\n` + : `\n[model_providers.${JSON.stringify(provider)}]\nname = "Offline checkpoint verifier"\nbase_url = "http://127.0.0.1:1"\nwire_api = "responses"\nrequires_openai_auth = false\nsupports_websockets = false\nrequest_max_retries = 0\nstream_max_retries = 0\n`).join(''); return 'openai_base_url = "http://127.0.0.1:1"\napproval_policy = "never"\nsandbox_mode = "read-only"\ncli_auth_credentials_store = "file"\nnotify = []\nweb_search = "disabled"\n[features]\nplugins = false\napps = false\nhooks = false\ngoals = false\nmulti_agent = true\nmulti_agent_v2 = true\nresponses_websockets = false\nresponses_websockets_v2 = false\ncode_mode_prewarm = false\nshell_snapshot = false\nshell_snapshot_v2 = false\n' + providers; } function within(root: string, path: string): boolean { const part = relative(root, path); return isAbsolute(root) && isAbsolute(path) && part !== '' && part !== '..' && !part.startsWith('../') && !isAbsolute(part); } diff --git a/tests/runner/codex-cold-verification.test.ts b/tests/runner/codex-cold-verification.test.ts index 4a13984..3f4f913 100644 --- a/tests/runner/codex-cold-verification.test.ts +++ b/tests/runner/codex-cold-verification.test.ts @@ -5,13 +5,24 @@ import { chown, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs import { join, resolve } from 'node:path'; import { tmpdir } from 'node:os'; import { setTimeout as delay } from 'node:timers/promises'; -import { expect, it } from 'vitest'; +import { expect, it, vi } from 'vitest'; import { assertCheckpointStartupSafe, prepareCodexColdVerification } from '../../src/runner/codex-cold-verification.js'; import { CodexRpcClient } from '../../src/adapters/codex-rpc.js'; import { nativeThreadHistory, type NativeCheckpointManifest } from '../../src/domain/native-checkpoint.js'; import { stopCheckpointWriters } from '../../src/runner/checkpoint-writers.js'; import { syncWorkspaceStorage } from '../../src/runner/workspace-sync.js'; +const invalidVerifierConfig = vi.hoisted(() => ({ enabled: false })); +vi.mock('node:fs/promises', async importOriginal => { + const original = await importOriginal(); + return { ...original, writeFile: async (...args: Parameters) => { + const [path, data, options] = args; + // Reproduce upstream's reserved-provider rejection at the config write boundary. + const invalid = invalidVerifierConfig.enabled && typeof path === 'string' && path.endsWith('/verify/home/config.toml') && typeof data === 'string'; + return original.writeFile(path, invalid ? `${data}\nname = "Invalid Bedrock override"\n` : data, options); + } }; +}); + it.each(['hooks', 'mcp_servers', 'plugins', 'notify', 'apps', 'browser_use', 'computer_use', 'desktop'])('rejects active %s before shutdown', key => { expect(() => assertCheckpointStartupSafe({ config: { [key]: { enabled: true } } }, {})).toThrow('startup integrations'); expect(() => assertCheckpointStartupSafe({ config: {} }, { [key]: ['active'] })).toThrow('startup integrations'); @@ -24,8 +35,13 @@ it('accepts absent defaults but rejects unknown native storage configuration', ( // Run this in the disposable worker image as root, with its dedicated agent UID. // It must never run in the user's host service or against their native home. const guest = it.skipIf(process.env.CODEX_COLD_NATIVE_E2E !== 'true'); -guest.each(['legacy', 'paginated'] as const)('verifies real root and child histories inside the offline namespace (%s)', async historyMode => { - const f = await guestFixture(historyMode); +const guestCases = [ + { modelProvider: 'openai', model: 'gpt-5.5' }, + { modelProvider: 'amazon-bedrock', model: 'openai.gpt-5.6-terra' }, + { modelProvider: 'amazon-bedrock-runtime', model: 'global.openai.gpt-5.6-terra' }, +].flatMap(provider => (['legacy', 'paginated'] as const).map(historyMode => ({ ...provider, historyMode }))); +guest.each(guestCases)('verifies real root and child histories inside the offline namespace ($modelProvider/$historyMode)', async fixture => { + const f = await guestFixture(fixture); try { const stages = new Set((await readdir(tmpdir())).filter(name => name.startsWith('rat-cold-native-'))); const verifier = await prepareCodexColdVerification({ request: f.request, stateRoot: f.root, manifest: f.manifest }); @@ -38,20 +54,20 @@ guest.each(['legacy', 'paginated'] as const)('verifies real root and child histo await writeFile(join(f.home, 'auth.json'), '{"access_token":"SYNTHETIC_DO_NOT_USE"}'); const before = f.requests(); const evidence = await verifier.verify(f.manifest); - expect(evidence).toMatchObject({ kind: 'codex-cold-native', threadIds: f.manifest.threads.map(thread => thread.id), historyModes: historyMode === 'legacy' ? ['legacy', 'paginated'] : ['paginated'] }); + expect(evidence).toMatchObject({ kind: 'codex-cold-native', threadIds: f.manifest.threads.map(thread => thread.id), historyModes: fixture.historyMode === 'legacy' ? ['legacy', 'paginated'] : ['paginated'] }); expect(f.requests()).toBe(before); expect(await readFile(join(f.home, 'auth.json'), 'utf8')).toContain('SYNTHETIC_DO_NOT_USE'); } finally { await verifier.close(); } } finally { await f.close(); } }, 60_000); -guest.each(['legacy', 'paginated'] as const)('rejects a corrupt stopped candidate without removing its source (%s)', async historyMode => { - const f = await guestFixture(historyMode); +guest.each(guestCases)('rejects a corrupt stopped candidate without removing its source ($modelProvider/$historyMode)', async fixture => { + const f = await guestFixture(fixture); try { const verifier = await prepareCodexColdVerification({ request: f.request, stateRoot: f.root, manifest: f.manifest }); try { await f.stop(); - if (historyMode === 'legacy') { + if (fixture.historyMode === 'legacy') { for (const path of await files(join(f.home, 'sessions'))) if (path.endsWith('.jsonl')) await writeFile(path, ''); } else { for (const path of await files(f.home)) if (/thread_history_\d+\.sqlite$/.test(path)) { @@ -59,13 +75,30 @@ guest.each(['legacy', 'paginated'] as const)('rejects a corrupt stopped candidat } } await syncWorkspaceStorage(f.root); + const before = f.requests(); await expect(verifier.verify(f.manifest)).rejects.toThrow(); + expect(f.requests()).toBe(before); expect((await readdir(f.home)).length).toBeGreaterThan(0); } finally { await verifier.close(); } } finally { await f.close(); } }, 60_000); -async function guestFixture(historyMode: 'legacy' | 'paginated') { +guest('rejects reserved-provider configuration fallback before accepting native recovery', async () => { + const f = await guestFixture({ modelProvider: 'amazon-bedrock', model: 'openai.gpt-5.6-terra', historyMode: 'legacy' }); + try { + const verifier = await prepareCodexColdVerification({ request: f.request, stateRoot: f.root, manifest: f.manifest }); + try { + await f.stop(); + invalidVerifierConfig.enabled = true; + const before = f.requests(); + await expect(verifier.verify(f.manifest)).rejects.toThrow('Cold native verifier configuration is invalid'); + expect(f.requests()).toBe(before); + expect((await readdir(f.home)).length).toBeGreaterThan(0); + } finally { invalidVerifierConfig.enabled = false; await verifier.close(); } + } finally { await f.close(); } +}, 60_000); + +async function guestFixture({ historyMode, modelProvider, model }: (typeof guestCases)[number]) { if (process.getuid?.() !== 0) throw new Error('Run CODEX_COLD_NATIVE_E2E only as root in the disposable worker container'); const root = await mkdtemp(join(tmpdir(), 'rat-native-cold-guest-')); const home = join(root, 'codex-home'); const workspace = join(root, 'workspace'); @@ -93,8 +126,12 @@ async function guestFixture(historyMode: 'legacy' | 'paginated') { const address = server.address(); if (!address || typeof address === 'string') throw new Error('Missing fixture port'); const completed = new Set(); let childId: string | undefined; let childActivityDone = false; const binary = process.env.CODEX_CONFORMANCE_BINARY ?? resolve('node_modules/.bin/codex'); - const rpc = new CodexRpcClient({ binary, cwd: workspace, environment: { PATH: '/usr/bin:/bin', HOME: home, CODEX_HOME: home, CODEX_API_KEY: 'synthetic-loopback-fixture-only' }, identity, - binaryArguments: ['-c', `openai_base_url="http://127.0.0.1:${address.port}"`, '-c', 'features.responses_websockets=false', '-c', 'features.responses_websockets_v2=false', 'app-server'], + const providerConfig = { [modelProvider === 'openai' ? 'openai_base_url' : `model_providers.${modelProvider}.base_url`]: `http://127.0.0.1:${address.port}` }; + const rpc = new CodexRpcClient({ binary, cwd: workspace, environment: { PATH: '/usr/bin:/bin', HOME: home, CODEX_HOME: home, + AWS_EC2_METADATA_DISABLED: 'true', AWS_CONFIG_FILE: '/dev/null', AWS_SHARED_CREDENTIALS_FILE: '/dev/null', + ...(modelProvider === 'openai' ? { CODEX_API_KEY: 'synthetic-loopback-fixture-only' } : { AWS_BEARER_TOKEN_BEDROCK: 'synthetic-loopback-fixture-only', AWS_REGION: 'us-east-1' }) }, identity, + binaryArguments: ['-c', `model_provider=${JSON.stringify(modelProvider)}`, ...Object.entries(providerConfig).flatMap(([key, value]) => ['-c', `${key}=${JSON.stringify(value)}`]), + '-c', 'features.responses_websockets=false', '-c', 'features.responses_websockets_v2=false', 'app-server'], onEvent: event => { const value = event.params.item as { type?: string; agentThreadId?: string; kind?: string } | undefined; if (value?.type === 'subAgentActivity' && value.agentThreadId) { childId = value.agentThreadId; if (value.kind === 'completed') childActivityDone = true; } @@ -105,8 +142,8 @@ async function guestFixture(historyMode: 'legacy' | 'paginated') { try { await rpc.initialize().catch(error => { throw new Error('Source fixture initialize failed', { cause: error }); }); const call = (method: string, params: unknown) => rpc.call(method, params).catch(error => { throw new Error(`Source fixture ${method} failed`, { cause: error }); }); - const started = await call('thread/start', { cwd: '/workspace', modelProvider: 'openai', model: 'gpt-5.5', historyMode, approvalPolicy: 'never', sandbox: 'read-only', ephemeral: false, - config: { openai_base_url: `http://127.0.0.1:${address.port}`, 'features.responses_websockets': false, 'features.responses_websockets_v2': false, + const started = await call('thread/start', { cwd: '/workspace', modelProvider, model, historyMode, approvalPolicy: 'never', sandbox: 'read-only', ephemeral: false, + config: { ...providerConfig, 'features.responses_websockets': false, 'features.responses_websockets_v2': false, 'features.multi_agent': true, 'features.multi_agent_v2': true, 'features.plugins': false, 'features.apps': false, 'features.hooks': false, 'features.goals': false } }) as { thread: { id: string } }; await call('turn/start', { threadId: started.thread.id, input: [{ type: 'text', text: 'ROOT_INPUT: delegate one child.' }] }); for (let count = 0; count < 1500 && (!childId || !completed.has(started.thread.id) || !completed.has(childId) || !childActivityDone); count++) await delay(10); @@ -116,8 +153,9 @@ async function guestFixture(historyMode: 'legacy' | 'paginated') { const result = await call('thread/read', { threadId, includeTurns: true }) as { thread: unknown }; threads.push(nativeThreadHistory(result.thread)); } + expect(threads.map(thread => thread.modelProvider)).toEqual([modelProvider, modelProvider]); const manifest: NativeCheckpointManifest = { rootThreadId: started.thread.id, hostJournalDigest: 'a'.repeat(64), threads }; - const request = { binary, workspace, executionWorkspace: '/workspace', environment: { CODEX_HOME: home }, identity, timeoutMs: 30_000, prompt: '', sandbox: 'read-only' as const, persistent: true, modelProvider: 'openai', networkAccess: false, + const request = { binary, workspace, executionWorkspace: '/workspace', environment: { CODEX_HOME: home }, identity, timeoutMs: 30_000, prompt: '', sandbox: 'read-only' as const, persistent: true, modelProvider, networkAccess: false, managedPolicy: { workspace: '/workspace', network: { access: 'disabled' as const, allowed_domains: [] }, credentialProxy: false } }; return { root, home, request, manifest, requests: () => requests, close, stop: async () => { if (stopped) return; await rpc.closeCheckpoint(); await stopCheckpointWriters(identity.uid); await syncWorkspaceStorage(root); stopped = true; } }; From a6f054c495aedb10d3f89ca94283c88c7b0bf640 Mon Sep 17 00:00:00 2001 From: Greg Pazo Date: Mon, 26 Oct 2026 21:12:34 -0700 Subject: [PATCH 2/5] Prove stock Codex network isolation from kernel namespace evidence --- testing/aws/egress-proof.ts | 64 ++++++++++++++++- tests/scripts/egress-proof.test.ts | 110 ++++++++++++++++++++++++++++- 2 files changed, 168 insertions(+), 6 deletions(-) diff --git a/testing/aws/egress-proof.ts b/testing/aws/egress-proof.ts index a85dca0..894ff56 100644 --- a/testing/aws/egress-proof.ts +++ b/testing/aws/egress-proof.ts @@ -24,6 +24,13 @@ export interface EgressReceipt { euid: number; input_sha256: string; program_sha256: string; + network_namespace?: { + id: string | null; + devices: string | null; + ipv4_routes: string | null; + ipv6_routes: string | null; + error: string | null; + }; public_tcp: { host: string; port: number; connected: boolean; error: string | null }; bedrock: { host: string; @@ -47,6 +54,20 @@ export function egressProbeProgram(spec: EgressProbeSpec): string { ' raise TimeoutError("DNS deadline")', 'def private_ipv4(address):', ' return any(ipaddress.ip_address(address) in ipaddress.ip_network(network) for network in ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"])', + 'def network_file(path):', + ' with open(path, "r", encoding="ascii") as source:', + ' value = source.read(2049)', + ' if len(value) > 2048:', + ' raise ValueError("Network inventory exceeds its bound")', + ' return value', + 'namespace = {"id": None, "devices": None, "ipv4_routes": None, "ipv6_routes": None, "error": None}', + 'try:', + ' namespace["id"] = os.readlink("/proc/self/ns/net")', + ' namespace["devices"] = network_file("/proc/net/dev")', + ' namespace["ipv4_routes"] = network_file("/proc/net/route")', + ' namespace["ipv6_routes"] = network_file("/proc/net/ipv6_route")', + 'except (OSError, ValueError) as error:', + ' namespace["error"] = error_code(error)', 'public = {"host": config["public_host"], "port": config["port"], "connected": False, "error": None}', 'try:', ' with socket.create_connection((public["host"], public["port"]), timeout=4):', @@ -75,7 +96,7 @@ export function egressProbeProgram(spec: EgressProbeSpec): string { ' input_sha256 = hashlib.sha256(source.read()).hexdigest()', 'with open(__file__, "rb") as source:', ' program_sha256 = hashlib.sha256(source.read()).hexdigest()', - 'receipt = {"schema": 1, "marker": config["marker"], "uid": os.getuid(), "euid": os.geteuid(), "input_sha256": input_sha256, "program_sha256": program_sha256, "public_tcp": public, "bedrock": bedrock}', + 'receipt = {"schema": 1, "marker": config["marker"], "uid": os.getuid(), "euid": os.geteuid(), "input_sha256": input_sha256, "program_sha256": program_sha256, "network_namespace": namespace, "public_tcp": public, "bedrock": bedrock}', 'serialized = json.dumps(receipt, sort_keys=True, separators=(",", ":"))', 'os.makedirs("/workspace/outputs", exist_ok=True)', 'with open(config["output_path"], "x") as output:', @@ -145,8 +166,9 @@ export function egressTurnEvidence(input: { assert(receipt.bedrock.tls_version?.startsWith('TLSv'), 'Bedrock must negotiate TLS'); assert.equal(receipt.bedrock.error, null); } else { - assert(receipt.public_tcp.error === 'PermissionError:1' || receipt.public_tcp.error === 'PermissionError:13', - 'Default guest network policy must explicitly deny the public numeric TCP probe'); + assert(receipt.public_tcp.error === 'PermissionError:1' || receipt.public_tcp.error === 'PermissionError:13' + || receipt.public_tcp.error === 'OSError:101' && isolatedNetworkNamespace(receipt.network_namespace), + 'Default guest network policy must explicitly deny the public numeric TCP probe or prove an isolated loopback namespace'); assert.equal(receipt.bedrock.tcp_connected, false, 'Default guest private probe must not establish TCP'); assert.equal(receipt.bedrock.tls_connected, false); assert(receipt.bedrock.error, 'Unsuccessful private probe must include its error'); @@ -162,6 +184,37 @@ function privateIpv4(address: string): boolean { return first === 10 || first === 172 && second! >= 16 && second! <= 31 || first === 192 && second === 168; } +// Stock Codex's proxy sandbox permits IP sockets inside an isolated namespace, so numeric +// TCP fails with ENETUNREACH. A missing outer route is insufficient: require the current +// kernel namespace's complete loopback-only interface and route inventories. /proc/net/dev +// remains readable when Codex's AF_NETLINK restriction denies socket.if_nameindex(). +function isolatedNetworkNamespace(namespace: EgressReceipt['network_namespace']): boolean { + if (!namespace || namespace.error !== null || namespace.id === null || !/^net:\[[1-9]\d*\]$/.test(namespace.id) + || namespace.devices === null || namespace.ipv4_routes === null || namespace.ipv6_routes === null) return false; + const devices = namespace.devices.trim().split('\n'); + if (devices.length !== 3 || devices[0]!.replace(/\s+/g, '') !== 'Inter-|Receive|Transmit' + || devices[1]!.replace(/\s+/g, '') !== 'face|bytespacketserrsdropfifoframecompressedmulticast|bytespacketserrsdropfifocollscarriercompressed' + || !/^\s*lo:\s+(?:\d+\s+){15}\d+\s*$/.test(devices[2]!)) return false; + // A fresh namespace has no IPv4 routes. Any data row, including a default via lo, + // is outside this proof rather than evidence we infer to be harmless. + if (namespace.ipv4_routes.trim().split(/\s+/).join(' ') + !== 'Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT') return false; + return namespace.ipv6_routes.trim() === '' || namespace.ipv6_routes.trim().split('\n').every(line => { + const fields = line.trim().split(/\s+/); + if (fields.length !== 10 || ![0, 2, 4].every(index => /^[0-9a-f]{32}$/.test(fields[index]!)) + || ![1, 3].every(index => /^[0-9a-f]{2}$/.test(fields[index]!)) + || ![5, 6, 7, 8].every(index => /^[0-9a-f]{8}$/.test(fields[index]!))) return false; + const [destination, prefix, source, sourcePrefix, gateway, , , , flags, device] = fields; + const zero = '0'.repeat(32); + if (device !== 'lo' || source !== zero || sourcePrefix !== '00' || gateway !== zero) return false; + // Linux includes unreachable ::/0 sentinels (RTF_REJECT), even without IPv6 egress. + // The only usable route accepted is ::1/128, with no gateway (RTF_GATEWAY). + const bits = Number.parseInt(flags!, 16); + return destination === zero && prefix === '00' && (bits & 0x200) !== 0 + || destination === `${'0'.repeat(31)}1` && prefix === '80' && (bits & 0x1) !== 0 && (bits & 0x2) === 0; + }); +} + function parseReceipt(value: unknown): EgressReceipt { assert(value !== null && typeof value === 'object', 'Receipt must be an object'); const receipt = value as Record; @@ -176,6 +229,11 @@ function parseReceipt(value: unknown): EgressReceipt { && Array.isArray(bedrock.resolved_ipv4) && bedrock.resolved_ipv4.every(address => typeof address === 'string') && nullableString(bedrock.address) && typeof bedrock.tcp_connected === 'boolean' && typeof bedrock.tls_connected === 'boolean' && nullableString(bedrock.tls_version) && nullableString(bedrock.error), 'Invalid Bedrock receipt'); + if (receipt.network_namespace !== undefined) { + assert(receipt.network_namespace !== null && typeof receipt.network_namespace === 'object', 'Invalid network namespace receipt'); + const namespace = receipt.network_namespace as Record; + assert(['id', 'devices', 'ipv4_routes', 'ipv6_routes', 'error'].every(key => nullableString(namespace[key])), 'Invalid network namespace fields'); + } return value as EgressReceipt; } diff --git a/tests/scripts/egress-proof.test.ts b/tests/scripts/egress-proof.test.ts index 6cda2d5..546bf69 100644 --- a/tests/scripts/egress-proof.test.ts +++ b/tests/scripts/egress-proof.test.ts @@ -1,7 +1,22 @@ -import { createHash } from 'node:crypto'; +import { execFile } from 'node:child_process'; +import { createHash, randomBytes, randomUUID } from 'node:crypto'; +import { mkdir, readFile, readlink, rm, writeFile } from 'node:fs/promises'; +import { promisify } from 'node:util'; import { describe, expect, it } from 'vitest'; import type { AgentSession, AgentSessionItem, SessionArtifact, Turn } from '../../src/domain/agents-api.js'; -import { egressProbeCommand, egressProbeProgram, egressTurnEvidence, isEgressProbeCommand, receiptLimit, type EgressReceipt, type EgressProbeSpec } from '../../testing/aws/egress-proof.js'; +import { hostedCodexArguments } from '../../src/runner/hosted-environment-planning.js'; +import { egressInputPath, egressProbeCommand, egressProbeProgram, egressProgramPath, egressTurnEvidence, isEgressProbeCommand, receiptLimit, type EgressReceipt, type EgressProbeSpec } from '../../testing/aws/egress-proof.js'; + +// Kernel snapshots captured under the official 0.164.0 rat_managed sandbox, UID 10001. +const loopbackDevices = 'Inter-| Receive | Transmit\n' + + ' face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n' + + ' lo: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n'; +const emptyIpv4Routes = 'Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\n'; +const rejectedIpv6Default = '00000000000000000000000000000000 00 00000000000000000000000000000000 00 00000000000000000000000000000000 ffffffff 00000001 00000000 00200200 lo\n'; +const localIpv6Route = '00000000000000000000000000000001 80 00000000000000000000000000000000 00 00000000000000000000000000000000 00000000 00000002 00000000 80200001 lo\n'; +type Namespace = NonNullable; +const isolatedNamespace = (): Namespace => ({ id: 'net:[4026532974]', devices: loopbackDevices, ipv4_routes: emptyIpv4Routes, + ipv6_routes: rejectedIpv6Default + localIpv6Route + rejectedIpv6Default, error: null }); function fixture(networkAccess: EgressProbeSpec['networkAccess'] = 'enabled') { const expected = { agentId: 'agent-proof', sessionId: 'session-proof', marker: 'unpredictable-marker', inputSha256: 'test-input-hash', @@ -53,13 +68,65 @@ describe('runtime egress receipt selection', () => { expect(egressTurnEvidence(value).receipt).toEqual(value.receipt); }); - it.each(['TimeoutError:None', 'ConnectionRefusedError:111', 'gaierror:-3'])( + it('accepts ENETUNREACH only with complete kernel loopback namespace evidence', () => { + const value = changedReceipt(receipt => { receipt.public_tcp.error = 'OSError:101'; receipt.network_namespace = isolatedNamespace(); }, 'disabled'); + const before = structuredClone(value); + expect(egressTurnEvidence(value).receipt).toEqual(value.receipt); + expect(value).toEqual(before); + }); + + it('accepts a readable empty IPv6 route inventory', () => { + const value = changedReceipt(receipt => { + receipt.public_tcp.error = 'OSError:101'; receipt.network_namespace = { ...isolatedNamespace(), ipv6_routes: '' }; + }, 'disabled'); + expect(egressTurnEvidence(value).receipt).toEqual(value.receipt); + }); + + it.each(['OSError:101', 'TimeoutError:None', 'ConnectionRefusedError:111', 'gaierror:-3'])( 'rejects default public error %s despite matching output and artifact', error => { const value = changedReceipt(receipt => { receipt.public_tcp.error = error; }, 'disabled'); expect(() => egressTurnEvidence(value)).toThrow(); }, ); + it.each(['TimeoutError:None', 'ConnectionRefusedError:111', 'gaierror:-3'])( + 'rejects %s even with complete loopback evidence', error => { + const value = changedReceipt(receipt => { receipt.public_tcp.error = error; receipt.network_namespace = isolatedNamespace(); }, 'disabled'); + expect(() => egressTurnEvidence(value)).toThrow(); + }, + ); + + it.each([ + ['unreadable inventory', (namespace: Namespace) => { namespace.error = 'PermissionError:13'; }], + ['missing namespace ID', (namespace: Namespace) => { namespace.id = null; }], + ['malformed namespace ID', (namespace: Namespace) => { namespace.id = 'net:[unknown]'; }], + ['missing devices', (namespace: Namespace) => { namespace.devices = null; }], + ['missing IPv4 routes', (namespace: Namespace) => { namespace.ipv4_routes = null; }], + ['missing IPv6 routes', (namespace: Namespace) => { namespace.ipv6_routes = null; }], + ['eth0 without routes', (namespace: Namespace) => { namespace.devices += loopbackDevices.split('\n')[2]!.replace('lo:', 'eth0:') + '\n'; }], + ['no loopback device', (namespace: Namespace) => { namespace.devices = loopbackDevices.replace('lo:', 'eth0:'); }], + ['malformed device counters', (namespace: Namespace) => { namespace.devices = loopbackDevices.replace('lo: 0', 'lo: unknown'); }], + ['truncated device inventory', (namespace: Namespace) => { namespace.devices = loopbackDevices.split('\n').slice(2).join('\n'); }], + ['empty IPv4 inventory', (namespace: Namespace) => { namespace.ipv4_routes = ''; }], + ['IPv4 external route', (namespace: Namespace) => { namespace.ipv4_routes += 'eth0 00000000 0100000A 0003 0 0 0 00000000 0 0 0\n'; }], + ['IPv4 loopback default', (namespace: Namespace) => { namespace.ipv4_routes += 'lo 00000000 00000000 0001 0 0 0 00000000 0 0 0\n'; }], + ['malformed IPv4 inventory', (namespace: Namespace) => { namespace.ipv4_routes = 'Iface Destination Gateway\n'; }], + ['IPv6 external interface', (namespace: Namespace) => { namespace.ipv6_routes = localIpv6Route.replace('lo', 'eth0'); }], + ['IPv6 usable default', (namespace: Namespace) => { namespace.ipv6_routes = rejectedIpv6Default.replace('00200200', '00000001'); }], + ['IPv6 global destination', (namespace: Namespace) => { namespace.ipv6_routes = localIpv6Route.replace(/^0{31}1/, '20010db8000000000000000000000001'); }], + ['IPv6 gateway', (namespace: Namespace) => { namespace.ipv6_routes = localIpv6Route.replace('00 00000000000000000000000000000000 00000000', '00 00000000000000000000000000000001 00000000'); }], + ['IPv6 gateway flag', (namespace: Namespace) => { namespace.ipv6_routes = localIpv6Route.replace('80200001', '80200003'); }], + ['IPv6 source route', (namespace: Namespace) => { namespace.ipv6_routes = localIpv6Route.replace('80 00000000000000000000000000000000 00', '80 20010db8000000000000000000000000 40'); }], + ['IPv6 invalid hex', (namespace: Namespace) => { namespace.ipv6_routes = rejectedIpv6Default.replace('00200200', 'not-hex!'); }], + ['IPv6 truncated fields', (namespace: Namespace) => { namespace.ipv6_routes = '00000000000000000000000000000000 00 lo\n'; }], + ['wrong inventory type', (namespace: Namespace) => { Reflect.set(namespace, 'devices', ['lo']); }], + ] as const)('rejects ENETUNREACH with %s', (_name, mutate) => { + const value = changedReceipt(receipt => { + receipt.public_tcp.error = 'OSError:101'; receipt.network_namespace = isolatedNamespace(); mutate(receipt.network_namespace); + }, 'disabled'); + expect(() => egressTurnEvidence(value)).toThrow(); + }); + it('accepts the captured Codex bash command shape and rejects appended shell work', () => { expect(isEgressProbeCommand("/bin/bash -lc 'python3 /workspace/egress-probe.py'")).toBe(true); expect(isEgressProbeCommand(egressProbeCommand)).toBe(true); @@ -121,3 +188,40 @@ describe('runtime egress receipt selection', () => { expect(() => egressTurnEvidence(value)).toThrow('Effective network policy'); }); }); + +// Opt in only inside a disposable worker image: no provider or inference request is made. +const native = process.env.CODEX_EGRESS_NATIVE_E2E === 'true' ? it : it.skip; +native('executes the uploaded probe through the stock managed sandbox and proves namespace isolation', async () => { + expect(process.getuid?.()).toBe(10001); + const value = fixture('disabled'); + const input = randomBytes(64); + value.expected.marker = `stock-egress-${randomUUID()}`; + value.expected.inputSha256 = createHash('sha256').update(input).digest('hex'); + value.expected.outputPath = `/workspace/outputs/${value.expected.marker}.json`; + value.artifact.path = value.expected.outputPath; + const created: string[] = []; + try { + const outerNamespace = await readlink('/proc/self/ns/net'); + const outerDevices = await readFile('/proc/net/dev', 'utf8'); + const outerIpv4Routes = await readFile('/proc/net/route', 'utf8'); + expect(outerDevices).toMatch(/\beth0:/); + expect(outerIpv4Routes).toMatch(/\beth0\s/); + await mkdir('/workspace/outputs', { recursive: true }); + for (const [path, bytes] of [[egressInputPath, input], [egressProgramPath, egressProbeProgram(value.expected)]] as const) { + await writeFile(path, bytes, { flag: 'wx' }); created.push(path); + } + const result = await promisify(execFile)('/opt/codex-runtime/bin/codex', hostedCodexArguments( + ['sandbox', '--permission-profile', 'rat_managed', '--', 'python3', egressProgramPath], { access: 'disabled', allowed_domains: [] }), { + cwd: '/workspace', timeout: 30_000, maxBuffer: receiptLimit, env: { HOME: '/home/agent', PATH: '/usr/local/bin:/usr/bin:/bin' }, + }); + created.push(value.expected.outputPath); + value.command.output = result.stdout; + value.artifactText = await readFile(value.expected.outputPath, 'utf8'); + value.artifact.size_bytes = Buffer.byteLength(value.artifactText); + const evidence = egressTurnEvidence(value); + expect(evidence.receipt.public_tcp.error).toBe('OSError:101'); + expect(evidence.receipt.network_namespace?.id).not.toBe(outerNamespace); + expect(evidence.receipt.network_namespace?.error).toBeNull(); + console.log(JSON.stringify({ stockManagedSandbox: { outerNamespace, outerDevices, outerIpv4Routes, receipt: evidence.receipt } })); + } finally { await Promise.all(created.map(path => rm(path, { force: true }))); } +}, 40_000); From 991a786a45723c389802d38d6c5e87ae9025fbff Mon Sep 17 00:00:00 2001 From: Greg Pazo Date: Mon, 26 Oct 2026 21:39:39 -0700 Subject: [PATCH 3/5] Pin the official Codex path at trusted worker startup --- microvm/runtime-process-policy.mjs | 3 ++- tests/microvm/runtime-process-policy.test.mjs | 15 ++++++++++++- tests/runner/codex-cold-verification.test.ts | 21 ++++++++++++++----- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/microvm/runtime-process-policy.mjs b/microvm/runtime-process-policy.mjs index 4c8edb0..4a362bc 100644 --- a/microvm/runtime-process-policy.mjs +++ b/microvm/runtime-process-policy.mjs @@ -6,7 +6,8 @@ export function trustedRunnerOptions({ uid, gid, environment }) { } return { cwd: '/opt/agent-runtime', - env: { ...environment, RUN_AGENT_UID: String(uid), RUN_AGENT_GID: String(gid) }, + // Pin the packaged artifact and its companions for launch and cold verification. + env: { ...environment, CODEX_BINARY: '/opt/codex-runtime/bin/codex', RUN_AGENT_UID: String(uid), RUN_AGENT_GID: String(gid) }, // The trusted runner reads credentials and commits fenced state. It drops // Codex and repository commands to the separate identity above. uid: 0, diff --git a/tests/microvm/runtime-process-policy.test.mjs b/tests/microvm/runtime-process-policy.test.mjs index 5faa18e..55217c9 100644 --- a/tests/microvm/runtime-process-policy.test.mjs +++ b/tests/microvm/runtime-process-policy.test.mjs @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest'; +import { planCodexLaunch } from '../../src/runner/agent-planning.js'; import { trustedRunnerOptions } from '../../microvm/runtime-process-policy.mjs'; describe('MicroVM runtime process policy', () => { @@ -7,13 +8,25 @@ describe('MicroVM runtime process policy', () => { expect(trustedRunnerOptions({ uid: 10001, gid: 10001, environment })).toEqual({ cwd: '/opt/agent-runtime', - env: { ...environment, RUN_AGENT_UID: '10001', RUN_AGENT_GID: '10001' }, + env: { ...environment, CODEX_BINARY: '/opt/codex-runtime/bin/codex', RUN_AGENT_UID: '10001', RUN_AGENT_GID: '10001' }, uid: 0, gid: 0, stdio: ['inherit', 'inherit', 'inherit', 'ipc'], }); }); + it.each([undefined, 'codex', '/workspace/untrusted-codex'])('pins packaged Codex through launch planning when the inherited binary is %s', binary => { + const environment = Object.freeze({ CODEX_BINARY: binary, PATH: '/workspace/bin', PERSISTENT_SESSION: 'true', + CODEX_HOME: '/mnt/session/codex-home', CODEX_SQLITE_HOME: '/mnt/session/codex-home/sqlite' }); + const options = trustedRunnerOptions({ uid: 10001, gid: 10001, environment }); + const plan = planCodexLaunch({ version: '1', prompt: '' }, '/mnt/session/workspace', 30_000, options.env); + expect(plan).toMatchObject({ binary: '/opt/codex-runtime/bin/codex', identity: { uid: 10001, gid: 10001 }, persistent: true, + environment: { CODEX_HOME: environment.CODEX_HOME, CODEX_SQLITE_HOME: environment.CODEX_SQLITE_HOME } }); + expect(options.uid).toBe(0); + expect(options.env).not.toBe(environment); + expect(environment.CODEX_BINARY).toBe(binary); + }); + it('refuses root or malformed identities', () => { expect(() => trustedRunnerOptions({ uid: 0, gid: 10001, environment: {} })) .toThrow('UID'); diff --git a/tests/runner/codex-cold-verification.test.ts b/tests/runner/codex-cold-verification.test.ts index 3f4f913..72864b6 100644 --- a/tests/runner/codex-cold-verification.test.ts +++ b/tests/runner/codex-cold-verification.test.ts @@ -2,7 +2,7 @@ import { spawn } from 'node:child_process'; import { createServer } from 'node:http'; import { once } from 'node:events'; import { chown, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; -import { join, resolve } from 'node:path'; +import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { setTimeout as delay } from 'node:timers/promises'; import { expect, it, vi } from 'vitest'; @@ -11,6 +11,10 @@ import { CodexRpcClient } from '../../src/adapters/codex-rpc.js'; import { nativeThreadHistory, type NativeCheckpointManifest } from '../../src/domain/native-checkpoint.js'; import { stopCheckpointWriters } from '../../src/runner/checkpoint-writers.js'; import { syncWorkspaceStorage } from '../../src/runner/workspace-sync.js'; +import { planCodexLaunch } from '../../src/runner/agent-planning.js'; +import { planSessionLaunch } from '../../src/runner/session-launch-planning.js'; +import { publicTemplate } from '../../src/domain/environment-planning.js'; +import { sessionAgent } from '../../src/core/session-planning.js'; const invalidVerifierConfig = vi.hoisted(() => ({ enabled: false })); vi.mock('node:fs/promises', async importOriginal => { @@ -100,6 +104,8 @@ guest('rejects reserved-provider configuration fallback before accepting native async function guestFixture({ historyMode, modelProvider, model }: (typeof guestCases)[number]) { if (process.getuid?.() !== 0) throw new Error('Run CODEX_COLD_NATIVE_E2E only as root in the disposable worker container'); + const processPolicyPath = '/opt/agent-runtime/runtime-process-policy.mjs'; + const { trustedRunnerOptions } = await import(processPolicyPath); const root = await mkdtemp(join(tmpdir(), 'rat-native-cold-guest-')); const home = join(root, 'codex-home'); const workspace = join(root, 'workspace'); const identity = { uid: 10001, gid: 10001 }; @@ -125,9 +131,12 @@ async function guestFixture({ historyMode, modelProvider, model }: (typeof guest server.listen(0, '127.0.0.1'); await once(server, 'listening'); const address = server.address(); if (!address || typeof address === 'string') throw new Error('Missing fixture port'); const completed = new Set(); let childId: string | undefined; let childActivityDone = false; - const binary = process.env.CODEX_CONFORMANCE_BINARY ?? resolve('node_modules/.bin/codex'); + const runner = trustedRunnerOptions({ ...identity, environment: { PATH: '/usr/bin:/bin', HOME: home, CODEX_HOME: home, + CODEX_SQLITE_HOME: join(home, 'sqlite'), PERSISTENT_SESSION: 'true' } }); + const plan = planCodexLaunch({ version: '1', prompt: '' }, workspace, 30_000, runner.env); + const binary = plan.binary; const providerConfig = { [modelProvider === 'openai' ? 'openai_base_url' : `model_providers.${modelProvider}.base_url`]: `http://127.0.0.1:${address.port}` }; - const rpc = new CodexRpcClient({ binary, cwd: workspace, environment: { PATH: '/usr/bin:/bin', HOME: home, CODEX_HOME: home, + const rpc = new CodexRpcClient({ binary, cwd: workspace, environment: { ...plan.environment, AWS_EC2_METADATA_DISABLED: 'true', AWS_CONFIG_FILE: '/dev/null', AWS_SHARED_CREDENTIALS_FILE: '/dev/null', ...(modelProvider === 'openai' ? { CODEX_API_KEY: 'synthetic-loopback-fixture-only' } : { AWS_BEARER_TOKEN_BEDROCK: 'synthetic-loopback-fixture-only', AWS_REGION: 'us-east-1' }) }, identity, binaryArguments: ['-c', `model_provider=${JSON.stringify(modelProvider)}`, ...Object.entries(providerConfig).flatMap(([key, value]) => ['-c', `${key}=${JSON.stringify(value)}`]), @@ -155,8 +164,10 @@ async function guestFixture({ historyMode, modelProvider, model }: (typeof guest } expect(threads.map(thread => thread.modelProvider)).toEqual([modelProvider, modelProvider]); const manifest: NativeCheckpointManifest = { rootThreadId: started.thread.id, hostJournalDigest: 'a'.repeat(64), threads }; - const request = { binary, workspace, executionWorkspace: '/workspace', environment: { CODEX_HOME: home }, identity, timeoutMs: 30_000, prompt: '', sandbox: 'read-only' as const, persistent: true, modelProvider, networkAccess: false, - managedPolicy: { workspace: '/workspace', network: { access: 'disabled' as const, allowed_domains: [] }, credentialProxy: false } }; + const environment = { ...publicTemplate({ network: { access: 'disabled' } }, { id: 'fixture-environment', created_at: 1, updated_at: 1 }), + type: 'openai_hosted' as const, files: [], skills: [] }; + const launch = { sessionId: 'fixture-session', turnId: 'fixture-turn', input: [], environment, agent: sessionAgent({ model }, 'fixture-agent', 1) }; + const request = { ...plan, ...planSessionLaunch(plan, launch), modelProvider }; return { root, home, request, manifest, requests: () => requests, close, stop: async () => { if (stopped) return; await rpc.closeCheckpoint(); await stopCheckpointWriters(identity.uid); await syncWorkspaceStorage(root); stopped = true; } }; } catch (error) { await close(); throw error; } From 41524c2a7f056938118fe9bb88e82b6c27eb6bb8 Mon Sep 17 00:00:00 2001 From: Greg Pazo Date: Mon, 26 Oct 2026 21:49:18 -0700 Subject: [PATCH 4/5] Package the cloud launcher in the x64 cold-test fixture --- .github/workflows/ci.yml | 3 ++- testing/cold-native-local-worker.Dockerfile | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 testing/cold-native-local-worker.Dockerfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7da90b5..d7a0d67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,7 +100,8 @@ jobs: CODEX_RUNTIME_ARTIFACT: ${{ github.workspace }}/.runtime/codex/linux-x64 run: | npm run runtime:local:build - docker buildx build --platform linux/amd64 --load --file testing/native-parity.Dockerfile --build-arg WORKER_IMAGE=rat-things-local:1 --tag rat-things-native-parity:x64 . + docker buildx build --platform linux/amd64 --load --file testing/cold-native-local-worker.Dockerfile --tag rat-things-cold-local-worker:x64 microvm + docker buildx build --platform linux/amd64 --load --file testing/native-parity.Dockerfile --build-arg WORKER_IMAGE=rat-things-cold-local-worker:x64 --tag rat-things-native-parity:x64 . docker run --rm --platform linux/amd64 --privileged --init --user 0 --env CODEX_COLD_NATIVE_E2E=true --entrypoint npx rat-things-native-parity:x64 --no-install vitest run tests/runner/codex-cold-verification.test.ts --no-file-parallelism - name: Archive verified Linux x64 runtime run: tar -C .runtime/codex/linux-x64 -czf .runtime/codex/linux-x64.tar.gz artifact.json runtime diff --git a/testing/cold-native-local-worker.Dockerfile b/testing/cold-native-local-worker.Dockerfile new file mode 100644 index 0000000..859d38f --- /dev/null +++ b/testing/cold-native-local-worker.Dockerfile @@ -0,0 +1,3 @@ +FROM rat-things-local:1 +# The local worker omits cloud orchestration; cold conformance uses its installed launch policy. +COPY --chown=0:0 --chmod=0555 runtime-process-policy.mjs /opt/agent-runtime/runtime-process-policy.mjs From e351536d2983dbdd504e700f293665e09c868afc Mon Sep 17 00:00:00 2001 From: Greg Pazo Date: Mon, 26 Oct 2026 22:22:07 -0700 Subject: [PATCH 5/5] Allow bounded cold initialization before sandbox verification --- src/adapters/codex-rpc.ts | 8 +- src/runner/managed-executor-verification.ts | 2 +- tests/adapters/codex-rpc-startup.test.ts | 83 +++++++++++++++++ tests/runner/managed-executor-startup.test.ts | 92 +++++++++++++++++++ 4 files changed, 182 insertions(+), 3 deletions(-) create mode 100644 tests/adapters/codex-rpc-startup.test.ts create mode 100644 tests/runner/managed-executor-startup.test.ts diff --git a/src/adapters/codex-rpc.ts b/src/adapters/codex-rpc.ts index 7b5c680..575998b 100644 --- a/src/adapters/codex-rpc.ts +++ b/src/adapters/codex-rpc.ts @@ -95,9 +95,13 @@ export class CodexRpcClient { } } - public async initialize(): Promise { - await this.call('initialize', { clientInfo: { name: 'rat-environment-service', version: '1' }, capabilities: { experimentalApi: true } }); + public async initialize(timeoutMs = 30_000): Promise { + const startedAt = performance.now(); + await this.call('initialize', { clientInfo: { name: 'rat-environment-service', version: '1' }, capabilities: { experimentalApi: true } }, timeoutMs); this.child.stdin.write(`${JSON.stringify({ method: 'initialized' })}\n`); + if (this.startupPurpose) console.error(JSON.stringify({ message: 'Native harness initialized', purpose: this.startupPurpose, ordinal: this.processOrdinal, + elapsedMs: Math.min(3_600_000, Math.max(0, Math.round(performance.now() - startedAt))), + })); } public call(method: string, params: unknown, timeoutMs = 30_000): Promise { diff --git a/src/runner/managed-executor-verification.ts b/src/runner/managed-executor-verification.ts index 2f7b0f4..f193b15 100644 --- a/src/runner/managed-executor-verification.ts +++ b/src/runner/managed-executor-verification.ts @@ -24,7 +24,7 @@ export async function verifyManagedExecutor(options: { ...(plan.identity ? { identity: plan.identity } : {}), ...(options.signal ? { signal: options.signal } : {}), onServerRequest: async () => { throw new Error('Sandbox verification cannot request approval'); }, }); - await rpc.initialize(); + await rpc.initialize(120_000); await verifyManagedCodexPolicy({ call: rpc.call.bind(rpc), policy: { workspace: options.workspace, network: options.network, credentialProxy: options.credentialProxy } }); await verifyManagedSandbox({ call: rpc.call.bind(rpc), workspace: options.workspace, home, ...(plan.identity ? { identity: plan.identity } : {}) }); } finally { diff --git a/tests/adapters/codex-rpc-startup.test.ts b/tests/adapters/codex-rpc-startup.test.ts new file mode 100644 index 0000000..a4d5b05 --- /dev/null +++ b/tests/adapters/codex-rpc-startup.test.ts @@ -0,0 +1,83 @@ +import { spawn } from 'node:child_process'; +import { EventEmitter } from 'node:events'; +import { PassThrough } from 'node:stream'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { CodexRpcClient } from '../../src/adapters/codex-rpc.js'; + +vi.mock('node:child_process', async original => ({ ...await original(), spawn: vi.fn() })); + +function fixture(signal?: AbortSignal, purpose?: 'setup' | 'session') { + const child = Object.assign(new EventEmitter(), { + pid: 123, exitCode: null as number | null, signalCode: null as string | null, + stdin: new PassThrough(), stdout: new PassThrough(), stderr: new PassThrough(), + kill: vi.fn((signal: string) => { child.signalCode = signal; child.emit('exit', null, signal); child.emit('close', null, signal); return true; }), + }); + vi.mocked(spawn).mockReturnValue(child as never); + const client = new CodexRpcClient({ binary: '/fixture/codex', cwd: '/', environment: {}, ...(signal ? { signal } : {}), ...(purpose ? { startupPurpose: purpose } : {}) }); + const requests: Array<{ id?: number; method: string }> = []; + child.stdin.on('data', data => requests.push(JSON.parse(String(data)))); + return { child, client, requests }; +} + +beforeEach(() => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + vi.spyOn(console, 'error').mockImplementation(() => {}); +}); +afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); }); + +it('allows a bounded cold initialize without extending subsequent RPC deadlines', async () => { + const { child, client, requests } = fixture(); + try { + const initialized = client.initialize(120_000); + await vi.advanceTimersByTimeAsync(40_000); + expect(requests.map(request => request.method)).toEqual(['initialize']); + child.stdout.write(`${JSON.stringify({ id: requests[0]!.id, result: {} })}\n`); + await initialized; + expect(requests.map(request => request.method)).toEqual(['initialize', 'initialized']); + const ordinary = expect(client.call('config/read', {})).rejects.toThrow('timed out'); + await vi.advanceTimersByTimeAsync(30_000); + await ordinary; + } finally { await client.close(); } +}); + +it.each([undefined, 120_000])('rejects initialize at its exact deadline (%s)', async timeout => { + const { client, requests } = fixture(); + try { + let settled = false; + const rejected = expect(client.initialize(timeout).finally(() => { settled = true; })).rejects.toThrow('timed out'); + await vi.advanceTimersByTimeAsync((timeout ?? 30_000) - 1); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + await rejected; + expect(requests.map(request => request.method)).toEqual(['initialize']); + } finally { await client.close(); } +}); + +it.each(['abort', 'exit'] as const)('interrupts cold initialization promptly on %s', async reason => { + const abort = new AbortController(); + const { child, client, requests } = fixture(abort.signal); + try { + const rejected = expect(client.initialize(120_000)).rejects.toThrow('closed'); + await vi.advanceTimersByTimeAsync(40_000); + if (reason === 'abort') abort.abort(); + else { child.exitCode = 1; child.emit('exit', 1, null); child.emit('close', 1, null); } + await rejected; + expect(requests.map(request => request.method)).toEqual(['initialize']); + if (reason === 'abort') expect(child.kill).toHaveBeenCalledWith('SIGTERM'); + } finally { await client.close(); } +}); + +it('reports only fixed startup labels and bounded elapsed time after successful initialization', async () => { + const clock = vi.spyOn(performance, 'now').mockReturnValue(0); + const { child, client, requests } = fixture(undefined, 'setup'); + try { + const initialized = client.initialize(120_000); + clock.mockReturnValue(40_000); + child.stdout.write(`${JSON.stringify({ id: requests[0]!.id, result: { private: 'native-output-must-not-be-logged' } })}\n`); + await initialized; + expect(console.error).toHaveBeenCalledTimes(1); + expect(JSON.parse(vi.mocked(console.error).mock.calls[0]![0] as string)).toEqual({ + message: 'Native harness initialized', purpose: 'setup', ordinal: expect.any(Number), elapsedMs: 40_000, + }); + } finally { await client.close(); } +}); diff --git a/tests/runner/managed-executor-startup.test.ts b/tests/runner/managed-executor-startup.test.ts new file mode 100644 index 0000000..0b391aa --- /dev/null +++ b/tests/runner/managed-executor-startup.test.ts @@ -0,0 +1,92 @@ +import { spawn } from 'node:child_process'; +import { EventEmitter } from 'node:events'; +import { access } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { PassThrough } from 'node:stream'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { planCodexLaunch } from '../../src/runner/agent-planning.js'; +import { verifyManagedExecutor } from '../../src/runner/managed-executor-verification.js'; + +vi.mock('node:child_process', async original => ({ ...await original(), spawn: vi.fn() })); + +type Request = { id?: number; method: string }; + +function fixture() { + const child = Object.assign(new EventEmitter(), { + pid: 123, exitCode: null as number | null, signalCode: null as string | null, + stdin: new PassThrough(), stdout: new PassThrough(), stderr: new PassThrough(), + kill: vi.fn((signal: string) => { child.signalCode = signal; child.emit('exit', null, signal); child.emit('close', null, signal); return true; }), + }); + vi.mocked(spawn).mockReturnValue(child as never); + const requests: Request[] = []; + const received = new EventEmitter(); + child.stdin.on('data', data => { + const request = JSON.parse(String(data)) as Request; + requests.push(request); + received.emit(request.method, request); + }); + const request = (method: string): Promise => { + const existing = requests.find(candidate => candidate.method === method); + return existing ? Promise.resolve(existing) : new Promise(resolve => received.once(method, resolve)); + }; + const reply = (request: Request, result: unknown) => child.stdout.write(`${JSON.stringify({ id: request.id, result })}\n`); + const workspace = tmpdir(); + const plan = planCodexLaunch({ version: '1', prompt: 'unused' }, workspace, 1000, { + CODEX_BINARY: '/fixture/codex', CODEX_AUTH_MODE: 'chatgpt', PATH: process.env.PATH, + }); + const verify = (signal?: AbortSignal) => verifyManagedExecutor({ + plan, workspace, network: { access: 'disabled', allowed_domains: [] }, credentialProxy: false, ...(signal ? { signal } : {}), + }); + const home = () => vi.mocked(spawn).mock.calls[0]![2]!.env!.CODEX_HOME!; + return { child, requests, request, reply, verify, home }; +} + +beforeEach(() => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + vi.spyOn(console, 'error').mockImplementation(() => {}); +}); +afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); }); + +it('waits beyond 30 seconds for the managed handshake before checking policy and refusing an invalid executor', async () => { + const f = fixture(); + const refused = expect(f.verify()).rejects.toThrow('Managed executor policy could not be verified'); + const initialize = await f.request('initialize'); + await vi.advanceTimersByTimeAsync(40_000); + expect(f.requests.map(request => request.method)).toEqual(['initialize']); + expect(f.child.kill).not.toHaveBeenCalled(); + f.reply(initialize, {}); + f.reply(await f.request('config/read'), {}); + f.reply(await f.request('configRequirements/read'), {}); + await refused; + expect(f.requests.map(request => request.method)).toEqual(['initialize', 'initialized', 'config/read', 'configRequirements/read']); + expect(f.child.kill).toHaveBeenCalledWith('SIGTERM'); + await expect(access(f.home())).rejects.toMatchObject({ code: 'ENOENT' }); +}); + +it('refuses managed startup at 120 seconds without running policy or sandbox commands', async () => { + const f = fixture(); + let settled = false; + const refused = expect(f.verify().finally(() => { settled = true; })).rejects.toThrow('timed out'); + await f.request('initialize'); + await vi.advanceTimersByTimeAsync(119_999); + expect(settled).toBe(false); + expect(f.child.kill).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + await refused; + expect(f.requests.map(request => request.method)).toEqual(['initialize']); + expect(f.child.kill).toHaveBeenCalledWith('SIGTERM'); + await expect(access(f.home())).rejects.toMatchObject({ code: 'ENOENT' }); +}); + +it('cancels managed startup promptly without running policy or sandbox commands', async () => { + const f = fixture(); + const abort = new AbortController(); + const refused = expect(f.verify(abort.signal)).rejects.toThrow('closed'); + await f.request('initialize'); + await vi.advanceTimersByTimeAsync(40_000); + abort.abort(); + await refused; + expect(f.requests.map(request => request.method)).toEqual(['initialize']); + expect(f.child.kill).toHaveBeenCalledWith('SIGTERM'); + await expect(access(f.home())).rejects.toMatchObject({ code: 'ENOENT' }); +});