Support Bedrock cold recovery with official Codex

.github/workflows/ci.yml

@@ -100,7 +100,8 @@ jobs:
           CODEX_RUNTIME_ARTIFACT: ${{ github.workspace }}/.runtime/codex/linux-x64
         run: |
           npm run runtime:local:build
-          docker buildx build --platform linux/amd64 --load --file testing/native-parity.Dockerfile --build-arg WORKER_IMAGE=rat-things-local:1 --tag rat-things-native-parity:x64 .
+          docker buildx build --platform linux/amd64 --load --file testing/cold-native-local-worker.Dockerfile --tag rat-things-cold-local-worker:x64 microvm
+          docker buildx build --platform linux/amd64 --load --file testing/native-parity.Dockerfile --build-arg WORKER_IMAGE=rat-things-cold-local-worker:x64 --tag rat-things-native-parity:x64 .
           docker run --rm --platform linux/amd64 --privileged --init --user 0 --env CODEX_COLD_NATIVE_E2E=true --entrypoint npx rat-things-native-parity:x64 --no-install vitest run tests/runner/codex-cold-verification.test.ts --no-file-parallelism
       - name: Archive verified Linux x64 runtime
         run: tar -C .runtime/codex/linux-x64 -czf .runtime/codex/linux-x64.tar.gz artifact.json runtime

docs/agents-api.md

@@ -414,9 +414,9 @@ resume. Verification has no network or credentials and cannot run model work or
 tools. Extra coordination details omitted by Codex's public history remain in
 the host journal. This does not verify Codex's hidden mailbox state.
 
-Codex cold checkpoints currently support the built-in OpenAI provider and custom
-Responses API providers. Bedrock, Ollama and LM Studio are refused before the
-running process stops because their cold recovery has not been validated. This
+Codex cold checkpoints support the built-in OpenAI and Amazon Bedrock providers
+and custom Responses API providers. Ollama and LM Studio are refused before the
+running process stops because their cold recovery is unsupported. This
 restriction does not prevent ordinary Sessions or Pi Durable execution.
 
 Checkpointing refuses unsupported startup integrations, external native storage,

microvm/runtime-process-policy.mjs

@@ -6,7 +6,8 @@ export function trustedRunnerOptions({ uid, gid, environment }) {
   }
   return {
     cwd: '/opt/agent-runtime',
-    env: { ...environment, RUN_AGENT_UID: String(uid), RUN_AGENT_GID: String(gid) },
+    // Pin the packaged artifact and its companions for launch and cold verification.
+    env: { ...environment, CODEX_BINARY: '/opt/codex-runtime/bin/codex', RUN_AGENT_UID: String(uid), RUN_AGENT_GID: String(gid) },
     // The trusted runner reads credentials and commits fenced state. It drops
     // Codex and repository commands to the separate identity above.
     uid: 0,

runtime/codex/README.md

@@ -70,10 +70,10 @@ verification must fail the handoff; it must not silently reconstruct a prompt or
 start a new native thread. The opt-in AWS checkpoint harness remains necessary
 before claiming cloud worker replacement acceptance.
 
-The cold verifier supports the built-in OpenAI provider and custom Responses API
-providers. It rejects Bedrock, Ollama and LM Studio before stopping the live
-process until their recovery paths are validated; ordinary Sessions and Pi
-Durable execution are unaffected.
+The cold verifier supports the built-in OpenAI and Amazon Bedrock providers and
+custom Responses API providers. It rejects Ollama and LM Studio before stopping
+the live process because their recovery paths are unsupported; ordinary Sessions
+and Pi Durable execution are unaffected.
 
 ## Upgrade and rollback
 

src/adapters/codex-rpc.ts

@@ -95,9 +95,13 @@ export class CodexRpcClient {
     }
   }
 
-  public async initialize(): Promise<void> {
-    await this.call('initialize', { clientInfo: { name: 'rat-environment-service', version: '1' }, capabilities: { experimentalApi: true } });
+  public async initialize(timeoutMs = 30_000): Promise<void> {
+    const startedAt = performance.now();
+    await this.call('initialize', { clientInfo: { name: 'rat-environment-service', version: '1' }, capabilities: { experimentalApi: true } }, timeoutMs);
     this.child.stdin.write(`${JSON.stringify({ method: 'initialized' })}\n`);
+    if (this.startupPurpose) console.error(JSON.stringify({ message: 'Native harness initialized', purpose: this.startupPurpose, ordinal: this.processOrdinal,
+      elapsedMs: Math.min(3_600_000, Math.max(0, Math.round(performance.now() - startedAt))),
+    }));
   }
 
   public call(method: string, params: unknown, timeoutMs = 30_000): Promise<unknown> {

src/runner/codex-cold-verification.ts

@@ -33,7 +33,7 @@ export function assertCheckpointStartupSafe(value: unknown, sessionConfig: unkno
 export async function prepareCodexColdVerification(options: { request: CodexAppServerRequest; stateRoot: string; manifest: NativeCheckpointManifest }): Promise<PreparedColdVerification> {
   const { request, stateRoot, manifest } = options;
   if (manifest.threads.some(thread => thread.cwd !== '/workspace' || typeof thread.modelProvider !== 'string' || !/^[a-zA-Z0-9_-]{1,100}$/.test(thread.modelProvider))) throw new Error('Cold native checkpoint thread paths or providers are unsupported');
-  if (manifest.threads.some(thread => ['amazon-bedrock', 'amazon-bedrock-runtime', 'ollama', 'lmstudio'].includes(String(thread.modelProvider)))) throw new Error('Cold native checkpoint provider is not yet supported');
+  if (manifest.threads.some(thread => ['ollama', 'lmstudio'].includes(String(thread.modelProvider)))) throw new Error('Cold native checkpoint provider is not yet supported');
   const uid = request.identity?.uid;
   const gid = request.identity?.gid;
   const home = request.environment.CODEX_HOME;
@@ -68,7 +68,7 @@ export async function prepareCodexColdVerification(options: { request: CodexAppS
     await mkdir(disposable, { mode: 0o711 });
     await ownedDirectory(join(disposable, 'home'), identity);
     await ownedDirectory(join(disposable, 'workspace'), identity);
-    const args = namespaceArguments({ binary, home, sqlite, workspace: request.workspace, disposable, identity });
+    const args = namespaceArguments({ binary, bwrap, home, sqlite, workspace: request.workspace, disposable, identity });
     // Check namespace support before sealing durable authority or stopping Codex.
     await checkedProcess(bwrap, [...args, '--', '/usr/bin/true'], identity);
     return {
@@ -92,13 +92,16 @@ export async function prepareCodexColdVerification(options: { request: CodexAppS
         await writeFile(join(disposable, 'home', 'config.toml'), verifierConfiguration(manifest), { mode: 0o600 });
         await chown(join(disposable, 'home', 'config.toml'), identity.uid, identity.gid);
         let activity = false;
+        let configurationFailure: Error | undefined;
         rpc = new CodexRpcClient({ binary: bwrap, binaryArguments: [...args, '--', binary, 'app-server'], cwd: '/', environment: {}, identity,
           onEvent: event => {
+            if (event.method === 'configWarning') configurationFailure ??= new Error('Cold native verifier configuration is invalid');
             if (event.requestId !== undefined || /^(turn\/(started|completed)|item\/|error$)/.test(event.method)) activity = true;
           }, onServerRequest: async () => { activity = true; throw new Error('Cold native verifier cannot perform tools'); },
         });
         try {
           await rpc.initialize();
+          if (configurationFailure) throw configurationFailure;
           for (const expected of manifest.threads) {
             const result = await rpc.call('thread/read', { threadId: expected.id, includeTurns: true });
             assertNativeHistory(expected, record(result) ? result.thread : undefined, String(expected.id));
@@ -112,33 +115,43 @@ export async function prepareCodexColdVerification(options: { request: CodexAppS
           }
           await rpc.closeCheckpoint();
           await stopCheckpointWriters(identity.uid);
+          if (configurationFailure) throw configurationFailure;
           if (activity) throw new Error('Cold native verification attempted activity');
           if (await treeDigest(candidate) !== candidateDigest) throw new Error('Cold native candidate changed during verification');
           return nativeCheckpointEvidence(manifest);
+        } catch (error) {
+          // A fallback warning can arrive before a native request also fails to load config.
+          throw configurationFailure ?? error;
         } finally { await rpc.close(); await stopCheckpointWriters(identity.uid); }
       },
     };
   } catch (error) { await close(); throw error; }
 }
 
-function namespaceArguments(options: { binary: string; home: string; sqlite: string | undefined; workspace: string; disposable: string; identity: { uid: number; gid: number } }): string[] {
-  const { binary, home, sqlite, workspace, disposable } = options;
+function namespaceArguments(options: { binary: string; bwrap: string; home: string; sqlite: string | undefined; workspace: string; disposable: string; identity: { uid: number; gid: number } }): string[] {
+  const { binary, bwrap, home, sqlite, workspace, disposable } = options;
   return ['--unshare-all', '--die-with-parent', '--new-session', '--cap-drop', 'ALL', '--clearenv',
     '--ro-bind', '/usr', '/usr', '--symlink', 'usr/bin', '/bin', '--symlink', 'usr/lib', '/lib', '--symlink', 'usr/lib64', '/lib64',
     '--proc', '/proc', '--dev', '/dev', '--tmpfs', '/tmp', '--dir', '/home/agent',
     '--ro-bind', binary, binary,
+    '--dir', '/rat-cold-bin', '--ro-bind', bwrap, '/rat-cold-bin/bwrap',
     '--bind', join(disposable, 'home'), home,
     '--ro-bind', join(disposable, 'workspace'), workspace,
     ...(workspace === '/workspace' ? [] : ['--ro-bind', join(disposable, 'workspace'), '/workspace']),
-    '--setenv', 'PATH', '/usr/bin:/bin', '--setenv', 'HOME', '/home/agent', '--setenv', 'CODEX_HOME', home,
+    '--setenv', 'PATH', '/rat-cold-bin:/usr/bin:/bin', '--setenv', 'HOME', '/home/agent', '--setenv', 'CODEX_HOME', home,
     ...(sqlite ? ['--setenv', 'CODEX_SQLITE_HOME', sqlite] : []),
     '--setenv', 'AWS_EC2_METADATA_DISABLED', 'true', '--setenv', 'AWS_CONFIG_FILE', '/dev/null', '--setenv', 'AWS_SHARED_CREDENTIALS_FILE', '/dev/null',
     '--setenv', 'LOG_FORMAT', 'json', '--setenv', 'RUST_LOG', 'warn', '--chdir', '/workspace'];
 }
 function verifierConfiguration(manifest: NativeCheckpointManifest): string {
   // Keep built-in OpenAI intact. Its prewarm has no credentials and can only
   // reach a closed loopback address in the private network namespace.
-  const providers = [...new Set(manifest.threads.map(thread => String(thread.modelProvider)))].filter(provider => provider !== 'openai').map(provider => `\n[model_providers.${JSON.stringify(provider)}]\nname = "Offline checkpoint verifier"\nbase_url = "http://127.0.0.1:1"\nwire_api = "responses"\nrequires_openai_auth = false\nsupports_websockets = false\nrequest_max_retries = 0\nstream_max_retries = 0\n`).join('');
+  // Built-in Bedrock permits endpoint overrides, not generic provider replacement.
+  // Its auth prewarm may fail without credentials; native read/resume remains local.
+  const providers = [...new Set(manifest.threads.map(thread => String(thread.modelProvider)))].filter(provider => provider !== 'openai').map(provider =>
+    ['amazon-bedrock', 'amazon-bedrock-runtime'].includes(provider)
+      ? `\n[model_providers.${JSON.stringify(provider)}]\nbase_url = "http://127.0.0.1:1"\n`
+      : `\n[model_providers.${JSON.stringify(provider)}]\nname = "Offline checkpoint verifier"\nbase_url = "http://127.0.0.1:1"\nwire_api = "responses"\nrequires_openai_auth = false\nsupports_websockets = false\nrequest_max_retries = 0\nstream_max_retries = 0\n`).join('');
   return 'openai_base_url = "http://127.0.0.1:1"\napproval_policy = "never"\nsandbox_mode = "read-only"\ncli_auth_credentials_store = "file"\nnotify = []\nweb_search = "disabled"\n[features]\nplugins = false\napps = false\nhooks = false\ngoals = false\nmulti_agent = true\nmulti_agent_v2 = true\nresponses_websockets = false\nresponses_websockets_v2 = false\ncode_mode_prewarm = false\nshell_snapshot = false\nshell_snapshot_v2 = false\n' + providers;
 }
 function within(root: string, path: string): boolean { const part = relative(root, path); return isAbsolute(root) && isAbsolute(path) && part !== '' && part !== '..' && !part.startsWith('../') && !isAbsolute(part); }

src/runner/managed-executor-verification.ts

@@ -24,7 +24,7 @@ export async function verifyManagedExecutor(options: {
       ...(plan.identity ? { identity: plan.identity } : {}), ...(options.signal ? { signal: options.signal } : {}),
       onServerRequest: async () => { throw new Error('Sandbox verification cannot request approval'); },
     });
-    await rpc.initialize();
+    await rpc.initialize(120_000);
     await verifyManagedCodexPolicy({ call: rpc.call.bind(rpc), policy: { workspace: options.workspace, network: options.network, credentialProxy: options.credentialProxy } });
     await verifyManagedSandbox({ call: rpc.call.bind(rpc), workspace: options.workspace, home, ...(plan.identity ? { identity: plan.identity } : {}) });
   } finally {

testing/aws/egress-proof.ts

@@ -24,6 +24,13 @@ export interface EgressReceipt {
   euid: number;
   input_sha256: string;
   program_sha256: string;
+  network_namespace?: {
+    id: string | null;
+    devices: string | null;
+    ipv4_routes: string | null;
+    ipv6_routes: string | null;
+    error: string | null;
+  };
   public_tcp: { host: string; port: number; connected: boolean; error: string | null };
   bedrock: {
     host: string;
@@ -47,6 +54,20 @@ export function egressProbeProgram(spec: EgressProbeSpec): string {
     '    raise TimeoutError("DNS deadline")',
     'def private_ipv4(address):',
     '    return any(ipaddress.ip_address(address) in ipaddress.ip_network(network) for network in ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"])',
+    'def network_file(path):',
+    '    with open(path, "r", encoding="ascii") as source:',
+    '        value = source.read(2049)',
+    '    if len(value) > 2048:',
+    '        raise ValueError("Network inventory exceeds its bound")',
+    '    return value',
+    'namespace = {"id": None, "devices": None, "ipv4_routes": None, "ipv6_routes": None, "error": None}',
+    'try:',
+    '    namespace["id"] = os.readlink("/proc/self/ns/net")',
+    '    namespace["devices"] = network_file("/proc/net/dev")',
+    '    namespace["ipv4_routes"] = network_file("/proc/net/route")',
+    '    namespace["ipv6_routes"] = network_file("/proc/net/ipv6_route")',
+    'except (OSError, ValueError) as error:',
+    '    namespace["error"] = error_code(error)',
     'public = {"host": config["public_host"], "port": config["port"], "connected": False, "error": None}',
     'try:',
     '    with socket.create_connection((public["host"], public["port"]), timeout=4):',
@@ -75,7 +96,7 @@ export function egressProbeProgram(spec: EgressProbeSpec): string {
     '    input_sha256 = hashlib.sha256(source.read()).hexdigest()',
     'with open(__file__, "rb") as source:',
     '    program_sha256 = hashlib.sha256(source.read()).hexdigest()',
-    'receipt = {"schema": 1, "marker": config["marker"], "uid": os.getuid(), "euid": os.geteuid(), "input_sha256": input_sha256, "program_sha256": program_sha256, "public_tcp": public, "bedrock": bedrock}',
+    'receipt = {"schema": 1, "marker": config["marker"], "uid": os.getuid(), "euid": os.geteuid(), "input_sha256": input_sha256, "program_sha256": program_sha256, "network_namespace": namespace, "public_tcp": public, "bedrock": bedrock}',
     'serialized = json.dumps(receipt, sort_keys=True, separators=(",", ":"))',
     'os.makedirs("/workspace/outputs", exist_ok=True)',
     'with open(config["output_path"], "x") as output:',
@@ -145,8 +166,9 @@ export function egressTurnEvidence(input: {
     assert(receipt.bedrock.tls_version?.startsWith('TLSv'), 'Bedrock must negotiate TLS');
     assert.equal(receipt.bedrock.error, null);
   } else {
-    assert(receipt.public_tcp.error === 'PermissionError:1' || receipt.public_tcp.error === 'PermissionError:13',
-      'Default guest network policy must explicitly deny the public numeric TCP probe');
+    assert(receipt.public_tcp.error === 'PermissionError:1' || receipt.public_tcp.error === 'PermissionError:13'
+      || receipt.public_tcp.error === 'OSError:101' && isolatedNetworkNamespace(receipt.network_namespace),
+    'Default guest network policy must explicitly deny the public numeric TCP probe or prove an isolated loopback namespace');
     assert.equal(receipt.bedrock.tcp_connected, false, 'Default guest private probe must not establish TCP');
     assert.equal(receipt.bedrock.tls_connected, false);
     assert(receipt.bedrock.error, 'Unsuccessful private probe must include its error');
@@ -162,6 +184,37 @@ function privateIpv4(address: string): boolean {
   return first === 10 || first === 172 && second! >= 16 && second! <= 31 || first === 192 && second === 168;
 }
 
+// Stock Codex's proxy sandbox permits IP sockets inside an isolated namespace, so numeric
+// TCP fails with ENETUNREACH. A missing outer route is insufficient: require the current
+// kernel namespace's complete loopback-only interface and route inventories. /proc/net/dev
+// remains readable when Codex's AF_NETLINK restriction denies socket.if_nameindex().
+function isolatedNetworkNamespace(namespace: EgressReceipt['network_namespace']): boolean {
+  if (!namespace || namespace.error !== null || namespace.id === null || !/^net:\[[1-9]\d*\]$/.test(namespace.id)
+    || namespace.devices === null || namespace.ipv4_routes === null || namespace.ipv6_routes === null) return false;
+  const devices = namespace.devices.trim().split('\n');
+  if (devices.length !== 3 || devices[0]!.replace(/\s+/g, '') !== 'Inter-|Receive|Transmit'
+    || devices[1]!.replace(/\s+/g, '') !== 'face|bytespacketserrsdropfifoframecompressedmulticast|bytespacketserrsdropfifocollscarriercompressed'
+    || !/^\s*lo:\s+(?:\d+\s+){15}\d+\s*$/.test(devices[2]!)) return false;
+  // A fresh namespace has no IPv4 routes. Any data row, including a default via lo,
+  // is outside this proof rather than evidence we infer to be harmless.
+  if (namespace.ipv4_routes.trim().split(/\s+/).join(' ')
+    !== 'Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT') return false;
+  return namespace.ipv6_routes.trim() === '' || namespace.ipv6_routes.trim().split('\n').every(line => {
+    const fields = line.trim().split(/\s+/);
+    if (fields.length !== 10 || ![0, 2, 4].every(index => /^[0-9a-f]{32}$/.test(fields[index]!))
+      || ![1, 3].every(index => /^[0-9a-f]{2}$/.test(fields[index]!))
+      || ![5, 6, 7, 8].every(index => /^[0-9a-f]{8}$/.test(fields[index]!))) return false;
+    const [destination, prefix, source, sourcePrefix, gateway, , , , flags, device] = fields;
+    const zero = '0'.repeat(32);
+    if (device !== 'lo' || source !== zero || sourcePrefix !== '00' || gateway !== zero) return false;
+    // Linux includes unreachable ::/0 sentinels (RTF_REJECT), even without IPv6 egress.
+    // The only usable route accepted is ::1/128, with no gateway (RTF_GATEWAY).
+    const bits = Number.parseInt(flags!, 16);
+    return destination === zero && prefix === '00' && (bits & 0x200) !== 0
+      || destination === `${'0'.repeat(31)}1` && prefix === '80' && (bits & 0x1) !== 0 && (bits & 0x2) === 0;
+  });
+}
+
 function parseReceipt(value: unknown): EgressReceipt {
   assert(value !== null && typeof value === 'object', 'Receipt must be an object');
   const receipt = value as Record<string, unknown>;
@@ -176,6 +229,11 @@ function parseReceipt(value: unknown): EgressReceipt {
     && Array.isArray(bedrock.resolved_ipv4) && bedrock.resolved_ipv4.every(address => typeof address === 'string')
     && nullableString(bedrock.address) && typeof bedrock.tcp_connected === 'boolean' && typeof bedrock.tls_connected === 'boolean'
     && nullableString(bedrock.tls_version) && nullableString(bedrock.error), 'Invalid Bedrock receipt');
+  if (receipt.network_namespace !== undefined) {
+    assert(receipt.network_namespace !== null && typeof receipt.network_namespace === 'object', 'Invalid network namespace receipt');
+    const namespace = receipt.network_namespace as Record<string, unknown>;
+    assert(['id', 'devices', 'ipv4_routes', 'ipv6_routes', 'error'].every(key => nullableString(namespace[key])), 'Invalid network namespace fields');
+  }
   return value as EgressReceipt;
 }