From f17bf1850284ebed60fef13c54a7e9bb2520d50b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 13:50:50 -0700 Subject: [PATCH 01/11] fix(codex): the provider supervisor outlives its provider group when stopped A signalled supervisor forwards the signal to the provider group, escalates to SIGKILL after the grace, and exits only once the group is gone, so recovery's proof that the recorded pid is dead also proves the provider is. It refuses to spawn when its parent is already not the owner named in its spec, and watches that owner rather than whichever parent it first saw. The grace is a spec field. Recovery's SIGTERM stage now outlasts the supervisor's own stop, since a SIGKILL that lands first cannot be handled and leaves the group running. --- ...erver-posix-supervisor.integration.test.ts | 202 ++++++++++++++++++ .../codex-app-server-posix-supervisor.test.ts | 7 +- .../codex-app-server-posix-supervisor.ts | 70 ++++-- ...-agent-session-recovery-resolution.test.ts | 75 ++++++- ...tured-agent-session-recovery-resolution.ts | 17 +- 5 files changed, 349 insertions(+), 22 deletions(-) create mode 100644 src/main/codex/codex-app-server-posix-supervisor.integration.test.ts diff --git a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts new file mode 100644 index 000000000000..5bb6f8b7523f --- /dev/null +++ b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts @@ -0,0 +1,202 @@ +import { spawn, type ChildProcess } from 'node:child_process' +import { existsSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS, + POSIX_PROVIDER_SUPERVISOR_SCRIPT, + supervisedPosixLaunch, + type ProviderSupervisorOptions +} from './codex-app-server-posix-supervisor' + +// The provider leads its own group; its grandchild shares that group and ignores SIGTERM. +const PROVIDER = String.raw` + const { spawn } = require('node:child_process') + if (process.env.ORCA_TEST_PROVIDER_IGNORES_SIGTERM) process.on('SIGTERM', () => {}) + const grandchild = spawn( + process.execPath, + ['-e', "process.on('SIGTERM', () => {}); process.stdout.write('armed'); setInterval(() => {}, 60000)"], + { stdio: ['ignore', 'pipe', 'ignore'] } + ) + grandchild.stdout.once('data', () => { + process.stdout.write(JSON.stringify({ provider: process.pid, grandchild: grandchild.pid }) + '\n') + }) + setInterval(() => {}, 60000) +` + +// Stands in for Orca: launches the supervisor as its own child, then can be killed outright. A +// second child holds the supervisor's stdin open, so only the parent-death watch can notice. +const OWNER = String.raw` + const { spawn } = require('node:child_process') + const spec = JSON.parse(Buffer.from(process.env.ORCA_PROVIDER_SUPERVISOR_SPEC, 'base64').toString()) + spec.ownerPid = process.pid + const supervisor = spawn(process.execPath, ['-e', process.env.ORCA_TEST_SUPERVISOR_SCRIPT], { + env: { ...process.env, ORCA_PROVIDER_SUPERVISOR_SPEC: Buffer.from(JSON.stringify(spec)).toString('base64') }, + stdio: ['pipe', 'pipe', 'ignore'], + detached: true + }) + const holder = spawn(process.execPath, ['-e', 'setInterval(() => {}, 60000)'], { + stdio: ['ignore', supervisor.stdin, 'ignore'] + }) + process.stdout.write(JSON.stringify({ supervisor: supervisor.pid, holder: holder.pid }) + '\n') + supervisor.stdout.pipe(process.stdout) + setInterval(() => {}, 60000) +` + +const recordedPids = new Set() + +function alive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch (error) { + return !(error instanceof Error && 'code' in error && error.code === 'ESRCH') + } +} + +async function waitFor(predicate: () => boolean, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs + while (!predicate()) { + if (Date.now() >= deadline) { + return false + } + await new Promise((resolve) => setTimeout(resolve, 25)) + } + return true +} + +function readPids(child: ChildProcess, keys: readonly string[]): Promise> { + return new Promise((resolve, reject) => { + const pids: Record = {} + let buffered = '' + const timeout = setTimeout(() => reject(new Error(`no ${keys.join('/')} pids`)), 10_000) + child.stdout!.on('data', (chunk: Buffer) => { + buffered += chunk.toString() + const lines = buffered.split('\n') + buffered = lines.pop() ?? '' + for (const line of lines) { + const parsed: unknown = JSON.parse(line) + for (const [key, pid] of Object.entries(parsed ?? {})) { + if (typeof pid === 'number') { + pids[key] = pid + recordedPids.add(pid) + } + } + } + if (keys.every((key) => key in pids)) { + clearTimeout(timeout) + resolve(pids) + } + }) + }) +} + +function launchSupervisor( + options: ProviderSupervisorOptions, + env: Record = {}, + provider: { command: string; args: string[] } = { + command: process.execPath, + args: ['-e', PROVIDER] + } +): { supervisor: ChildProcess; exit: Promise<{ code: number | null; signal: string | null }> } { + const launch = supervisedPosixLaunch(provider, { ...process.env, ...env }, options) + const supervisor = spawn(launch.command, launch.args, { + env: launch.env, + stdio: ['pipe', 'pipe', 'ignore'], + detached: true + }) + recordedPids.add(supervisor.pid!) + const exit = new Promise<{ code: number | null; signal: string | null }>((resolve) => + supervisor.once('exit', (code, signal) => resolve({ code, signal })) + ) + return { supervisor, exit } +} + +afterEach(() => { + for (const pid of recordedPids) { + if (alive(pid)) { + process.kill(pid, 'SIGKILL') + } + } + recordedPids.clear() +}) + +describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processes', () => { + it('reaps the provider group on SIGTERM and exits only after the group is gone', async () => { + const { supervisor, exit } = launchSupervisor({ graceMs: 300 }) + const { provider, grandchild } = await readPids(supervisor, ['provider', 'grandchild']) + + let groupAliveAtExit: boolean | null = null + void exit.then(() => { + groupAliveAtExit = alive(-provider) + }) + supervisor.kill('SIGTERM') + + await expect(exit).resolves.toEqual({ code: null, signal: 'SIGTERM' }) + expect(groupAliveAtExit).toBe(false) + expect(alive(provider)).toBe(false) + expect(alive(grandchild)).toBe(false) + }) + + it('escalates a SIGTERM-ignoring provider to SIGKILL after the grace from the spec', async () => { + const graceMs = 200 + const { supervisor, exit } = launchSupervisor( + { graceMs }, + { ORCA_TEST_PROVIDER_IGNORES_SIGTERM: '1' } + ) + const { provider, grandchild } = await readPids(supervisor, ['provider', 'grandchild']) + + const signalledAt = Date.now() + supervisor.kill('SIGTERM') + const exited = await Promise.race([exit, new Promise((resolve) => setTimeout(resolve, 5_000))]) + + expect(exited).toEqual({ code: null, signal: 'SIGTERM' }) + expect(Date.now() - signalledAt).toBeGreaterThanOrEqual(graceMs) + expect(Date.now() - signalledAt).toBeLessThan(DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS) + expect(alive(provider)).toBe(false) + expect(alive(grandchild)).toBe(false) + }) + + it('never spawns the provider when its owner is not its parent at start', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-supervisor-owner-')) + try { + const marker = join(dir, 'provider-started') + const { exit } = launchSupervisor( + { ownerPid: process.pid === 1 ? 2 : 1 }, + {}, + { command: 'touch', args: [marker] } + ) + + await expect(exit).resolves.toEqual({ code: 1, signal: null }) + await new Promise((resolve) => setTimeout(resolve, 200)) + expect(existsSync(marker)).toBe(false) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('reaps the provider group when its owner dies', async () => { + const launch = supervisedPosixLaunch( + { command: process.execPath, args: ['-e', PROVIDER] }, + process.env + ) + const owner = spawn(process.execPath, ['-e', OWNER], { + env: { ...launch.env, ORCA_TEST_SUPERVISOR_SCRIPT: POSIX_PROVIDER_SUPERVISOR_SCRIPT }, + stdio: ['ignore', 'pipe', 'ignore'] + }) + recordedPids.add(owner.pid!) + const { supervisor, provider, grandchild } = await readPids(owner, [ + 'supervisor', + 'holder', + 'provider', + 'grandchild' + ]) + + owner.kill('SIGKILL') + + expect(await waitFor(() => !alive(-provider), 3_000)).toBe(true) + expect(await waitFor(() => !alive(supervisor), 3_000)).toBe(true) + expect(alive(grandchild)).toBe(false) + }) +}) diff --git a/src/main/codex/codex-app-server-posix-supervisor.test.ts b/src/main/codex/codex-app-server-posix-supervisor.test.ts index f51157a443c7..58d58535f1b0 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import type { CodexAppServerLaunch } from './codex-app-server-connection' import { createProviderSpawnSpec, + DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS, POSIX_PROVIDER_SUPERVISOR_SCRIPT, supervisedPosixLaunch } from './codex-app-server-posix-supervisor' @@ -27,7 +28,9 @@ describe('structured provider supervision', () => { expect.objectContaining({ command: '/opt/codex', args: ['app-server', '--flag'], - cwd: '/work/repo' + cwd: '/work/repo', + ownerPid: process.pid, + graceMs: DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS }) ) expect( @@ -38,7 +41,7 @@ describe('structured provider supervision', () => { ) expect(POSIX_PROVIDER_SUPERVISOR_SCRIPT).toContain('delete childEnv.ELECTRON_RUN_AS_NODE') expect(spec.env.ELECTRON_RUN_AS_NODE).toBe('1') - expect(POSIX_PROVIDER_SUPERVISOR_SCRIPT).toContain('process.ppid !== originalParent') + expect(POSIX_PROVIDER_SUPERVISOR_SCRIPT).toContain('process.ppid !== spec.ownerPid') expect(POSIX_PROVIDER_SUPERVISOR_SCRIPT).toContain( "process.stdin.once('close', scheduleOwnerShutdown)" ) diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index f83ba6f374bc..a74b72e3b077 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -1,9 +1,22 @@ import type { CodexAppServerLaunch } from './codex-app-server-connection' +/** + * Time the provider gets to exit on its own after stdin ends or an owner signal, before SIGKILL. + * Session recovery's SIGTERM window is sized from it; a longer launch grace must widen that too. + */ +export const DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS = 1250 +/** How long the supervisor waits for a SIGKILLed provider group to disappear. */ +export const PROVIDER_GROUP_REAP_TIMEOUT_MS = 1500 + /** Inline supervisor source kept dependency-free for the spawned Node child. */ export const POSIX_PROVIDER_SUPERVISOR_SCRIPT = ` const { spawn } = require('node:child_process') const spec = JSON.parse(Buffer.from(process.env.ORCA_PROVIDER_SUPERVISOR_SPEC, 'base64').toString()) +// A detached supervisor is reparented when its owner exits. The new parent may +// be PID 1 or a platform subreaper, so any other parent means no live owner. +const ownerGone = () => process.ppid !== spec.ownerPid +// Orca can die before this runs; spawning then would start a provider nothing watches. +if (ownerGone()) process.exit(1) const childEnv = { ...process.env } delete childEnv.ORCA_PROVIDER_SUPERVISOR_SPEC delete childEnv.ELECTRON_RUN_AS_NODE @@ -13,7 +26,6 @@ const child = spawn(spec.command, spec.args, { stdio: ['pipe', 'pipe', 'pipe'], detached: true }) -const originalParent = process.ppid let timer let ownerShutdownTimer let settling = false @@ -26,18 +38,21 @@ const providerGroupExists = () => { return Boolean(error && error.code !== 'ESRCH') } } -const reapOwnedProviderGroup = async () => { - if (!child.pid) return false - try { process.kill(-child.pid, 'SIGKILL') } catch (error) { - if (error && error.code !== 'ESRCH') return false - } - const deadline = Date.now() + 1500 +const waitForProviderGroupExit = async (timeoutMs) => { + const deadline = Date.now() + timeoutMs while (providerGroupExists()) { if (Date.now() >= deadline) return false await new Promise((resolve) => setTimeout(resolve, 25)) } return true } +const reapOwnedProviderGroup = async () => { + if (!child.pid) return false + try { process.kill(-child.pid, 'SIGKILL') } catch (error) { + if (error && error.code !== 'ESRCH') return false + } + return waitForProviderGroupExit(${PROVIDER_GROUP_REAP_TIMEOUT_MS}) +} const terminateOwnedGroup = () => { if (settling) return settling = true @@ -48,7 +63,7 @@ const scheduleOwnerShutdown = () => { if (settling || ownerShutdownTimer) return // A normal close ends the provider's stdin first; allow it to flush and // exit before forcing the group, while still bounding an orphaned child. - ownerShutdownTimer = setTimeout(terminateOwnedGroup, 1250) + ownerShutdownTimer = setTimeout(terminateOwnedGroup, spec.graceMs) ownerShutdownTimer.unref() } process.stdin.once('end', scheduleOwnerShutdown) @@ -59,6 +74,8 @@ child.stderr.pipe(process.stderr) for (const stream of [process.stdin, child.stdin, child.stdout, child.stderr]) stream.on('error', () => {}) const finishWithProviderOutcome = (code, signal) => { if (!signal) return process.exit(code ?? 1) + // Re-raise with the default action; this supervisor's own handler would swallow it. + process.removeAllListeners(signal) process.kill(process.pid, signal) } const reapProviderExit = async (code, signal) => { @@ -69,13 +86,21 @@ const reapProviderExit = async (code, signal) => { if (!(await reapOwnedProviderGroup())) return process.exit(1) finishWithProviderOutcome(code, signal) } +// Whoever stops this pid is judging the provider by it: stay alive until the group is gone, +// so a dead supervisor always means a dead provider group. +const stopOnOwnerSignal = (signal) => { + if (settling) return + settling = true + clearInterval(timer) + if (ownerShutdownTimer) clearTimeout(ownerShutdownTimer) + try { process.kill(-child.pid, signal) } catch {} + void waitForProviderGroupExit(spec.graceMs) + .then((exited) => exited || reapOwnedProviderGroup()) + .then((reaped) => (reaped ? finishWithProviderOutcome(null, signal) : process.exit(1))) +} +for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) process.on(signal, () => stopOnOwnerSignal(signal)) timer = setInterval(() => { - // A detached supervisor is reparented when its owner exits. The new parent - // may be PID 1 or a platform subreaper, so any parent change is proof that - // this process group no longer has a live Orca owner. - if (process.ppid !== originalParent) { - terminateOwnedGroup() - } + if (ownerGone()) terminateOwnedGroup() }, 100) timer.unref() child.once('error', () => { @@ -87,16 +112,29 @@ child.once('exit', (code, signal) => { }) ` +export type ProviderSupervisorOptions = { + cwd?: string + /** The process the supervisor serves; it must be the supervisor's parent. */ + ownerPid?: number + graceMs?: number +} + export function supervisedPosixLaunch( launch: CodexAppServerLaunch, childEnv: NodeJS.ProcessEnv, - cwd = launch.cwd ?? process.cwd() + { + cwd = launch.cwd ?? process.cwd(), + ownerPid = process.pid, + graceMs = DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS + }: ProviderSupervisorOptions = {} ): { command: string; args: string[]; env: NodeJS.ProcessEnv } { const supervisorSpec = Buffer.from( JSON.stringify({ command: launch.command, args: launch.args, - cwd + cwd, + ownerPid, + graceMs }) ).toString('base64') return { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.test.ts index 7fbd785894d8..063e8c64af71 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.test.ts @@ -1,3 +1,4 @@ +import { spawn } from 'node:child_process' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -8,6 +9,7 @@ import { writeOlderBuildLease } from '../../runtime/agent-session-older-build-lease.test-fixture' import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { supervisedPosixLaunch } from '../../codex/codex-app-server-posix-supervisor' import { resolveStructuredSessionRecovery, type StructuredSessionRecoveryResolutionDeps @@ -62,7 +64,7 @@ async function reserve(store: AgentSessionRecordStore) { }) } -async function liveOwner(store: AgentSessionRecordStore) { +async function liveOwner(store: AgentSessionRecordStore, pid = 4242) { const reserved = await reserve(store) const fence = reserved.record.lease.runtimeFence await store.commitProcessIdentity({ @@ -70,7 +72,7 @@ async function liveOwner(store: AgentSessionRecordStore) { fence, process: { hostId: 'local', - pid: 4242, + pid, processStartTimeMs: NOW - 1_000, spawnToken: 'spawn-recovery' }, @@ -302,3 +304,72 @@ describe('structured session recovery resolution', () => { ).toBe('not-applicable') }) }) + +describe.runIf(process.platform !== 'win32')( + 'structured session recovery of a supervised owner', + () => { + const recordedPids: number[] = [] + const alive = (pid: number): boolean => { + try { + process.kill(pid, 0) + return true + } catch (error) { + return !(error instanceof Error && 'code' in error && error.code === 'ESRCH') + } + } + + afterEach(() => { + for (const pid of recordedPids.splice(0)) { + if (alive(pid)) { + process.kill(pid, 'SIGKILL') + } + } + }) + + it('evicts only after the supervisor has reaped a provider that ignores SIGTERM', async () => { + const launch = supervisedPosixLaunch( + { + command: process.execPath, + args: [ + '-e', + "process.on('SIGTERM', () => {}); process.stdout.write(process.pid + '\\n'); setInterval(() => {}, 60000)" + ] + }, + process.env + ) + const supervisor = spawn(launch.command, launch.args, { + env: launch.env, + stdio: ['pipe', 'pipe', 'ignore'], + detached: true + }) + recordedPids.push(supervisor.pid!) + const provider = await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error('provider never started')), 10_000) + supervisor.stdout.once('data', (chunk: Buffer) => { + clearTimeout(timeout) + resolve(Number(chunk.toString().trim())) + }) + }) + recordedPids.push(provider) + const store = await openStore() + await liveOwner(store, supervisor.pid!) + await latch(store) + + const result = await resolveStructuredSessionRecovery( + { + store, + // The recorded pid is the supervisor's; its absence is the proof recovery evicts on. + probeRecord: async () => + alive(supervisor.pid!) ? MATCHED : { outcome: 'pid-absent' as const }, + now: () => NOW + 10_000 + }, + SESSION + ) + + expect(result).toBe('resolved') + // Evicted on proof of death, which must hold for the provider too, not only the supervisor. + expect(store.getRecord(SESSION)?.lease.deathEvidence).not.toBeNull() + expect(alive(provider)).toBe(false) + }) + } +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts index 5e04cbdf90ef..6ef151ebe7bb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts @@ -15,6 +15,10 @@ import { type AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { + DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS, + PROVIDER_GROUP_REAP_TIMEOUT_MS +} from '../../codex/codex-app-server-posix-supervisor' import { releaseUnprovenAgentSessionOwner } from '../../runtime/agent-session-lease-transitions' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' @@ -28,8 +32,17 @@ export type StructuredSessionRecoveryResolutionDeps = { delay?: (ms: number) => Promise } -const STOP_PROBES_PER_SIGNAL = 4 const STOP_PROBE_INTERVAL_MS = 250 +// A POSIX Codex owner is its provider supervisor, which exits only after its provider group. A +// SIGKILL that lands first leaves the group running, so SIGTERM outlasts the supervisor's stop. +const STOP_PROBES: Record = { + SIGTERM: + Math.ceil( + (DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS + PROVIDER_GROUP_REAP_TIMEOUT_MS) / + STOP_PROBE_INTERVAL_MS + ) + 1, + SIGKILL: 4 +} const UNRESOLVED_REFUSALS: ReadonlySet = new Set([ 'agent_session_ownership_unknown', @@ -94,7 +107,7 @@ async function stopOwnerAndReprobe( let probe: AgentSessionOwnerProbe = { outcome: 'indeterminate', reason: 'owner stop requested' } for (const signal of ['SIGTERM', 'SIGKILL'] as const) { stop(pid, signal) - for (let attempt = 0; attempt < STOP_PROBES_PER_SIGNAL; attempt += 1) { + for (let attempt = 0; attempt < STOP_PROBES[signal]; attempt += 1) { probe = await deps.probeRecord(record) if (isProvenDeadProbe(probe)) { return probe From 75a7e73a4a0ef7bfac65e9c421430c61c6a38f98 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 14:04:49 -0700 Subject: [PATCH 02/11] fix(codex): a closed owner pipe no longer ends the supervisor before its provider group When Orca dies, the supervisor's stdout pipe has no reader. Provider output in the window before the parent-death watch fired raised an unhandled EPIPE that exited the supervisor with the provider group still running. --- ...-server-posix-supervisor.integration.test.ts | 17 +++++++++++++---- .../codex/codex-app-server-posix-supervisor.ts | 5 ++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts index 5bb6f8b7523f..53ef985343a5 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts @@ -14,6 +14,7 @@ import { const PROVIDER = String.raw` const { spawn } = require('node:child_process') if (process.env.ORCA_TEST_PROVIDER_IGNORES_SIGTERM) process.on('SIGTERM', () => {}) + process.stdout.on('error', () => {}) const grandchild = spawn( process.execPath, ['-e', "process.on('SIGTERM', () => {}); process.stdout.write('armed'); setInterval(() => {}, 60000)"], @@ -21,6 +22,7 @@ const PROVIDER = String.raw` ) grandchild.stdout.once('data', () => { process.stdout.write(JSON.stringify({ provider: process.pid, grandchild: grandchild.pid }) + '\n') + if (process.env.ORCA_TEST_PROVIDER_STREAMS_OUTPUT) setInterval(() => process.stdout.write('.'), 2) }) setInterval(() => {}, 60000) ` @@ -71,7 +73,7 @@ function readPids(child: ChildProcess, keys: readonly string[]): Promise = {} let buffered = '' const timeout = setTimeout(() => reject(new Error(`no ${keys.join('/')} pids`)), 10_000) - child.stdout!.on('data', (chunk: Buffer) => { + const onData = (chunk: Buffer): void => { buffered += chunk.toString() const lines = buffered.split('\n') buffered = lines.pop() ?? '' @@ -86,9 +88,12 @@ function readPids(child: ChildProcess, keys: readonly string[]): Promise key in pids)) { clearTimeout(timeout) + // Later output is not pids; the stream keeps flowing without a listener. + child.stdout!.off('data', onData) resolve(pids) } - }) + } + child.stdout!.on('data', onData) }) } @@ -176,10 +181,14 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe } }) - it('reaps the provider group when its owner dies', async () => { + it.each([ + ['', {}], + // Output after the owner's death meets a closed pipe, which must not end the supervisor first. + [' while the provider is writing output', { ORCA_TEST_PROVIDER_STREAMS_OUTPUT: '1' }] + ])('reaps the provider group when its owner dies%s', async (_, env) => { const launch = supervisedPosixLaunch( { command: process.execPath, args: ['-e', PROVIDER] }, - process.env + { ...process.env, ...env } ) const owner = spawn(process.execPath, ['-e', OWNER], { env: { ...launch.env, ORCA_TEST_SUPERVISOR_SCRIPT: POSIX_PROVIDER_SUPERVISOR_SCRIPT }, diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index a74b72e3b077..f105dedacfd9 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -71,7 +71,10 @@ process.stdin.once('close', scheduleOwnerShutdown) process.stdin.pipe(child.stdin) child.stdout.pipe(process.stdout) child.stderr.pipe(process.stderr) -for (const stream of [process.stdin, child.stdin, child.stdout, child.stderr]) stream.on('error', () => {}) +// A dead owner's stdout pipe raises EPIPE; unhandled, it would end this pid before the group. +for (const stream of [process.stdin, process.stdout, process.stderr, child.stdin, child.stdout, child.stderr]) { + stream.on('error', () => {}) +} const finishWithProviderOutcome = (code, signal) => { if (!signal) return process.exit(code ?? 1) // Re-raise with the default action; this supervisor's own handler would swallow it. From 401380bf15c2d3252cfc37726fd8c71a1f9bd50d Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 14:15:25 -0700 Subject: [PATCH 03/11] fix(codex): bound the supervisor grace so recovery's SIGTERM stage always covers it Recovery sized its SIGTERM stage from the default grace, so a launch with a longer grace would be SIGKILLed mid-stop and orphan its group with no test noticing. The spec now refuses any grace above one exported maximum, and recovery derives its SIGTERM stage from that maximum. --- .../codex-app-server-posix-supervisor.test.ts | 10 ++++++++++ .../codex/codex-app-server-posix-supervisor.ts | 15 +++++++++++---- ...tructured-agent-session-recovery-resolution.ts | 11 ++--------- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/main/codex/codex-app-server-posix-supervisor.test.ts b/src/main/codex/codex-app-server-posix-supervisor.test.ts index 58d58535f1b0..f0c834178905 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.test.ts @@ -3,6 +3,7 @@ import type { CodexAppServerLaunch } from './codex-app-server-connection' import { createProviderSpawnSpec, DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS, + MAX_PROVIDER_SUPERVISOR_GRACE_MS, POSIX_PROVIDER_SUPERVISOR_SCRIPT, supervisedPosixLaunch } from './codex-app-server-posix-supervisor' @@ -52,6 +53,15 @@ describe('structured provider supervision', () => { expect(POSIX_PROVIDER_SUPERVISOR_SCRIPT).not.toContain('process.ppid === 1') }) + it('refuses a grace longer than recovery waits before SIGKILL', () => { + expect(() => + supervisedPosixLaunch(launch, {}, { graceMs: MAX_PROVIDER_SUPERVISOR_GRACE_MS }) + ).not.toThrow() + expect(() => + supervisedPosixLaunch(launch, {}, { graceMs: MAX_PROVIDER_SUPERVISOR_GRACE_MS + 1 }) + ).toThrow(RangeError) + }) + it('uses direct provider spawning on Windows because the job owns the tree', () => { expect(createProviderSpawnSpec(launch, { PATH: '/bin' }, 'win32')).toEqual({ program: '/opt/codex', diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index f105dedacfd9..ca67a82b518d 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -1,12 +1,14 @@ import type { CodexAppServerLaunch } from './codex-app-server-connection' -/** - * Time the provider gets to exit on its own after stdin ends or an owner signal, before SIGKILL. - * Session recovery's SIGTERM window is sized from it; a longer launch grace must widen that too. - */ +/** Time the provider gets to exit on its own after stdin ends or an owner signal, before SIGKILL. */ export const DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS = 1250 +/** Largest grace a spec may carry; raising it widens recovery's SIGTERM stage with it. */ +export const MAX_PROVIDER_SUPERVISOR_GRACE_MS = 1250 /** How long the supervisor waits for a SIGKILLed provider group to disappear. */ export const PROVIDER_GROUP_REAP_TIMEOUT_MS = 1500 +/** Longest a signalled supervisor can outlive the signal; a SIGKILL sooner can orphan its group. */ +export const PROVIDER_SUPERVISOR_MAX_STOP_MS = + MAX_PROVIDER_SUPERVISOR_GRACE_MS + PROVIDER_GROUP_REAP_TIMEOUT_MS /** Inline supervisor source kept dependency-free for the spawned Node child. */ export const POSIX_PROVIDER_SUPERVISOR_SCRIPT = ` @@ -131,6 +133,11 @@ export function supervisedPosixLaunch( graceMs = DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS }: ProviderSupervisorOptions = {} ): { command: string; args: string[]; env: NodeJS.ProcessEnv } { + if (!(graceMs >= 0 && graceMs <= MAX_PROVIDER_SUPERVISOR_GRACE_MS)) { + throw new RangeError( + `Provider supervisor grace ${graceMs} ms is outside 0-${MAX_PROVIDER_SUPERVISOR_GRACE_MS} ms` + ) + } const supervisorSpec = Buffer.from( JSON.stringify({ command: launch.command, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts index 6ef151ebe7bb..36d950a09a48 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts @@ -15,10 +15,7 @@ import { type AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { - DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS, - PROVIDER_GROUP_REAP_TIMEOUT_MS -} from '../../codex/codex-app-server-posix-supervisor' +import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../../codex/codex-app-server-posix-supervisor' import { releaseUnprovenAgentSessionOwner } from '../../runtime/agent-session-lease-transitions' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' @@ -36,11 +33,7 @@ const STOP_PROBE_INTERVAL_MS = 250 // A POSIX Codex owner is its provider supervisor, which exits only after its provider group. A // SIGKILL that lands first leaves the group running, so SIGTERM outlasts the supervisor's stop. const STOP_PROBES: Record = { - SIGTERM: - Math.ceil( - (DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS + PROVIDER_GROUP_REAP_TIMEOUT_MS) / - STOP_PROBE_INTERVAL_MS - ) + 1, + SIGTERM: Math.ceil(PROVIDER_SUPERVISOR_MAX_STOP_MS / STOP_PROBE_INTERVAL_MS) + 1, SIGKILL: 4 } From 3b374050c62c8f5e12397a060ea12f943f29cf2d Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 14:26:34 -0700 Subject: [PATCH 04/11] fix(codex): every supervisor stop asks the provider with SIGTERM first Owner death, stdin end after the grace, and a signal to the supervisor now all take one path: SIGTERM the provider group, SIGKILL it after the grace, and exit only once it is gone. The signal handlers are registered before the provider is spawned, so a stop that lands in the spawn window still reaps it. The longest stop grows to two graces plus the reap wait, and both recovery's SIGTERM stage and the connection's graceful close now wait that long before forcing, since forcing the supervisor sooner can orphan its group. --- .../codex/codex-app-server-connection.test.ts | 12 +- src/main/codex/codex-app-server-connection.ts | 11 +- ...erver-posix-supervisor.integration.test.ts | 133 +++++++++++++----- .../codex-app-server-posix-supervisor.ts | 53 +++---- 4 files changed, 142 insertions(+), 67 deletions(-) diff --git a/src/main/codex/codex-app-server-connection.test.ts b/src/main/codex/codex-app-server-connection.test.ts index aa5a4474235a..face44f13aa9 100644 --- a/src/main/codex/codex-app-server-connection.test.ts +++ b/src/main/codex/codex-app-server-connection.test.ts @@ -10,8 +10,12 @@ import { type CodexAppServerConnection, type CodexAppServerConnectionHandlers } from './codex-app-server-connection' +import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from './codex-app-server-posix-supervisor' import { isCodexAppServerUnsupportedError } from './codex-app-server-session' +// close() waits out the supervisor's own stop before forcing the tree. +const GRACEFUL_EXIT_MS = process.platform === 'win32' ? 1_500 : PROVIDER_SUPERVISOR_MAX_STOP_MS + const originalCodexHome = process.env.CODEX_HOME afterEach(() => { @@ -388,7 +392,7 @@ describe('openCodexAppServerConnection', () => { openCodexAppServerConnection({ command: 'codex', args: ['app-server'] }, {}, spawnImpl) ) - await vi.advanceTimersByTimeAsync(5_000) + await vi.advanceTimersByTimeAsync(GRACEFUL_EXIT_MS + 3_500) const error = (await opening) as Error & { connection?: CodexAppServerConnection } expect(error.name).toBe('CodexAppServerHandshakeExitUnprovenError') @@ -430,7 +434,7 @@ describe('openCodexAppServerConnection', () => { }) const closing = connection.close() - await vi.advanceTimersByTimeAsync(2_000) + await vi.advanceTimersByTimeAsync(GRACEFUL_EXIT_MS + 500) await closing await vi.waitFor(() => expect(child.kill).toHaveBeenCalledWith('SIGKILL')) @@ -464,7 +468,7 @@ describe('openCodexAppServerConnection', () => { const first = connection.close() const second = connection.close() - await vi.advanceTimersByTimeAsync(4_100) + await vi.advanceTimersByTimeAsync(GRACEFUL_EXIT_MS + 2_600) await expect(Promise.all([first, second])).resolves.toEqual([true, true]) expect(child.kill.mock.calls.map(([signal]) => signal)).toEqual(['SIGSTOP', 'SIGKILL']) @@ -481,7 +485,7 @@ describe('openCodexAppServerConnection', () => { ) const first = connection.close() - await vi.advanceTimersByTimeAsync(5_000) + await vi.advanceTimersByTimeAsync(GRACEFUL_EXIT_MS + 3_500) await expect(first).resolves.toBe(false) child.emit('exit', 0, null) diff --git a/src/main/codex/codex-app-server-connection.ts b/src/main/codex/codex-app-server-connection.ts index c0ac7f07400a..b9b6aa20aa1a 100644 --- a/src/main/codex/codex-app-server-connection.ts +++ b/src/main/codex/codex-app-server-connection.ts @@ -1,6 +1,9 @@ import { spawnProcess } from '../../shared/child-process/run-process' import { RetryableProcessExitProof } from '../../shared/child-process/retryable-process-exit-proof' -import { createProviderSpawnSpec } from './codex-app-server-posix-supervisor' +import { + createProviderSpawnSpec, + PROVIDER_SUPERVISOR_MAX_STOP_MS +} from './codex-app-server-posix-supervisor' import { buildCodexAppServerExitError } from './codex-app-server-exit-error' import { initializeCodexAppServerConnection } from './codex-app-server-handshake' import { CodexAppServerHandshakeExitUnprovenError } from './codex-app-server-handshake-exit-proof' @@ -249,7 +252,11 @@ export async function openCodexAppServerConnection( // Already destroyed; the reap below still runs. } if (!exited) { - await waitForProcessExitUntil(exitPromise, GRACEFUL_EXIT_MS) + // The POSIX supervisor stops its own provider group; forcing it any sooner can orphan it. + await waitForProcessExitUntil( + exitPromise, + process.platform === 'win32' ? GRACEFUL_EXIT_MS : PROVIDER_SUPERVISOR_MAX_STOP_MS + ) if (!exited) { const treeExited = await terminateProcessTree() if (!treeExited) { diff --git a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts index 53ef985343a5..8e993bf861f0 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts @@ -1,5 +1,5 @@ import { spawn, type ChildProcess } from 'node:child_process' -import { existsSync, mkdtempSync, rmSync } from 'node:fs' +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' @@ -14,6 +14,12 @@ import { const PROVIDER = String.raw` const { spawn } = require('node:child_process') if (process.env.ORCA_TEST_PROVIDER_IGNORES_SIGTERM) process.on('SIGTERM', () => {}) + if (process.env.ORCA_TEST_PROVIDER_SIGNAL_FILE) { + process.on('SIGTERM', () => { + require('node:fs').writeFileSync(process.env.ORCA_TEST_PROVIDER_SIGNAL_FILE, 'SIGTERM') + process.exit(0) + }) + } process.stdout.on('error', () => {}) const grandchild = spawn( process.execPath, @@ -46,7 +52,26 @@ const OWNER = String.raw` setInterval(() => {}, 60000) ` +// Preloaded into the supervisor: signals it the instant its provider exists, the spawn window. +const SIGNAL_AFTER_SPAWN_PRELOAD = String.raw` + const childProcess = require('node:child_process') + const spawn = childProcess.spawn + childProcess.spawn = (...args) => { + const child = spawn(...args) + require('node:fs').writeFileSync(process.env.ORCA_TEST_PROVIDER_PID_FILE, String(child.pid)) + process.kill(process.pid, 'SIGTERM') + return child + } +` + const recordedPids = new Set() +const tempDirs: string[] = [] + +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'orca-supervisor-')) + tempDirs.push(dir) + return dir +} function alive(pid: number): boolean { try { @@ -103,10 +128,11 @@ function launchSupervisor( provider: { command: string; args: string[] } = { command: process.execPath, args: ['-e', PROVIDER] - } + }, + nodeArgs: string[] = [] ): { supervisor: ChildProcess; exit: Promise<{ code: number | null; signal: string | null }> } { const launch = supervisedPosixLaunch(provider, { ...process.env, ...env }, options) - const supervisor = spawn(launch.command, launch.args, { + const supervisor = spawn(launch.command, [...nodeArgs, ...launch.args], { env: launch.env, stdio: ['pipe', 'pipe', 'ignore'], detached: true @@ -118,6 +144,24 @@ function launchSupervisor( return { supervisor, exit } } +async function launchUnderOwner( + options: ProviderSupervisorOptions, + env: Record = {} +): Promise<{ owner: ChildProcess; pids: Record }> { + const launch = supervisedPosixLaunch( + { command: process.execPath, args: ['-e', PROVIDER] }, + { ...process.env, ...env }, + options + ) + const owner = spawn(process.execPath, ['-e', OWNER], { + env: { ...launch.env, ORCA_TEST_SUPERVISOR_SCRIPT: POSIX_PROVIDER_SUPERVISOR_SCRIPT }, + stdio: ['ignore', 'pipe', 'ignore'] + }) + recordedPids.add(owner.pid!) + const pids = await readPids(owner, ['supervisor', 'holder', 'provider', 'grandchild']) + return { owner, pids } +} + afterEach(() => { for (const pid of recordedPids) { if (alive(pid)) { @@ -125,6 +169,9 @@ afterEach(() => { } } recordedPids.clear() + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } }) describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processes', () => { @@ -163,22 +210,35 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe expect(alive(grandchild)).toBe(false) }) + it('reaps a provider spawned in the instant before a stop arrives', async () => { + const dir = tempDir() + const preload = join(dir, 'signal-after-spawn.js') + const pidFile = join(dir, 'provider-pid') + writeFileSync(preload, SIGNAL_AFTER_SPAWN_PRELOAD) + const { exit } = launchSupervisor( + { graceMs: 300 }, + { ORCA_TEST_PROVIDER_PID_FILE: pidFile }, + { command: process.execPath, args: ['-e', 'setInterval(() => {}, 60000)'] }, + ['--require', preload] + ) + + await expect(exit).resolves.toEqual({ code: null, signal: 'SIGTERM' }) + const provider = Number(readFileSync(pidFile, 'utf8')) + recordedPids.add(provider) + expect(await waitFor(() => !alive(provider), 3_000)).toBe(true) + }) + it('never spawns the provider when its owner is not its parent at start', async () => { - const dir = mkdtempSync(join(tmpdir(), 'orca-supervisor-owner-')) - try { - const marker = join(dir, 'provider-started') - const { exit } = launchSupervisor( - { ownerPid: process.pid === 1 ? 2 : 1 }, - {}, - { command: 'touch', args: [marker] } - ) - - await expect(exit).resolves.toEqual({ code: 1, signal: null }) - await new Promise((resolve) => setTimeout(resolve, 200)) - expect(existsSync(marker)).toBe(false) - } finally { - rmSync(dir, { recursive: true, force: true }) - } + const marker = join(tempDir(), 'provider-started') + const { exit } = launchSupervisor( + { ownerPid: process.pid === 1 ? 2 : 1 }, + {}, + { command: 'touch', args: [marker] } + ) + + await expect(exit).resolves.toEqual({ code: 1, signal: null }) + await new Promise((resolve) => setTimeout(resolve, 200)) + expect(existsSync(marker)).toBe(false) }) it.each([ @@ -186,26 +246,29 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe // Output after the owner's death meets a closed pipe, which must not end the supervisor first. [' while the provider is writing output', { ORCA_TEST_PROVIDER_STREAMS_OUTPUT: '1' }] ])('reaps the provider group when its owner dies%s', async (_, env) => { - const launch = supervisedPosixLaunch( - { command: process.execPath, args: ['-e', PROVIDER] }, - { ...process.env, ...env } + const graceMs = 300 + const { owner, pids } = await launchUnderOwner({ graceMs }, env) + + const killedAt = Date.now() + owner.kill('SIGKILL') + + expect(await waitFor(() => !alive(-pids.provider), 3_000)).toBe(true) + // The grandchild ignores SIGTERM, so the group lasts until the grace ends in SIGKILL. + expect(Date.now() - killedAt).toBeGreaterThanOrEqual(graceMs) + expect(await waitFor(() => !alive(pids.supervisor), 3_000)).toBe(true) + expect(alive(pids.grandchild)).toBe(false) + }) + + it('asks the provider to stop with SIGTERM when its owner dies', async () => { + const signalFile = join(tempDir(), 'provider-signal') + const { owner, pids } = await launchUnderOwner( + { graceMs: 300 }, + { ORCA_TEST_PROVIDER_SIGNAL_FILE: signalFile } ) - const owner = spawn(process.execPath, ['-e', OWNER], { - env: { ...launch.env, ORCA_TEST_SUPERVISOR_SCRIPT: POSIX_PROVIDER_SUPERVISOR_SCRIPT }, - stdio: ['ignore', 'pipe', 'ignore'] - }) - recordedPids.add(owner.pid!) - const { supervisor, provider, grandchild } = await readPids(owner, [ - 'supervisor', - 'holder', - 'provider', - 'grandchild' - ]) owner.kill('SIGKILL') - expect(await waitFor(() => !alive(-provider), 3_000)).toBe(true) - expect(await waitFor(() => !alive(supervisor), 3_000)).toBe(true) - expect(alive(grandchild)).toBe(false) + expect(await waitFor(() => !alive(-pids.provider), 3_000)).toBe(true) + expect(existsSync(signalFile) && readFileSync(signalFile, 'utf8')).toBe('SIGTERM') }) }) diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index ca67a82b518d..7327c8258c60 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -1,14 +1,17 @@ import type { CodexAppServerLaunch } from './codex-app-server-connection' -/** Time the provider gets to exit on its own after stdin ends or an owner signal, before SIGKILL. */ +/** Time the provider gets after stdin ends before SIGTERM, and after SIGTERM before SIGKILL. */ export const DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS = 1250 /** Largest grace a spec may carry; raising it widens recovery's SIGTERM stage with it. */ export const MAX_PROVIDER_SUPERVISOR_GRACE_MS = 1250 /** How long the supervisor waits for a SIGKILLed provider group to disappear. */ export const PROVIDER_GROUP_REAP_TIMEOUT_MS = 1500 -/** Longest a signalled supervisor can outlive the signal; a SIGKILL sooner can orphan its group. */ +/** + * Longest a supervisor can take to stop once asked (stdin end, grace, SIGTERM, grace, SIGKILL, + * reap); a SIGKILL sooner can orphan its group. + */ export const PROVIDER_SUPERVISOR_MAX_STOP_MS = - MAX_PROVIDER_SUPERVISOR_GRACE_MS + PROVIDER_GROUP_REAP_TIMEOUT_MS + 2 * MAX_PROVIDER_SUPERVISOR_GRACE_MS + PROVIDER_GROUP_REAP_TIMEOUT_MS /** Inline supervisor source kept dependency-free for the spawned Node child. */ export const POSIX_PROVIDER_SUPERVISOR_SCRIPT = ` @@ -17,6 +20,8 @@ const spec = JSON.parse(Buffer.from(process.env.ORCA_PROVIDER_SUPERVISOR_SPEC, ' // A detached supervisor is reparented when its owner exits. The new parent may // be PID 1 or a platform subreaper, so any other parent means no live owner. const ownerGone = () => process.ppid !== spec.ownerPid +// Registered before the spawn, so a stop that lands while the provider starts still reaps it. +for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) process.on(signal, () => stopProviderGroup(signal)) // Orca can die before this runs; spawning then would start a provider nothing watches. if (ownerGone()) process.exit(1) const childEnv = { ...process.env } @@ -55,17 +60,32 @@ const reapOwnedProviderGroup = async () => { } return waitForProviderGroupExit(${PROVIDER_GROUP_REAP_TIMEOUT_MS}) } -const terminateOwnedGroup = () => { +const finishWithProviderOutcome = (code, signal) => { + if (!signal) return process.exit(code ?? 1) + // Re-raise with the default action; this supervisor's own handler would swallow it. + process.removeAllListeners(signal) + process.kill(process.pid, signal) +} +// Every stop is the same: SIGTERM the group, SIGKILL it after the grace, and exit only once it is +// gone. Whoever stops this pid judges the provider by it, so a dead supervisor means a dead group. +const stopProviderGroup = (receivedSignal) => { if (settling) return settling = true clearInterval(timer) - void reapOwnedProviderGroup().then((reaped) => process.exit(reaped ? 137 : 1)) + if (ownerShutdownTimer) clearTimeout(ownerShutdownTimer) + try { process.kill(-child.pid, 'SIGTERM') } catch {} + void waitForProviderGroupExit(spec.graceMs) + .then((exited) => exited || reapOwnedProviderGroup()) + .then((reaped) => { + if (!reaped) return process.exit(1) + finishWithProviderOutcome(137, receivedSignal) + }) } const scheduleOwnerShutdown = () => { if (settling || ownerShutdownTimer) return // A normal close ends the provider's stdin first; allow it to flush and // exit before forcing the group, while still bounding an orphaned child. - ownerShutdownTimer = setTimeout(terminateOwnedGroup, spec.graceMs) + ownerShutdownTimer = setTimeout(() => stopProviderGroup(null), spec.graceMs) ownerShutdownTimer.unref() } process.stdin.once('end', scheduleOwnerShutdown) @@ -77,12 +97,6 @@ child.stderr.pipe(process.stderr) for (const stream of [process.stdin, process.stdout, process.stderr, child.stdin, child.stdout, child.stderr]) { stream.on('error', () => {}) } -const finishWithProviderOutcome = (code, signal) => { - if (!signal) return process.exit(code ?? 1) - // Re-raise with the default action; this supervisor's own handler would swallow it. - process.removeAllListeners(signal) - process.kill(process.pid, signal) -} const reapProviderExit = async (code, signal) => { if (settling) return settling = true @@ -91,21 +105,8 @@ const reapProviderExit = async (code, signal) => { if (!(await reapOwnedProviderGroup())) return process.exit(1) finishWithProviderOutcome(code, signal) } -// Whoever stops this pid is judging the provider by it: stay alive until the group is gone, -// so a dead supervisor always means a dead provider group. -const stopOnOwnerSignal = (signal) => { - if (settling) return - settling = true - clearInterval(timer) - if (ownerShutdownTimer) clearTimeout(ownerShutdownTimer) - try { process.kill(-child.pid, signal) } catch {} - void waitForProviderGroupExit(spec.graceMs) - .then((exited) => exited || reapOwnedProviderGroup()) - .then((reaped) => (reaped ? finishWithProviderOutcome(null, signal) : process.exit(1))) -} -for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) process.on(signal, () => stopOnOwnerSignal(signal)) timer = setInterval(() => { - if (ownerGone()) terminateOwnedGroup() + if (ownerGone()) stopProviderGroup(null) }, 100) timer.unref() child.once('error', () => { From d146ae851df4d8aa22f8c12404fd674099cfdcff Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 14:28:58 -0700 Subject: [PATCH 05/11] fix(claude): run the structured Claude child under the POSIX provider supervisor A close now stops Claude with a SIGTERM through the supervisor instead of letting stdin end finish the turn, and Orca's death stops it through the supervisor. --- .../claude-agent-sdk-process-spawn.test.ts | 35 ++- .../claude/claude-agent-sdk-process-spawn.ts | 35 ++- .../claude-child-exit-proof-ladder.test.ts | 72 ++++++ .../claude/claude-child-exit-proof-ladder.ts | 17 +- .../claude/claude-child-root-termination.ts | 3 + .../claude-stream-json-connection.test.ts | 101 +++++--- .../claude/claude-stream-json-connection.ts | 7 +- ...claude-supervised-stop.integration.test.ts | 222 ++++++++++++++++++ .../codex-app-server-posix-supervisor.ts | 5 +- ...tured-agent-session-recovery-resolution.ts | 4 +- 10 files changed, 453 insertions(+), 48 deletions(-) create mode 100644 src/main/claude/claude-child-exit-proof-ladder.test.ts create mode 100644 src/main/claude/claude-supervised-stop.integration.test.ts diff --git a/src/main/claude/claude-agent-sdk-process-spawn.test.ts b/src/main/claude/claude-agent-sdk-process-spawn.test.ts index cd3520cf6d53..e972174c7e24 100644 --- a/src/main/claude/claude-agent-sdk-process-spawn.test.ts +++ b/src/main/claude/claude-agent-sdk-process-spawn.test.ts @@ -43,7 +43,7 @@ function sdkOptions(overrides: Partial = {}): SdkSpawnOptions { describe('claude agent SDK process spawn', () => { it('routes the SDK spawn through Orca and retains the pid the lease adjudicates on', () => { const process = fakeSpawn() - const spawn = createClaudeCodeProcessSpawn(process.spawnImpl) + const spawn = createClaudeCodeProcessSpawn(process.spawnImpl, 'win32') expect(spawn.pid).toBeUndefined() expect(spawn.child).toBeNull() @@ -52,15 +52,46 @@ describe('claude agent SDK process spawn', () => { expect(child).toBe(process.child) expect(spawn.child).toBe(process.child) expect(spawn.pid).toBe(4321) + // Windows has no supervisor: Claude itself is the child. + expect(spawn.supervised).toBe(false) expect(process.specs[0]).toEqual({ program: '/usr/local/bin/claude', args: ['--output-format', 'stream-json'], cwd: '/work/repo', env: { PATH: '/usr/bin', CLAUDE_CONFIG_DIR: '/accounts/one' }, + detached: false, stdio: ['pipe', 'pipe', 'pipe'] }) }) + it.each(['darwin', 'linux'] as const)( + 'starts Claude under the provider supervisor on %s, which is then the pid the lease records', + (platform) => { + const process = fakeSpawn() + const spawn = createClaudeCodeProcessSpawn(process.spawnImpl, platform) + spawn.spawn(sdkOptions()) + + const spec = process.specs[0] as ProcessSpec + expect(spawn.supervised).toBe(true) + expect(spawn.pid).toBe(4321) + expect(spec.program).toBe(globalThis.process.execPath) + expect(spec.args?.[0]).toBe('-e') + expect(spec.detached).toBe(true) + expect(spec.cwd).toBe('/work/repo') + const supervisorSpec = JSON.parse( + Buffer.from(String(spec.env?.ORCA_PROVIDER_SUPERVISOR_SPEC), 'base64').toString() + ) + expect(supervisorSpec).toMatchObject({ + command: '/usr/local/bin/claude', + args: ['--output-format', 'stream-json'], + cwd: '/work/repo', + ownerPid: globalThis.process.pid + }) + // The supervisor passes its env to Claude minus its own two keys. + expect(spec.env).toMatchObject({ PATH: '/usr/bin', CLAUDE_CONFIG_DIR: '/accounts/one' }) + } + ) + it('keeps the child out of the SDK abort path so exit proof stays Orca-owned', () => { const process = fakeSpawn() const controller = new AbortController() @@ -86,7 +117,7 @@ describe('claude agent SDK process spawn', () => { it('hands a Windows .cmd shim to Orca\u2019s argument encoder', () => { const process = fakeSpawn() - createClaudeCodeProcessSpawn(process.spawnImpl).spawn( + createClaudeCodeProcessSpawn(process.spawnImpl, 'win32').spawn( sdkOptions({ command: 'C:\\Users\\dev\\AppData\\npm\\claude.cmd', args: ['--setting-sources=user,project,local', '--session-id', 'a b&c'] diff --git a/src/main/claude/claude-agent-sdk-process-spawn.ts b/src/main/claude/claude-agent-sdk-process-spawn.ts index a2b1ad7f1585..c0691bfffbcf 100644 --- a/src/main/claude/claude-agent-sdk-process-spawn.ts +++ b/src/main/claude/claude-agent-sdk-process-spawn.ts @@ -1,5 +1,6 @@ import type { SpawnOptions as ClaudeAgentSdkSpawnOptions } from '@anthropic-ai/claude-agent-sdk' import { spawnProcess } from '../../shared/child-process/run-process' +import { createProviderSpawnSpec } from '../codex/codex-app-server-posix-supervisor' /** Derived rather than imported: only src/shared/child-process may name node:child_process. */ type ClaudeCodeChild = ReturnType @@ -11,8 +12,13 @@ export type ClaudeCodeProcessSpawn = { spawn: (options: ClaudeAgentSdkSpawnOptions) => ClaudeCodeChild /** The retained child, so Orca keeps its own tree-kill and exit-proof ladder. Null until the SDK spawns. */ readonly child: ClaudeCodeChild | null - /** Ownership proof: the durable lease adjudicates on this pid plus start time plus the spawn token. */ + /** + * Ownership proof: the durable lease adjudicates on this pid plus start time plus the spawn + * token. On POSIX it is the provider supervisor's, which outlives Claude by construction. + */ readonly pid: number | undefined + /** Whether the child is the POSIX supervisor, whose SIGTERM stops Claude and then itself. */ + readonly supervised: boolean readonly stderrTail: string } @@ -31,21 +37,37 @@ function definedEnv(env: Record): Record { + const spec = createProviderSpawnSpec( + { + command: options.command, + args: [...options.args], + ...(options.cwd === undefined ? {} : { cwd: options.cwd }) + }, + definedEnv(options.env), + platform + ) // Why `options.signal` is dropped: it would let the SDK kill the child outside // Orca's ladder, and close() may never report an exit it did not observe. const spawned = spawnImpl({ - program: options.command, - args: [...options.args], - ...(options.cwd === undefined ? {} : { cwd: options.cwd }), - env: definedEnv(options.env), + program: spec.program, + args: spec.args, + cwd: spec.cwd, + env: spec.env, + detached: spec.detached, stdio: ['pipe', 'pipe', 'pipe'] }) child = spawned @@ -62,6 +84,7 @@ export function createClaudeCodeProcessSpawn( get pid() { return child?.pid }, + supervised, get stderrTail() { return stderrTail } diff --git a/src/main/claude/claude-child-exit-proof-ladder.test.ts b/src/main/claude/claude-child-exit-proof-ladder.test.ts new file mode 100644 index 000000000000..2d8da7f629dc --- /dev/null +++ b/src/main/claude/claude-child-exit-proof-ladder.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it, vi } from 'vitest' +import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../codex/codex-app-server-posix-supervisor' +import type { ClaudeChildTreeReaper } from './claude-agent-sdk-exit-proof' +import { proveClaudeChildExitWithReaper } from './claude-child-exit-proof-ladder' + +function fakeTree(): ClaudeChildTreeReaper & { reap: ReturnType } { + return { + capture: vi.fn(async () => {}), + refresh: vi.fn(async () => {}), + reap: vi.fn(async () => 'exited' as const), + treeVerdict: 'exited' + } +} + +/** A root that leaves only once a SIGTERM has had `stopMs` to act, the way a supervisor does. */ +function rootStoppedBySigterm(stopMs: number) { + let exited = false + let settle = (): void => {} + const exitPromise = new Promise((resolve) => { + settle = resolve + }) + const kill = vi.fn((signal?: NodeJS.Signals | number) => { + if (signal === 'SIGTERM') { + setTimeout(() => { + exited = true + settle() + }, stopMs) + } + return true + }) + const stdin = { end: vi.fn() } + return { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The ladder reads only pid, kill and stdin.end from its child. + child: { pid: 4321, kill, stdin } as unknown as Parameters< + typeof proveClaudeChildExitWithReaper + >[0]['child'], + kill, + stdin, + exitPromise, + exited: () => exited + } +} + +describe('Claude child exit proof ladder', () => { + it('stops a supervised child with SIGTERM and waits out the supervisor stop before forcing', async () => { + // Slower than the unsupervised 1.5 s grace, still inside the supervisor's own bound. + const root = rootStoppedBySigterm(PROVIDER_SUPERVISOR_MAX_STOP_MS - 500) + const tree = fakeTree() + + await expect( + proveClaudeChildExitWithReaper({ ...root, supervised: true, tree }, () => tree) + ).resolves.toBe(true) + + expect(root.stdin.end).toHaveBeenCalled() + expect(root.kill).toHaveBeenCalledWith('SIGTERM') + // Forcing here would SIGKILL the supervisor mid-stop and orphan Claude in its own group. + expect(root.kill).not.toHaveBeenCalledWith('SIGKILL') + expect(tree.reap).not.toHaveBeenCalled() + }, 10_000) + + it('never signals an unsupervised child for the graceful stop', async () => { + const root = rootStoppedBySigterm(0) + const tree = fakeTree() + + await proveClaudeChildExitWithReaper({ ...root, tree }, () => tree) + + // On Windows a direct SIGTERM is TerminateProcess: stdin end stays the only graceful rung. + expect(root.stdin.end).toHaveBeenCalled() + expect(root.kill).not.toHaveBeenCalledWith('SIGTERM') + expect(tree.reap).toHaveBeenCalled() + }, 10_000) +}) diff --git a/src/main/claude/claude-child-exit-proof-ladder.ts b/src/main/claude/claude-child-exit-proof-ladder.ts index 85ed629f1b92..881a6ffd81ff 100644 --- a/src/main/claude/claude-child-exit-proof-ladder.ts +++ b/src/main/claude/claude-child-exit-proof-ladder.ts @@ -1,8 +1,11 @@ import type { SpawnedProcess } from '../../shared/child-process/run-process' import { waitForProcessExitUntil } from '../codex/codex-process-exit-deadline' +import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../codex/codex-app-server-posix-supervisor' import type { ClaudeChildTreeReaper } from './claude-agent-sdk-exit-proof' const GRACEFUL_EXIT_MS = 1_500 +// A signalled supervisor escalates on its own; forcing it sooner kills it and orphans Claude. +export const SUPERVISED_GRACEFUL_EXIT_MS = PROVIDER_SUPERVISOR_MAX_STOP_MS + 500 const FORCED_EXIT_MS = 1_000 export type ClaudeChildExitProofInput = { @@ -10,6 +13,8 @@ export type ClaudeChildExitProofInput = { exitPromise: Promise exited: () => boolean tree?: ClaudeChildTreeReaper + /** The child is the POSIX provider supervisor: SIGTERM stops Claude, which reaps its tools. */ + supervised?: boolean } export async function proveClaudeChildExitWithReaper( @@ -17,16 +22,24 @@ export async function proveClaudeChildExitWithReaper( createTree: () => ClaudeChildTreeReaper ): Promise { const tree = input.tree ?? createTree() - // Arm before stdin closes: only a live root can identify its descendants. + // Arm before the stop: only a live root can identify its descendants. await tree.capture() try { input.child.stdin?.end() } catch { // The reap below still owns the process. } + // Stdin end alone lets Claude finish its turn, tools and edits included; a close is a stop. + // Windows has no supervisor, and a direct SIGTERM there is TerminateProcess. + if (input.supervised && !input.exited()) { + input.child.kill('SIGTERM') + } let reaped = false if (!input.exited()) { - await waitForProcessExitUntil(input.exitPromise, GRACEFUL_EXIT_MS) + await waitForProcessExitUntil( + input.exitPromise, + input.supervised ? SUPERVISED_GRACEFUL_EXIT_MS : GRACEFUL_EXIT_MS + ) if (!input.exited()) { reaped = true await tree.refresh?.() diff --git a/src/main/claude/claude-child-root-termination.ts b/src/main/claude/claude-child-root-termination.ts index bed422532e19..bba299195789 100644 --- a/src/main/claude/claude-child-root-termination.ts +++ b/src/main/claude/claude-child-root-termination.ts @@ -19,6 +19,9 @@ type RootTerminationInput = { * nothing. A probe here could only let an unreadable process table cost the tree * the one fallback that still works once every table read has failed. * + * On POSIX the root is the provider supervisor, killed only after its own stop had + * its whole bound; Claude, in its own group, is reached by the descendant kill. + * * False means no signal was sent, because the root had already left. */ export function terminateClaudeRoot(input: RootTerminationInput): boolean { diff --git a/src/main/claude/claude-stream-json-connection.test.ts b/src/main/claude/claude-stream-json-connection.test.ts index 51ffa085163d..6af37e2e2025 100644 --- a/src/main/claude/claude-stream-json-connection.test.ts +++ b/src/main/claude/claude-stream-json-connection.test.ts @@ -106,6 +106,18 @@ async function open( return connection } +function launchedArgv(spec: ProcessSpec | undefined): string[] { + const supervised = spec?.env?.ORCA_PROVIDER_SUPERVISOR_SPEC + if (!supervised) { + return [spec?.program ?? '', ...(spec?.args ?? [])] + } + const launch: unknown = JSON.parse(Buffer.from(supervised, 'base64').toString()) + if (!launch || typeof launch !== 'object' || !('command' in launch) || !('args' in launch)) { + return [] + } + return [String(launch.command), ...(Array.isArray(launch.args) ? launch.args.map(String) : [])] +} + function childEnv(): Record { return (spawned.at(-1)?.env ?? {}) as Record } @@ -195,8 +207,9 @@ describe('Claude stream-json connection', () => { const report = await until(() => readReportSafely(scenario), 'the scripted CLI report') expect(report.argv[0]).toBe(FAKE_CLI) // The .mjs fixture makes the SDK run it under node; a real CLI path is the program - // itself. Either way the resolved path is what Orca's spawner is asked to execute. - expect([spawned.at(-1)?.program, ...(spawned.at(-1)?.args ?? [])]).toContain(FAKE_CLI) + // itself. Either way the resolved path is what Orca's spawner is asked to execute, + // through the POSIX supervisor's spec where there is one. + expect(launchedArgv(spawned.at(-1))).toContain(FAKE_CLI) expect(report.argv).toContain('--replay-user-messages') expect(report.argv).toContain(`--session-id=${SESSION_ID}`) }) @@ -636,37 +649,63 @@ describe('Claude stream-json connection', () => { 20_000 ) - it('settles a spawn error followed by close as processless and closes idempotently', async () => { - const scenario = scriptScenario([HOLD_OPEN]) - const missingCli = join(scenario.cwd, 'claude-that-does-not-exist') - let fault: Error | null = null - let exit: Error | null = null - const connection = await open( - { ...launchFor(scenario), pathToClaudeCodeExecutable: missingCli }, - { - onFault: (error) => { - fault = error - }, - onExit: (error) => { - exit = error + it.skipIf(process.platform === 'win32')( + 'reports a missing CLI under the supervisor as a root exit that names the spawn error', + async () => { + const scenario = scriptScenario([HOLD_OPEN]) + const missingCli = join(scenario.cwd, 'claude-that-does-not-exist') + let exit: Error | null = null + const connection = await open( + { ...launchFor(scenario), pathToClaudeCodeExecutable: missingCli }, + { + onExit: (error) => { + exit = error + } } - } - ) + ) - await until( - () => (connection.exitVerdict.root === 'processless' ? connection.exitVerdict : null), - 'the processless spawn settlement' - ) - expect(connection.pid).toBeUndefined() - expect(fault).toBeInstanceOf(Error) - expect(exit).toBeNull() - await expect(Promise.all([connection.close(), connection.close()])).resolves.toEqual([ - true, - true - ]) - await expect(connection.close()).resolves.toBe(true) - expect(connection.exitVerdict).toEqual({ root: 'processless', tree: 'exited' }) - }) + const reported = await until(() => exit, 'the supervised spawn failure') + // The supervisor spawned, so this is its exit; only its stderr can say why. + expect(reported.message).toMatch(/\(code 127\).*ENOENT/s) + // A first-hand root exit, which releases the lease like a processless start did. + expect(connection.exitVerdict.root).toBe('exited') + } + ) + + it.skipIf(process.platform !== 'win32')( + 'settles a spawn error followed by close as processless and closes idempotently', + async () => { + const scenario = scriptScenario([HOLD_OPEN]) + const missingCli = join(scenario.cwd, 'claude-that-does-not-exist') + let fault: Error | null = null + let exit: Error | null = null + const connection = await open( + { ...launchFor(scenario), pathToClaudeCodeExecutable: missingCli }, + { + onFault: (error) => { + fault = error + }, + onExit: (error) => { + exit = error + } + } + ) + + await until( + () => (connection.exitVerdict.root === 'processless' ? connection.exitVerdict : null), + 'the processless spawn settlement' + ) + expect(connection.pid).toBeUndefined() + expect(fault).toBeInstanceOf(Error) + expect(exit).toBeNull() + await expect(Promise.all([connection.close(), connection.close()])).resolves.toEqual([ + true, + true + ]) + await expect(connection.close()).resolves.toBe(true) + expect(connection.exitVerdict).toEqual({ root: 'processless', tree: 'exited' }) + } + ) it('does not treat a child error event as first-hand root exit proof', async () => { const scenario = scriptScenario([HOLD_OPEN]) diff --git a/src/main/claude/claude-stream-json-connection.ts b/src/main/claude/claude-stream-json-connection.ts index c6da4419f79d..e9fd8653252b 100644 --- a/src/main/claude/claude-stream-json-connection.ts +++ b/src/main/claude/claude-stream-json-connection.ts @@ -303,15 +303,16 @@ export async function openClaudeStreamJsonConnection( closePromise ??= (async () => { closing = true resumeReading() - // Arm the descendant proof before ending stdin. The SDK may exit the root - // immediately; a post-exit walk cannot recover descendants that reparented. + // Arm the descendant proof before the stop. The root may exit immediately; + // a post-exit walk cannot recover descendants that reparented. await (tree.refresh?.() ?? tree.capture()) inbox.end() const proven = await proveClaudeChildExit({ child, exitPromise, exited: rootSettled, - tree + tree, + supervised: spawner.supervised }) inbox.fail(new Error('claude stream-json connection closed')) if (!proven) { diff --git a/src/main/claude/claude-supervised-stop.integration.test.ts b/src/main/claude/claude-supervised-stop.integration.test.ts new file mode 100644 index 000000000000..1a15b8a778d2 --- /dev/null +++ b/src/main/claude/claude-supervised-stop.integration.test.ts @@ -0,0 +1,222 @@ +import { spawn, type ChildProcess } from 'node:child_process' +import { existsSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import type { SpawnOptions as SdkSpawnOptions } from '@anthropic-ai/claude-agent-sdk' +import { spawnProcess } from '../../shared/child-process/run-process' +import type { ProcessSpec } from '../../shared/child-process/process-spec' +import { + DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS, + PROVIDER_SUPERVISOR_MAX_STOP_MS +} from '../codex/codex-app-server-posix-supervisor' +import { proveClaudeChildExit } from './claude-agent-sdk-exit-proof' +import { createClaudeCodeProcessSpawn } from './claude-agent-sdk-process-spawn' + +// Stands in for Claude mid-turn: stdin end does not stop it, the way EOF lets the real CLI finish +// its turn. Its tool leads its own group, as the CLI's Bash tool shells do, and only Claude's own +// SIGTERM handler reaps it. The daemon double-forks out of the tree before anything looks. +const FAKE_CLAUDE = String.raw` + const { spawn } = require('node:child_process') + const { writeFileSync } = require('node:fs') + process.stdin.resume() + process.stdout.on('error', () => {}) + const tool = spawn(process.execPath, ['-e', 'setInterval(() => {}, 60000)'], { + detached: true, + stdio: 'ignore' + }) + const forker = spawn( + process.execPath, + ['-e', "const d = require('node:child_process').spawn(process.execPath, ['-e', 'setInterval(() => {}, 60000)'], { detached: true, stdio: 'ignore' }); d.unref(); process.stdout.write(String(d.pid))"], + { stdio: ['ignore', 'pipe', 'ignore'] } + ) + let daemon = '' + forker.stdout.on('data', (chunk) => { daemon += chunk }) + forker.once('exit', () => { + process.stdout.write(JSON.stringify({ claude: process.pid, tool: tool.pid, daemon: Number(daemon) }) + '\n') + }) + process.on('SIGTERM', () => { + if (process.env.ORCA_TEST_CLAUDE_IGNORES_SIGTERM) return + try { process.kill(-tool.pid, 'SIGKILL') } catch {} + writeFileSync(process.env.ORCA_TEST_SIGTERM_MARKER, 'reaped') + process.exit(143) + }) + setInterval(() => {}, 60000) +` + +// Stands in for Orca's main: starts exactly the spec Claude's spawner built, then can be SIGKILLed. +const OWNER = String.raw` + const { spawn } = require('node:child_process') + const spec = JSON.parse(process.env.ORCA_TEST_SPAWN_SPEC) + const env = { ...spec.env } + if (env.ORCA_PROVIDER_SUPERVISOR_SPEC) { + const supervisor = JSON.parse(Buffer.from(env.ORCA_PROVIDER_SUPERVISOR_SPEC, 'base64').toString()) + supervisor.ownerPid = process.pid + env.ORCA_PROVIDER_SUPERVISOR_SPEC = Buffer.from(JSON.stringify(supervisor)).toString('base64') + } + const child = spawn(spec.program, spec.args, { + cwd: spec.cwd, + env, + detached: spec.detached, + stdio: ['pipe', 'pipe', 'ignore'] + }) + process.stdout.write(JSON.stringify({ root: child.pid }) + '\n') + child.stdout.pipe(process.stdout) + setInterval(() => {}, 60000) +` + +const recordedPids = new Set() +const tempDirs: string[] = [] + +function alive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch (error) { + return !(error instanceof Error && 'code' in error && error.code === 'ESRCH') + } +} + +async function waitFor(predicate: () => boolean, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs + while (!predicate()) { + if (Date.now() >= deadline) { + return false + } + await new Promise((resolve) => setTimeout(resolve, 25)) + } + return true +} + +function readPids(child: ChildProcess, keys: readonly string[]): Promise> { + return new Promise((resolve, reject) => { + const pids: Record = {} + let buffered = '' + const timeout = setTimeout(() => reject(new Error(`no ${keys.join('/')} pids`)), 10_000) + const onData = (chunk: Buffer): void => { + buffered += chunk.toString() + const lines = buffered.split('\n') + buffered = lines.pop() ?? '' + for (const line of lines) { + const parsed: unknown = JSON.parse(line) + for (const [key, pid] of Object.entries(parsed ?? {})) { + if (typeof pid === 'number' && pid > 0) { + pids[key] = pid + recordedPids.add(pid) + } + } + } + if (keys.every((key) => key in pids)) { + clearTimeout(timeout) + child.stdout!.off('data', onData) + child.stdout!.resume() + resolve(pids) + } + } + child.stdout!.on('data', onData) + }) +} + +function sdkOptions(env: Record): SdkSpawnOptions { + const dir = mkdtempSync(join(tmpdir(), 'orca-claude-supervised-')) + tempDirs.push(dir) + return { + command: process.execPath, + args: ['-e', FAKE_CLAUDE], + cwd: dir, + env: { + ...process.env, + ORCA_TEST_SIGTERM_MARKER: join(dir, 'sigterm-reap'), + ...env + }, + signal: new AbortController().signal + } +} + +/** Claude spawned through Orca's own spawner and stopped by Orca's own close ladder. */ +async function spawnClaude(env: Record = {}) { + const spawner = createClaudeCodeProcessSpawn(spawnProcess) + const options = sdkOptions(env) + const child = spawner.spawn(options) + recordedPids.add(child.pid!) + let exited = false + const exit = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => + child.once('exit', (code, signal) => { + exited = true + resolve({ code, signal }) + }) + ) + const pids = await readPids(child, ['claude', 'tool', 'daemon']) + const close = (): Promise => + proveClaudeChildExit({ + child, + exitPromise: exit.then(() => undefined), + exited: () => exited, + supervised: spawner.supervised + }) + return { child, exit, pids, close, marker: String(options.env.ORCA_TEST_SIGTERM_MARKER) } +} + +afterEach(() => { + for (const pid of recordedPids) { + if (alive(pid)) { + process.kill(pid, 'SIGKILL') + } + } + recordedPids.clear() + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } +}) + +describe.runIf(process.platform !== 'win32')('Claude under the POSIX provider supervisor', () => { + it('stops a mid-turn Claude on close instead of letting stdin end finish its turn', async () => { + const { child, exit, pids, close, marker } = await spawnClaude() + expect(child.pid).not.toBe(pids.claude) + + const startedAt = Date.now() + await expect(close()).resolves.toBe(true) + + // Claude's own SIGTERM reap ran, well before the supervisor's stdin-end grace would force it. + expect(Date.now() - startedAt).toBeLessThan(DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS) + expect(existsSync(marker)).toBe(true) + await expect(exit).resolves.toEqual({ code: null, signal: 'SIGTERM' }) + expect(alive(pids.claude)).toBe(false) + expect(alive(pids.tool)).toBe(false) + }) + + it('lets the supervisor escalate a Claude that ignores SIGTERM, and exits only after it', async () => { + const { exit, pids, close } = await spawnClaude({ ORCA_TEST_CLAUDE_IGNORES_SIGTERM: '1' }) + + const startedAt = Date.now() + await expect(close()).resolves.toBe(true) + + const elapsed = Date.now() - startedAt + expect(elapsed).toBeGreaterThanOrEqual(DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS) + expect(elapsed).toBeLessThan(PROVIDER_SUPERVISOR_MAX_STOP_MS + 1_000) + // The supervisor's own SIGTERM stop finished the job; nothing forced the supervisor itself. + await expect(exit).resolves.toEqual({ code: null, signal: 'SIGTERM' }) + expect(alive(pids.claude)).toBe(false) + }) + + it("stops Claude when Orca's main dies, and leaves a daemon that left its tree alone", async () => { + const specs: ProcessSpec[] = [] + createClaudeCodeProcessSpawn((spec) => { + specs.push(spec) + return spawnProcess({ program: 'true' }) + }).spawn(sdkOptions({})) + const owner = spawn(process.execPath, ['-e', OWNER], { + env: { ...process.env, ORCA_TEST_SPAWN_SPEC: JSON.stringify(specs[0]) }, + stdio: ['ignore', 'pipe', 'ignore'] + }) + recordedPids.add(owner.pid!) + const pids = await readPids(owner, ['root', 'claude', 'tool', 'daemon']) + + owner.kill('SIGKILL') + + expect(await waitFor(() => !alive(pids.claude), PROVIDER_SUPERVISOR_MAX_STOP_MS)).toBe(true) + expect(await waitFor(() => !alive(pids.root), PROVIDER_SUPERVISOR_MAX_STOP_MS)).toBe(true) + // Not the conversation's writer: nothing proves it orphaned, so the stop never reaches it. + expect(alive(pids.daemon)).toBe(true) + }) +}) diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index ca67a82b518d..15ef15a2a87e 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -108,9 +108,10 @@ timer = setInterval(() => { if (ownerGone()) terminateOwnedGroup() }, 100) timer.unref() -child.once('error', () => { +child.once('error', (error) => { clearInterval(timer) - process.exit(127) + // The owner sees only this pid's exit; stderr is where a missing provider binary can say so. + process.stderr.write(String(error && error.message) + '\\n', () => process.exit(127)) }) child.once('exit', (code, signal) => { void reapProviderExit(code, signal) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts index 36d950a09a48..385d438f7a04 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts @@ -30,8 +30,8 @@ export type StructuredSessionRecoveryResolutionDeps = { } const STOP_PROBE_INTERVAL_MS = 250 -// A POSIX Codex owner is its provider supervisor, which exits only after its provider group. A -// SIGKILL that lands first leaves the group running, so SIGTERM outlasts the supervisor's stop. +// A POSIX structured owner is its provider supervisor, which exits only after its provider +// group. A SIGKILL that lands first leaves the group running, so SIGTERM outlasts its stop. const STOP_PROBES: Record = { SIGTERM: Math.ceil(PROVIDER_SUPERVISOR_MAX_STOP_MS / STOP_PROBE_INTERVAL_MS) + 1, SIGKILL: 4 From 2fbd0ca87d00dbdd0c0ee13baaf84cf28605db34 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 14:33:46 -0700 Subject: [PATCH 06/11] test(claude): pin the supervised stop against a real Claude CLI, opt-in --- .../claude-agent-sdk-process-spawn.test.ts | 5 +- .../claude-supervised-stop-real-cli.test.ts | 241 ++++++++++++++++++ 2 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 src/main/claude/claude-supervised-stop-real-cli.test.ts diff --git a/src/main/claude/claude-agent-sdk-process-spawn.test.ts b/src/main/claude/claude-agent-sdk-process-spawn.test.ts index e972174c7e24..90056c18ff08 100644 --- a/src/main/claude/claude-agent-sdk-process-spawn.test.ts +++ b/src/main/claude/claude-agent-sdk-process-spawn.test.ts @@ -71,7 +71,10 @@ describe('claude agent SDK process spawn', () => { const spawn = createClaudeCodeProcessSpawn(process.spawnImpl, platform) spawn.spawn(sdkOptions()) - const spec = process.specs[0] as ProcessSpec + const [spec] = process.specs + if (!spec) { + throw new Error('the spawner never built a spec') + } expect(spawn.supervised).toBe(true) expect(spawn.pid).toBe(4321) expect(spec.program).toBe(globalThis.process.execPath) diff --git a/src/main/claude/claude-supervised-stop-real-cli.test.ts b/src/main/claude/claude-supervised-stop-real-cli.test.ts new file mode 100644 index 000000000000..ee8da56021be --- /dev/null +++ b/src/main/claude/claude-supervised-stop-real-cli.test.ts @@ -0,0 +1,241 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { homedir, tmpdir } from 'node:os' +import { join } from 'node:path' +import { randomUUID } from 'node:crypto' +import { afterEach, describe, expect, it } from 'vitest' +import { runProcess } from '../../shared/child-process/run-process' +import { resolveSessionFilePath } from '../native-chat/session-file-resolver' +import { SUPERVISED_GRACEFUL_EXIT_MS } from './claude-child-exit-proof-ladder' +import { + openClaudeStreamJsonConnection, + type ClaudeStreamJsonConnection +} from './claude-stream-json-connection' +import { + CLAUDE_STRUCTURED_BASE_OPTIONS, + claudeStructuredPermissionOptions +} from './claude-structured-launch-resolution' + +// Opt-in only: spends a real (haiku) turn per case. Run it in an isolated HOME with +// ORCA_REAL_CLAUDE_BIN set, and ORCA_REAL_CLAUDE_SETTINGS when auth lives in a settings file. +const CLAUDE_BIN = process.env.ORCA_REAL_CLAUDE_BIN ?? '' +const enabled = + process.env.ORCA_REAL_CLAUDE_SUPERVISED_STOP === '1' && + process.platform !== 'win32' && + CLAUDE_BIN.length > 0 +const FRAMES_OUT = process.env.ORCA_REAL_CLAUDE_FRAMES_OUT +const TOOL_MARKER = 'orca_supervised_stop_probe' +const TOOL_PROMPT = `Use the Bash tool to run exactly this command, with no timeout argument, and nothing else: python3 -c "import time; time.sleep(120) # ${TOOL_MARKER}"` + +type Frame = { at: number; message: Record } +type Row = { pid: number; ppid: number; command: string } + +const recordedPids = new Set() +const tempDirs: string[] = [] +const connections: ClaudeStreamJsonConnection[] = [] + +function alive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch (error) { + return !(error instanceof Error && 'code' in error && error.code === 'ESRCH') + } +} + +async function processTable(): Promise { + const result = await runProcess({ program: 'ps', args: ['-axo', 'pid=,ppid=,command='] }) + return result.stdout.split('\n').flatMap((line) => { + const match = /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line) + return match ? [{ pid: Number(match[1]), ppid: Number(match[2]), command: match[3] }] : [] + }) +} + +async function descendantsOf(rootPid: number): Promise { + const rows = await processTable() + const found: Row[] = [] + const frontier = [rootPid] + while (frontier.length > 0) { + const parent = frontier.pop() + for (const row of rows.filter((candidate) => candidate.ppid === parent)) { + found.push(row) + frontier.push(row.pid) + } + } + return found +} + +async function until(read: () => Promise | T | null, what: string, ms = 90_000) { + const deadline = Date.now() + ms + for (;;) { + const value = await read() + if (value !== null) { + return value + } + if (Date.now() >= deadline) { + throw new Error(`timed out waiting for ${what}`) + } + await new Promise((resolve) => setTimeout(resolve, 250)) + } +} + +async function open(sessionId: string, cwd: string, resume: boolean, frames: Frame[]) { + const settings = process.env.ORCA_REAL_CLAUDE_SETTINGS + const permission = claudeStructuredPermissionOptions('bypassPermissions') + const connection = await openClaudeStreamJsonConnection( + { + pathToClaudeCodeExecutable: CLAUDE_BIN, + options: { + ...CLAUDE_STRUCTURED_BASE_OPTIONS, + model: 'haiku', + extraArgs: { + ...CLAUDE_STRUCTURED_BASE_OPTIONS.extraArgs, + ...permission.extraArgs, + ...(settings ? { settings } : {}) + }, + ...(resume ? { resume: sessionId } : { sessionId }) + }, + cwd + }, + { onMessage: (message) => frames.push({ at: Date.now(), message }) } + ) + connections.push(connection) + recordedPids.add(connection.pid!) + return connection +} + +function userMessage(text: string): Record { + return { + type: 'user', + message: { role: 'user', content: [{ type: 'text', text }] }, + parent_tool_use_id: null, + session_id: '' + } +} + +async function transcriptLines(sessionId: string): Promise { + const configDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const path = await until( + () => + resolveSessionFilePath('claude', sessionId, { + claudeProjectsDir: join(configDir, 'projects') + }), + 'the transcript', + 15_000 + ) + return readFileSync(path, 'utf8') +} + +function expectLineAtomic(contents: string): void { + expect(contents.length).toBeGreaterThan(0) + expect(contents.endsWith('\n')).toBe(true) + for (const line of contents.split('\n').filter((entry) => entry.trim())) { + expect(() => JSON.parse(line)).not.toThrow() + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null +} + +function summarize(frames: Frame[], since: number): string[] { + return frames + .filter((frame) => frame.at >= since) + .map(({ at, message }) => { + const inner = isRecord(message.message) ? message.message.content : undefined + const blocks = (Array.isArray(inner) ? inner : []) + .filter(isRecord) + .map((block) => + block.type === 'tool_result' + ? `tool_result(is_error=${String(block.is_error)}:${JSON.stringify(block.content).slice(0, 80)})` + : String(block.type) + ) + return `+${at - since}ms ${String(message.type)}/${String(message.subtype ?? '')}${ + message.type === 'result' ? `(is_error=${String(message.is_error)})` : '' + } ${blocks.join(',')}`.trim() + }) +} + +async function resumeAndAsk(sessionId: string, cwd: string): Promise { + const frames: Frame[] = [] + const resumed = await open(sessionId, cwd, true, frames) + await resumed.send(userMessage('In one short line: what command did I ask you to run?')) + const result = await until( + () => frames.find((frame) => frame.message.type === 'result')?.message ?? null, + 'the resumed turn result' + ) + expect(result.subtype).toBe('success') + await expect(resumed.close()).resolves.toBe(true) +} + +type StopCase = 'close mid-tool' | 'SIGTERM to the supervisor mid-tool' | 'close while idle' + +afterEach(async () => { + for (const pid of recordedPids) { + if (alive(pid)) { + process.kill(pid, 'SIGKILL') + } + } + recordedPids.clear() + connections.splice(0) + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } +}) + +describe.runIf(enabled)('real Claude stopped through the POSIX supervisor', () => { + it.each(['close mid-tool', 'SIGTERM to the supervisor mid-tool', 'close while idle'])( + '%s: stops Claude and its tool, keeps the transcript line-atomic, and resumes', + async (stopCase) => { + const cwd = mkdtempSync(join(tmpdir(), 'orca-real-claude-stop-')) + tempDirs.push(cwd) + const sessionId = randomUUID() + const frames: Frame[] = [] + const connection = await open(sessionId, cwd, false, frames) + const supervisor = connection.pid! + const midTool = stopCase !== 'close while idle' + await connection.send(userMessage(midTool ? TOOL_PROMPT : 'Reply with the single word: ok')) + const descendants = midTool + ? await until(async () => { + const rows = await descendantsOf(supervisor) + return rows.some((row) => row.command.includes(TOOL_MARKER)) ? rows : null + }, 'the Bash tool') + : await until( + async () => + frames.some((frame) => frame.message.type === 'result') + ? await descendantsOf(supervisor) + : null, + 'the idle turn' + ) + for (const row of descendants) { + recordedPids.add(row.pid) + } + + const signalledAt = Date.now() + if (stopCase === 'SIGTERM to the supervisor mid-tool') { + process.kill(supervisor, 'SIGTERM') + } else { + await expect(connection.close()).resolves.toBe(true) + } + const gone = await until( + () => ([supervisor, ...descendants.map((row) => row.pid)].some(alive) ? null : true), + 'the supervisor, Claude and its tools to exit', + SUPERVISED_GRACEFUL_EXIT_MS + 2_000 + ) + const stoppedMs = Date.now() - signalledAt + const after = summarize(frames, signalledAt) + console.log(`[${stopCase}] stopped in ${stoppedMs} ms; frames after the stop:`, after) + if (FRAMES_OUT) { + writeFileSync( + `${FRAMES_OUT}.${stopCase.replaceAll(' ', '-')}.json`, + JSON.stringify({ stoppedMs, after }, null, 2) + ) + } + expect(gone).toBe(true) + + expectLineAtomic(await transcriptLines(sessionId)) + await resumeAndAsk(sessionId, cwd) + expectLineAtomic(await transcriptLines(sessionId)) + }, + 180_000 + ) +}) From 7d1c0f9fe6b7e5ff8117cc520ff6f3c632540db8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 14:39:39 -0700 Subject: [PATCH 07/11] test(claude): a requested stop reads interrupted through the frames the supervised SIGTERM makes Claude emit --- .../claude-structured-requested-stop.test.ts | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 src/main/claude/claude-structured-requested-stop.test.ts diff --git a/src/main/claude/claude-structured-requested-stop.test.ts b/src/main/claude/claude-structured-requested-stop.test.ts new file mode 100644 index 000000000000..993109f4a378 --- /dev/null +++ b/src/main/claude/claude-structured-requested-stop.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from 'vitest' +import type { + AgentJournalItemBody, + AgentJournalItemIdentity +} from '../../shared/agent-session-journal-types' +import { readAgentJournalTurn } from '../../shared/agent-session-turn-record' +import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' +import type { ClaudeStructuredSessionEvent } from './claude-structured-session-state' +import { + PROVIDER_SESSION_ID, + USER_MESSAGE, + adapterFor, + fakeClaude, + identityFor +} from './claude-structured-session-test-support' + +function journalSink() { + const items: { identity: AgentJournalItemIdentity; body: AgentJournalItemBody }[] = [] + const sink: StructuredAgentSessionEventSink = { + appendItem: (identity, body) => items.push({ identity, body }), + appendTombstone: () => {}, + publish: () => {} + } + return { sink, items } +} + +function frame(type: 'assistant' | 'user', uuid: string, content: unknown[]) { + return { + type, + uuid, + session_id: PROVIDER_SESSION_ID, + parent_tool_use_id: null, + message: { role: type, content } + } +} + +describe('a requested stop of a structured Claude chat', () => { + it('reads interrupted, not failed or crashed, through the frames the stop makes Claude emit', async () => { + const claude = fakeClaude({ replayUuid: 'turn-1' }) + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = adapterFor(claude, {}, events) + const journal = journalSink() + await adapter.acquire({ + identity: identityFor(), + fence: 7, + spawnToken: 'spawn-9', + events: journal.sink + }) + await adapter.dispatch({ + sessionId: 'session-1', + clientMessageId: 'client-1', + body: USER_MESSAGE, + fence: 7 + }) + const connection = claude.connections[0]! + connection.handlers.onMessage?.( + frame('assistant', 'assistant-1', [ + { type: 'tool_use', id: 'tool-1', name: 'Bash', input: { command: 'sleep 120' } } + ]) + ) + // What Claude 2.1.283 emitted within 35 ms of the supervised SIGTERM mid-tool, recorded + // against the real CLI: the killed tool's result, a status frame, and no `result` frame. + connection.close = async () => { + connection.handlers.onMessage?.( + frame('user', 'user-2', [ + { type: 'tool_result', tool_use_id: 'tool-1', is_error: true, content: 'Exit code 137' } + ]) + ) + connection.handlers.onMessage?.({ + type: 'system', + subtype: 'status', + uuid: 'status-1', + session_id: PROVIDER_SESSION_ID + }) + connection.closed = true + return true + } + + await expect(adapter.closeSession('session-1')).resolves.toBe(true) + + const turns = journal.items.flatMap((item) => { + const turn = readAgentJournalTurn(item.body) + return turn ? [turn] : [] + }) + expect(turns.at(-1)).toMatchObject({ turnId: 'turn-1', state: 'interrupted' }) + const ended = events.filter((event) => event.type === 'ended') + expect(ended).toEqual([expect.objectContaining({ reason: 'claude session closed' })]) + expect(ended[0]).not.toHaveProperty('cause') + }) +}) From 84934e4e2c01fdba6a6a8d916d73e43e085e5c3c Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 14:41:57 -0700 Subject: [PATCH 08/11] test(claude): show what the real CLI did when it never ran the tool --- src/main/claude/claude-supervised-stop-real-cli.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/claude/claude-supervised-stop-real-cli.test.ts b/src/main/claude/claude-supervised-stop-real-cli.test.ts index ee8da56021be..82432e3fd6e6 100644 --- a/src/main/claude/claude-supervised-stop-real-cli.test.ts +++ b/src/main/claude/claude-supervised-stop-real-cli.test.ts @@ -198,7 +198,10 @@ describe.runIf(enabled)('real Claude stopped through the POSIX supervisor', () = ? await until(async () => { const rows = await descendantsOf(supervisor) return rows.some((row) => row.command.includes(TOOL_MARKER)) ? rows : null - }, 'the Bash tool') + }, 'the Bash tool').catch((error: unknown) => { + console.log('[no tool] frames so far:', summarize(frames, 0)) + throw error + }) : await until( async () => frames.some((frame) => frame.message.type === 'result') From 297ea48cd052c4d8f1ae4bc6bcfc2ce487947b43 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 14:43:16 -0700 Subject: [PATCH 09/11] test(claude): Orca's death now reaps Claude's own tool through its SIGTERM --- src/main/claude/claude-supervised-stop.integration.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/claude/claude-supervised-stop.integration.test.ts b/src/main/claude/claude-supervised-stop.integration.test.ts index 1a15b8a778d2..d38d57c8c6e8 100644 --- a/src/main/claude/claude-supervised-stop.integration.test.ts +++ b/src/main/claude/claude-supervised-stop.integration.test.ts @@ -177,7 +177,7 @@ describe.runIf(process.platform !== 'win32')('Claude under the POSIX provider su const startedAt = Date.now() await expect(close()).resolves.toBe(true) - // Claude's own SIGTERM reap ran, well before the supervisor's stdin-end grace would force it. + // Claude's own SIGTERM reap ran at once, not after the supervisor's stdin-end grace. expect(Date.now() - startedAt).toBeLessThan(DEFAULT_PROVIDER_SUPERVISOR_GRACE_MS) expect(existsSync(marker)).toBe(true) await expect(exit).resolves.toEqual({ code: null, signal: 'SIGTERM' }) @@ -199,7 +199,7 @@ describe.runIf(process.platform !== 'win32')('Claude under the POSIX provider su expect(alive(pids.claude)).toBe(false) }) - it("stops Claude when Orca's main dies, and leaves a daemon that left its tree alone", async () => { + it("stops Claude and its tool when Orca's main dies, and leaves a daemon that left its tree alone", async () => { const specs: ProcessSpec[] = [] createClaudeCodeProcessSpawn((spec) => { specs.push(spec) @@ -216,6 +216,8 @@ describe.runIf(process.platform !== 'win32')('Claude under the POSIX provider su expect(await waitFor(() => !alive(pids.claude), PROVIDER_SUPERVISOR_MAX_STOP_MS)).toBe(true) expect(await waitFor(() => !alive(pids.root), PROVIDER_SUPERVISOR_MAX_STOP_MS)).toBe(true) + // In its own group, so only Claude's SIGTERM handler could have reaped it. + expect(alive(pids.tool)).toBe(false) // Not the conversation's writer: nothing proves it orphaned, so the stop never reaches it. expect(alive(pids.daemon)).toBe(true) }) From a6a1605fe7eccd2644bceada836d93006d1f9695 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:45:19 -0700 Subject: [PATCH 10/11] refactor(claude): take supervision from the spawn spec so the close ladder cannot disagree with the spawn createProviderSpawnSpec now reports whether it wrapped the provider in the supervisor, and the Claude spawner reads that instead of repeating the platform check. The close ladder's SIGTERM follows the process actually spawned. --- .../claude-agent-sdk-process-spawn.test.ts | 66 ++++++++++++++++++- .../claude/claude-agent-sdk-process-spawn.ts | 9 ++- .../codex-app-server-posix-supervisor.test.ts | 3 +- .../codex-app-server-posix-supervisor.ts | 21 ++++-- 4 files changed, 87 insertions(+), 12 deletions(-) diff --git a/src/main/claude/claude-agent-sdk-process-spawn.test.ts b/src/main/claude/claude-agent-sdk-process-spawn.test.ts index 90056c18ff08..1b6c192d433a 100644 --- a/src/main/claude/claude-agent-sdk-process-spawn.test.ts +++ b/src/main/claude/claude-agent-sdk-process-spawn.test.ts @@ -4,14 +4,22 @@ import { describe, expect, it, vi } from 'vitest' import type { SpawnOptions as SdkSpawnOptions } from '@anthropic-ai/claude-agent-sdk' import { resolveSpawn, type spawnProcess } from '../../shared/child-process/run-process' import type { ProcessSpec } from '../../shared/child-process/process-spec' +import type * as ProviderSupervisor from '../codex/codex-app-server-posix-supervisor' +import { createProviderSpawnSpec } from '../codex/codex-app-server-posix-supervisor' import { createClaudeCodeProcessSpawn } from './claude-agent-sdk-process-spawn' +import { proveClaudeChildExitWithReaper } from './claude-child-exit-proof-ladder' + +vi.mock('../codex/codex-app-server-posix-supervisor', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, createProviderSpawnSpec: vi.fn(actual.createProviderSpawnSpec) } +}) type FakeChild = EventEmitter & { pid: number stdin: PassThrough stdout: PassThrough stderr: PassThrough - kill: ReturnType + kill: ReturnType boolean>> } function fakeSpawn() { @@ -20,7 +28,7 @@ function fakeSpawn() { child.stdin = new PassThrough() child.stdout = new PassThrough() child.stderr = new PassThrough() - child.kill = vi.fn(() => true) + child.kill = vi.fn((_signal?: NodeJS.Signals | number) => true) const specs: ProcessSpec[] = [] const spawnImpl = ((spec: ProcessSpec) => { specs.push(spec) @@ -95,6 +103,60 @@ describe('claude agent SDK process spawn', () => { } ) + it.each([ + { platform: 'darwin', specSupervised: false }, + { platform: 'win32', specSupervised: true } + ] as const)( + 'stops Claude by the spawn spec\u2019s supervision on $platform, never the platform', + async ({ platform, specSupervised }) => { + const actual = await vi.importActual( + '../codex/codex-app-server-posix-supervisor' + ) + vi.mocked(createProviderSpawnSpec).mockImplementationOnce((...args) => ({ + ...actual.createProviderSpawnSpec(...args), + supervised: specSupervised + })) + const process = fakeSpawn() + const spawn = createClaudeCodeProcessSpawn(process.spawnImpl, platform) + spawn.spawn(sdkOptions()) + expect(spawn.supervised).toBe(specSupervised) + + let exited = false + let settle = (): void => {} + const exitPromise = new Promise((resolve) => { + settle = resolve + }) + // Claude leaves shortly after stdin ends, so the ladder never needs its forced rung. + process.child.stdin.on('finish', () => + setTimeout(() => { + exited = true + settle() + }, 10) + ) + const tree = { + capture: vi.fn(async () => {}), + reap: vi.fn(async () => 'exited' as const), + treeVerdict: 'exited' as const + } + await proveClaudeChildExitWithReaper( + { + child: process.child, + exitPromise, + exited: () => exited, + tree, + supervised: spawn.supervised + }, + () => tree + ) + // SIGTERM to an unsupervised Claude on Windows is TerminateProcess; a skipped one leaves it running. + if (specSupervised) { + expect(process.child.kill).toHaveBeenCalledWith('SIGTERM') + } else { + expect(process.child.kill).not.toHaveBeenCalled() + } + } + ) + it('keeps the child out of the SDK abort path so exit proof stays Orca-owned', () => { const process = fakeSpawn() const controller = new AbortController() diff --git a/src/main/claude/claude-agent-sdk-process-spawn.ts b/src/main/claude/claude-agent-sdk-process-spawn.ts index c0691bfffbcf..bbb4b295aade 100644 --- a/src/main/claude/claude-agent-sdk-process-spawn.ts +++ b/src/main/claude/claude-agent-sdk-process-spawn.ts @@ -17,7 +17,7 @@ export type ClaudeCodeProcessSpawn = { * token. On POSIX it is the provider supervisor's, which outlives Claude by construction. */ readonly pid: number | undefined - /** Whether the child is the POSIX supervisor, whose SIGTERM stops Claude and then itself. */ + /** The spawn spec's verdict, so the close ladder never re-decides it. False until the SDK spawns. */ readonly supervised: boolean readonly stderrTail: string } @@ -48,7 +48,7 @@ export function createClaudeCodeProcessSpawn( ): ClaudeCodeProcessSpawn { let child: ClaudeCodeChild | null = null let stderrTail = '' - const supervised = platform !== 'win32' + let supervised = false return { spawn: (options) => { const spec = createProviderSpawnSpec( @@ -71,6 +71,7 @@ export function createClaudeCodeProcessSpawn( stdio: ['pipe', 'pipe', 'pipe'] }) child = spawned + supervised = spec.supervised // The SDK drains stderr only for its own local spawn, so a custom spawner must: // otherwise the child blocks on a full pipe and exit errors lose their tail. spawned.stderr.setEncoding('utf8').on('data', (chunk: string) => { @@ -84,7 +85,9 @@ export function createClaudeCodeProcessSpawn( get pid() { return child?.pid }, - supervised, + get supervised() { + return supervised + }, get stderrTail() { return stderrTail } diff --git a/src/main/codex/codex-app-server-posix-supervisor.test.ts b/src/main/codex/codex-app-server-posix-supervisor.test.ts index f0c834178905..bdb68ff8db52 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.test.ts @@ -68,7 +68,8 @@ describe('structured provider supervision', () => { args: ['app-server', '--flag'], env: { PATH: '/bin' }, cwd: '/work/repo', - detached: false + detached: false, + supervised: false }) }) }) diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index 8efc59cc495c..a64e7ec4df1a 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -166,13 +166,22 @@ export function createProviderSpawnSpec( launch: CodexAppServerLaunch, childEnv: NodeJS.ProcessEnv, platform: NodeJS.Platform -): { program: string; args: string[]; env: NodeJS.ProcessEnv; cwd: string; detached: boolean } { - const supervised = platform === 'win32' ? null : supervisedPosixLaunch(launch, childEnv) +): { + program: string + args: string[] + env: NodeJS.ProcessEnv + cwd: string + detached: boolean + /** The child is the supervisor, whose SIGTERM stops the provider and then itself. */ + supervised: boolean +} { + const supervisor = platform === 'win32' ? null : supervisedPosixLaunch(launch, childEnv) return { - program: supervised?.command ?? launch.command, - args: supervised?.args ?? launch.args, - env: supervised?.env ?? childEnv, + program: supervisor?.command ?? launch.command, + args: supervisor?.args ?? launch.args, + env: supervisor?.env ?? childEnv, cwd: launch.cwd ?? process.cwd(), - detached: platform !== 'win32' + detached: platform !== 'win32', + supervised: supervisor !== null } } From 07d6876d5d86bef5824922c3e251fa7f9de021e7 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:39:19 -0700 Subject: [PATCH 11/11] fix(native-chat): derive quit's chat-eviction bound from the longest supervised provider close Quit's child-eviction phase was a hand-picked 8 s. It is now the sink drain plus the longest supervised close over Claude and Codex plus a named 1 s margin, so a provider close that grows widens it instead of silently outrunning it. A close's tree-kill fallback stays outside the bound: once main exits, the supervisor stops its provider group on owner death, which a new test now proves for a clean owner exit, and next launch's recovery settles the lease. --- .../claude/claude-child-exit-proof-ladder.ts | 2 +- src/main/codex/codex-app-server-connection.ts | 2 +- ...erver-posix-supervisor.integration.test.ts | 32 +++++++-- ...ctured-agent-session-host-teardown.test.ts | 68 ++++++++++++++++--- .../structured-agent-session-host-teardown.ts | 21 ++++-- 5 files changed, 102 insertions(+), 23 deletions(-) diff --git a/src/main/claude/claude-child-exit-proof-ladder.ts b/src/main/claude/claude-child-exit-proof-ladder.ts index 881a6ffd81ff..37d4e47508d7 100644 --- a/src/main/claude/claude-child-exit-proof-ladder.ts +++ b/src/main/claude/claude-child-exit-proof-ladder.ts @@ -3,7 +3,7 @@ import { waitForProcessExitUntil } from '../codex/codex-process-exit-deadline' import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../codex/codex-app-server-posix-supervisor' import type { ClaudeChildTreeReaper } from './claude-agent-sdk-exit-proof' -const GRACEFUL_EXIT_MS = 1_500 +export const GRACEFUL_EXIT_MS = 1_500 // A signalled supervisor escalates on its own; forcing it sooner kills it and orphans Claude. export const SUPERVISED_GRACEFUL_EXIT_MS = PROVIDER_SUPERVISOR_MAX_STOP_MS + 500 const FORCED_EXIT_MS = 1_000 diff --git a/src/main/codex/codex-app-server-connection.ts b/src/main/codex/codex-app-server-connection.ts index ef566bca3360..5bed3b594d7d 100644 --- a/src/main/codex/codex-app-server-connection.ts +++ b/src/main/codex/codex-app-server-connection.ts @@ -46,7 +46,7 @@ export type CodexAppServerLaunch = { } const DEFAULT_REQUEST_TIMEOUT_MS = 30_000 -const GRACEFUL_EXIT_MS = 1_500 +export const GRACEFUL_EXIT_MS = 1_500 const FORCED_EXIT_MS = 1_000 const STDERR_TAIL_MAX_BYTES = 8192 diff --git a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts index e23cdbc88c34..846afaa973f4 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts @@ -52,8 +52,11 @@ const RECORDS_SIGTERM_PROVIDER = String.raw` // Stands in for Orca: launches the supervisor as its own child, then can be killed outright. A // second child holds the supervisor's stdin open, so only the parent-death watch can notice. +// A clean-quit owner has no holder and exits normally on SIGUSR2, the way Orca quits. const OWNER = String.raw` const { spawn } = require('node:child_process') + const quitsCleanly = Boolean(process.env.ORCA_TEST_OWNER_QUITS_CLEANLY) + if (quitsCleanly) process.on('SIGUSR2', () => process.exit(0)) const spec = JSON.parse(Buffer.from(process.env.ORCA_PROVIDER_SUPERVISOR_SPEC, 'base64').toString()) spec.ownerPid = process.pid const supervisor = spawn(process.execPath, ['-e', process.env.ORCA_TEST_SUPERVISOR_SCRIPT], { @@ -61,10 +64,12 @@ const OWNER = String.raw` stdio: ['pipe', 'pipe', 'ignore'], detached: true }) - const holder = spawn(process.execPath, ['-e', 'setInterval(() => {}, 60000)'], { - stdio: ['ignore', supervisor.stdin, 'ignore'] - }) - process.stdout.write(JSON.stringify({ supervisor: supervisor.pid, holder: holder.pid }) + '\n') + const holder = quitsCleanly + ? null + : spawn(process.execPath, ['-e', 'setInterval(() => {}, 60000)'], { + stdio: ['ignore', supervisor.stdin, 'ignore'] + }) + process.stdout.write(JSON.stringify({ supervisor: supervisor.pid, ...(holder && { holder: holder.pid }) }) + '\n') supervisor.stdout.pipe(process.stdout) setInterval(() => {}, 60000) ` @@ -175,7 +180,7 @@ async function launchUnderOwner( stdio: ['ignore', 'pipe', 'ignore'] }) recordedPids.add(owner.pid!) - const pids = await readPids(owner, ['supervisor', 'holder', 'provider', 'grandchild']) + const pids = await readPids(owner, ['supervisor', 'provider', 'grandchild']) return { owner, pids } } @@ -318,6 +323,23 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe expect(alive(provider)).toBe(false) }) + it('reaps the provider group and exits when its owner quits cleanly', async () => { + const { owner, pids } = await launchUnderOwner( + { sigtermGraceMs: 300 }, + { ORCA_TEST_OWNER_QUITS_CLEANLY: '1' } + ) + const ownerExit = new Promise((resolve) => + owner.once('exit', (code, signal) => resolve({ code, signal })) + ) + + owner.kill('SIGUSR2') + + await expect(ownerExit).resolves.toEqual({ code: 0, signal: null }) + expect(await waitFor(() => !alive(-pids.provider), 3_000)).toBe(true) + expect(await waitFor(() => !alive(pids.supervisor), 3_000)).toBe(true) + expect(alive(pids.grandchild)).toBe(false) + }) + it('asks the provider to stop with SIGTERM when its owner dies', async () => { const signalFile = join(tempDir(), 'provider-signal') const { owner, pids } = await launchUnderOwner( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.test.ts index 760b2603d99c..1baaafb6f6b7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.test.ts @@ -1,17 +1,23 @@ import { describe, expect, it, vi } from 'vitest' +import { WILL_QUIT_TEARDOWN_DEADLINE_MS } from '../../../shared/quit-teardown-deadline' +import { + GRACEFUL_EXIT_MS as CLAUDE_WINDOWS_GRACEFUL_EXIT_MS, + SUPERVISED_GRACEFUL_EXIT_MS +} from '../../claude/claude-child-exit-proof-ladder' +import { GRACEFUL_EXIT_MS as CODEX_WINDOWS_GRACEFUL_EXIT_MS } from '../../codex/codex-app-server-connection' import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../../codex/codex-app-server-posix-supervisor' import { SNAPSHOT_DRAIN_TIMEOUT_MS } from './structured-agent-session-eviction' import { CHILD_EVICTION_TIMEOUT_MS, + EVICTION_MARGIN_MS, + RESUME_MARKER_RECORD_TIMEOUT_MS, structuredAgentSessionHostTeardownPhases } from './structured-agent-session-host-teardown' -// Codex close observes the supervisor's exit after its timers fire, which run late on a loaded host. -const SUPERVISOR_EXIT_OBSERVATION_HEADROOM_MS = 1_000 +const noop = async (): Promise => undefined describe('structured agent-session host teardown', () => { it('names every phase, so the quit-path order is pinned rather than incidental', () => { - const noop = async (): Promise => undefined const phases = structuredAgentSessionHostTeardownPhases({ idleSweep: { dispose: noop }, runtimeState: { stopLeaseRenewal: () => undefined, flushAllEventSinks: noop }, @@ -31,14 +37,54 @@ describe('structured agent-session host teardown', () => { ]) }) - it("fits the Codex supervisor's longest stop inside quit's child-eviction bound", () => { - // A quit that times out first leaves the provider running and its lease unreleased. Eviction - // drains the sink for the resume offer before it stops the child, inside the same bound. - expect( - SNAPSHOT_DRAIN_TIMEOUT_MS + - PROVIDER_SUPERVISOR_MAX_STOP_MS + - SUPERVISOR_EXIT_OBSERVATION_HEADROOM_MS - ).toBeLessThan(CHILD_EVICTION_TIMEOUT_MS) + it("derives quit's child-eviction bound from every provider's supervised close", () => { + // Eviction drains the sink for the resume offer before it stops the child, inside one bound. + // Each close's tree-kill fallback is deliberately outside it: once main exits the supervisor + // stops its group itself, and next launch's recovery settles the lease. + const closes = { + claude: SUPERVISED_GRACEFUL_EXIT_MS, + codex: PROVIDER_SUPERVISOR_MAX_STOP_MS, + // No supervisor on Windows, so its closes wait less than any supervised one. + claudeWindows: CLAUDE_WINDOWS_GRACEFUL_EXIT_MS, + codexWindows: CODEX_WINDOWS_GRACEFUL_EXIT_MS + } + expect(CHILD_EVICTION_TIMEOUT_MS).toBe( + SNAPSHOT_DRAIN_TIMEOUT_MS + Math.max(closes.claude, closes.codex) + EVICTION_MARGIN_MS + ) + for (const closeMs of Object.values(closes)) { + expect(SNAPSHOT_DRAIN_TIMEOUT_MS + closeMs + EVICTION_MARGIN_MS).toBeLessThanOrEqual( + CHILD_EVICTION_TIMEOUT_MS + ) + } + // The resume markers recorded after eviction still fit under quit's global deadline. + expect(CHILD_EVICTION_TIMEOUT_MS + RESUME_MARKER_RECORD_TIMEOUT_MS).toBeLessThan( + WILL_QUIT_TEARDOWN_DEADLINE_MS + ) + }) + + it('ends child eviction as soon as every chat has closed, not at its bound', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + const evict = structuredAgentSessionHostTeardownPhases({ + idleSweep: { dispose: noop }, + runtimeState: { stopLeaseRenewal: () => undefined, flushAllEventSinks: noop }, + tasks: { drainAttaches: noop }, + evictOwnedSessions: noop, + beginResumeMarkers: () => {}, + recordResumeMarkers: noop + }).find((phase) => phase.name === 'evict-owned-sessions') + try { + let finished = false + const run = Promise.resolve(evict?.run()).then(() => { + finished = true + }) + // No timer advances: the phase settles with the eviction, not at CHILD_EVICTION_TIMEOUT_MS. + await vi.advanceTimersByTimeAsync(0) + expect(finished).toBe(true) + await run + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } }) it('bounds stalled recovery publication without preventing later cleanup', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts index eade95a7dd2c..bc9a48c5dacb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts @@ -7,6 +7,9 @@ // nothing can ever close them. import type { AgentSessionResumeTrigger } from '../../../shared/agent-session-resume-marker' +import { SUPERVISED_GRACEFUL_EXIT_MS } from '../../claude/claude-child-exit-proof-ladder' +import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../../codex/codex-app-server-posix-supervisor' +import { SNAPSHOT_DRAIN_TIMEOUT_MS } from './structured-agent-session-eviction' import type { StructuredAgentSessionRestartResume } from './structured-agent-session-restart-resume-host' import { abandonQueuedStructuredAgentSessionMessages, @@ -23,12 +26,20 @@ export type StructuredAgentSessionTeardownPhase = { } /** Advisory persistence must not hold shutdown open. */ -const RESUME_MARKER_RECORD_TIMEOUT_MS = 2_000 +export const RESUME_MARKER_RECORD_TIMEOUT_MS = 2_000 -/** Eight steps at ten seconds each would outlast the global quit deadline, and a quit that dies - * mid-eviction leaves the lease unreleased — the exact state restart has to clean up. Bounded - * well below that deadline so the phases after this one still get to run. */ -export const CHILD_EVICTION_TIMEOUT_MS = 8_000 +/** Covers a provider's stop observed late on a loaded host. */ +export const EVICTION_MARGIN_MS = 1_000 + +/** A quit that dies mid-eviction leaves the lease unreleased — the exact state restart has to + * clean up — so this covers the sink drain plus the longest supervised provider close, well below + * the global quit deadline so later phases still run. A close's tree-kill fallback is outside it: + * once main exits the supervisor stops its group itself, and next launch's recovery settles the + * lease. Windows closes have no supervisor and wait less. */ +export const CHILD_EVICTION_TIMEOUT_MS = + SNAPSHOT_DRAIN_TIMEOUT_MS + + Math.max(SUPERVISED_GRACEFUL_EXIT_MS, PROVIDER_SUPERVISOR_MAX_STOP_MS) + + EVICTION_MARGIN_MS /** Bounds a phase without swallowing its failure, which `withTimeout` alone would. */ async function withPhaseTimeout(run: () => Promise, timeoutMs: number): Promise {