From ad2b5ac0352fc1de3a93e1946a3f16e719eaf238 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:39:52 -0700 Subject: [PATCH 01/38] fix(supervisor): add a one-shot lifetime that runs past stdin end, and relay a provider's last output A one-shot CLI (claude -p, codex exec -) reads its request until stdin ends, so the supervisor's session rule (stdin end means the owner is closing) would stop it about 1 s into its answer. The one-shot lifetime passes stdin end through and leaves only the owner-death watch and explicit signals to stop it. The supervisor also exited as soon as its provider did, dropping output still in the pipes when the owner reads slowly. It now waits, bounded, for the provider's output to be relayed before exiting. --- ...erver-posix-supervisor.integration.test.ts | 109 +++++++++++++++++- .../codex-app-server-posix-supervisor.test.ts | 20 ++++ .../codex-app-server-posix-supervisor.ts | 41 ++++++- 3 files changed, 164 insertions(+), 6 deletions(-) diff --git a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts index 846afaa973f4..2efcdbc5ef99 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts @@ -50,12 +50,27 @@ const RECORDS_SIGTERM_PROVIDER = String.raw` setInterval(() => {}, 60000) ` +// Answers a one-shot request only after the session stdin-end grace, then exits with its own code. +const ONE_SHOT_PROVIDER = String.raw` + process.on('SIGTERM', () => { + require('node:fs').writeFileSync(process.env.ORCA_TEST_PROVIDER_SIGNAL_FILE, 'SIGTERM') + }) + process.stdin.resume().on('end', () => { + setTimeout(() => { + process.stdout.write('ok') + process.exitCode = 3 + }, Number(process.env.ORCA_TEST_PROVIDER_ANSWER_DELAY_MS)) + }) +` + // Stands in for Orca: launches the supervisor as its own child, then can be killed outright. A // second child holds the supervisor's stdin open, so only the parent-death watch can notice. -// A clean-quit owner has no holder and exits normally on SIGUSR2, the way Orca quits. +// A clean-quit owner has no holder and exits normally on SIGUSR2, the way Orca quits. A one-shot +// owner ends the supervisor's stdin at once, as Orca does after writing a one-shot's request. const OWNER = String.raw` const { spawn } = require('node:child_process') const quitsCleanly = Boolean(process.env.ORCA_TEST_OWNER_QUITS_CLEANLY) + const endsStdin = Boolean(process.env.ORCA_TEST_OWNER_ENDS_STDIN) if (quitsCleanly) process.on('SIGUSR2', () => process.exit(0)) const spec = JSON.parse(Buffer.from(process.env.ORCA_PROVIDER_SUPERVISOR_SPEC, 'base64').toString()) spec.ownerPid = process.pid @@ -64,7 +79,8 @@ const OWNER = String.raw` stdio: ['pipe', 'pipe', 'ignore'], detached: true }) - const holder = quitsCleanly + if (endsStdin) supervisor.stdin.end() + const holder = quitsCleanly || endsStdin ? null : spawn(process.execPath, ['-e', 'setInterval(() => {}, 60000)'], { stdio: ['ignore', supervisor.stdin, 'ignore'] @@ -352,4 +368,93 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe expect(await waitFor(() => !alive(-pids.provider), 3_000)).toBe(true) expect(existsSync(signalFile) && readFileSync(signalFile, 'utf8')).toBe('SIGTERM') }) + + describe('one-shot lifetime', () => { + it('lets a one-shot answer after its stdin ends and relays its exit code', async () => { + const signalFile = join(tempDir(), 'provider-signal') + const { supervisor, exit } = launchSupervisor( + { lifetime: 'one-shot' }, + { + ORCA_TEST_PROVIDER_SIGNAL_FILE: signalFile, + ORCA_TEST_PROVIDER_ANSWER_DELAY_MS: String(PROVIDER_STDIN_END_GRACE_MS * 1.5) + }, + { command: process.execPath, args: ['-e', ONE_SHOT_PROVIDER] } + ) + let stdout = '' + supervisor.stdout!.on('data', (chunk: Buffer) => (stdout += chunk.toString())) + + supervisor.stdin!.end('request') + + await expect(exit).resolves.toEqual({ code: 3, signal: null }) + expect(stdout).toBe('ok') + expect(existsSync(signalFile)).toBe(false) + }) + + it.each([ + ['dies', 'SIGKILL', {}], + ['quits cleanly', 'SIGUSR2', { ORCA_TEST_OWNER_QUITS_CLEANLY: '1' }] + ] as const)( + 'reaps a one-shot that ignores its stdin end when its owner %s', + async (_, signal, env) => { + const { owner, pids } = await launchUnderOwner( + { lifetime: 'one-shot', sigtermGraceMs: 300 }, + { ORCA_TEST_OWNER_ENDS_STDIN: '1', ...env } + ) + // Past the session grace: a stdin end alone must not have stopped it. + await new Promise((resolve) => setTimeout(resolve, PROVIDER_STDIN_END_GRACE_MS + 300)) + expect(alive(-pids.provider)).toBe(true) + + owner.kill(signal) + + expect(await waitFor(() => !alive(-pids.provider), PROVIDER_SUPERVISOR_MAX_STOP_MS)).toBe( + true + ) + expect(await waitFor(() => !alive(pids.supervisor), 3_000)).toBe(true) + expect(alive(pids.grandchild)).toBe(false) + } + ) + + it('escalates a SIGTERM-ignoring one-shot to SIGKILL after the grace from the spec', async () => { + const graceMs = 200 + const { supervisor, exit } = launchSupervisor( + { lifetime: 'one-shot', sigtermGraceMs: graceMs }, + { ORCA_TEST_PROVIDER_IGNORES_SIGTERM: '1' } + ) + const { provider, grandchild } = await readPids(supervisor, ['provider', 'grandchild']) + supervisor.stdin!.end() + + const signalledAt = Date.now() + supervisor.kill('SIGTERM') + + await expect(exit).resolves.toEqual({ code: null, signal: 'SIGTERM' }) + expect(Date.now() - signalledAt).toBeGreaterThanOrEqual(graceMs) + expect(Date.now() - signalledAt).toBeLessThan(PROVIDER_SIGTERM_GRACE_MS) + expect(alive(provider)).toBe(false) + expect(alive(grandchild)).toBe(false) + }) + + it('relays all of the provider output to a slow owner before exiting', async () => { + const bytes = 4 * 1024 * 1024 + const { supervisor, exit } = launchSupervisor( + { lifetime: 'one-shot' }, + {}, + { + command: process.execPath, + args: ['-e', `process.stdout.write('x'.repeat(${bytes})); process.exitCode = 3`] + } + ) + let received = 0 + supervisor.stdout!.on('data', (chunk: Buffer) => { + received += chunk.byteLength + supervisor.stdout!.pause() + setTimeout(() => supervisor.stdout!.resume(), 5) + }) + const closed = new Promise((resolve) => supervisor.once('close', resolve)) + supervisor.stdin!.end() + + await expect(exit).resolves.toEqual({ code: 3, signal: null }) + await closed + expect(received).toBe(bytes) + }) + }) }) diff --git a/src/main/codex/codex-app-server-posix-supervisor.test.ts b/src/main/codex/codex-app-server-posix-supervisor.test.ts index 99cbaf100088..1d01e6120ada 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import type { CodexAppServerLaunch } from './codex-app-server-connection' import { createProviderSpawnSpec, + isSupervisedProviderNotFound, POSIX_PROVIDER_SUPERVISOR_SCRIPT, PROVIDER_SIGTERM_GRACE_MS, PROVIDER_STDIN_END_GRACE_MS, @@ -31,6 +32,7 @@ describe('structured provider supervision', () => { args: ['app-server', '--flag'], cwd: '/work/repo', ownerPid: process.pid, + lifetime: 'session', stdinEndGraceMs: PROVIDER_STDIN_END_GRACE_MS, sigtermGraceMs: PROVIDER_SIGTERM_GRACE_MS }) @@ -49,6 +51,24 @@ describe('structured provider supervision', () => { expect(POSIX_PROVIDER_SUPERVISOR_SCRIPT).not.toContain('process.ppid === 1') }) + it('carries a one-shot lifetime through the spawn spec', () => { + const spec = createProviderSpawnSpec(launch, { PATH: '/bin' }, 'darwin', { + lifetime: 'one-shot' + }) + + expect(spec).toMatchObject({ program: process.execPath, detached: true, supervised: true }) + expect( + JSON.parse(Buffer.from(spec.env.ORCA_PROVIDER_SUPERVISOR_SPEC!, 'base64').toString()) + ).toMatchObject({ command: '/opt/codex', args: ['app-server', '--flag'], lifetime: 'one-shot' }) + }) + + it('reads only the supervisor own missing-binary exit as not found', () => { + expect(isSupervisedProviderNotFound(127, 'spawn /opt/my tools/claude ENOENT\n')).toBe(true) + expect(isSupervisedProviderNotFound(1, 'spawn claude ENOENT\n')).toBe(false) + expect(isSupervisedProviderNotFound(127, 'sh: claude: command not found\n')).toBe(false) + expect(isSupervisedProviderNotFound(127, 'warning\nspawn claude ENOENT\n')).toBe(false) + }) + it('refuses a grace longer than recovery waits before SIGKILL', () => { const stdinEnd = (stdinEndGraceMs: number) => () => supervisedPosixLaunch(launch, {}, { stdinEndGraceMs }) diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index 21dd56aeb5b5..aea2100919e3 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -10,6 +10,8 @@ export const PROVIDER_SIGTERM_GRACE_MS = 3_000 * recovery on a process stuck in the kernel, such as one blocked on a hung network drive. */ export const PROVIDER_GROUP_REAP_TIMEOUT_MS = 1_500 +/** Longest a supervisor waits, after its provider exits, to relay output still in the pipes. */ +export const PROVIDER_OUTPUT_DRAIN_TIMEOUT_MS = 1_000 /** * Longest a supervisor can take to stop once asked (stdin end, grace, SIGTERM, grace, SIGKILL, * reap); a SIGKILL sooner can orphan its group. The graces are also the largest a spec may carry. @@ -92,8 +94,11 @@ const scheduleOwnerShutdown = () => { ownerShutdownTimer = setTimeout(() => stopProviderGroup(null), spec.stdinEndGraceMs) ownerShutdownTimer.unref() } -process.stdin.once('end', scheduleOwnerShutdown) -process.stdin.once('close', scheduleOwnerShutdown) +// A one-shot's stdin end is the end of its request, not a stop; only its owner's death stops it. +if (spec.lifetime !== 'one-shot') { + process.stdin.once('end', scheduleOwnerShutdown) + process.stdin.once('close', scheduleOwnerShutdown) +} process.stdin.pipe(child.stdin) child.stdout.pipe(process.stdout) child.stderr.pipe(process.stderr) @@ -101,12 +106,24 @@ child.stderr.pipe(process.stderr) for (const stream of [process.stdin, process.stdout, process.stderr, child.stdin, child.stdout, child.stderr]) { stream.on('error', () => {}) } +// Exit can land before the provider's last output is relayed; a one-shot's answer is that output. +const drainProviderOutput = () => { + const ended = (stream) => stream.readableEnded || stream.destroyed ? null : new Promise((resolve) => { + stream.once('end', resolve) + stream.once('close', resolve) + }) + const flushed = (stream) => new Promise((resolve) => stream.write('', resolve)) + const drained = Promise.all([child.stdout, child.stderr].map(ended)) + .then(() => Promise.all([process.stdout, process.stderr].map(flushed))) + return Promise.race([drained, new Promise((resolve) => setTimeout(resolve, ${PROVIDER_OUTPUT_DRAIN_TIMEOUT_MS}))]) +} const reapProviderExit = async (code, signal) => { if (settling) return settling = true clearInterval(timer) if (ownerShutdownTimer) clearTimeout(ownerShutdownTimer) if (!(await reapOwnedProviderGroup())) return process.exit(1) + await drainProviderOutput() finishWithProviderOutcome(code, signal) } timer = setInterval(() => { @@ -123,8 +140,20 @@ child.once('exit', (code, signal) => { }) ` +/** + * `session`: the owner ending stdin is a close, so the provider is stopped after a grace. + * `one-shot`: stdin end only completes the request; the provider runs until it exits or is stopped. + */ +export type ProviderSupervisorLifetime = 'session' | 'one-shot' + +/** The supervisor could not start its provider: it relays Node's spawn ENOENT and exits 127. */ +export function isSupervisedProviderNotFound(code: number | null, stderr: string): boolean { + return code === 127 && /^spawn .+ ENOENT$/.test(stderr.trim()) +} + export type ProviderSupervisorOptions = { cwd?: string + lifetime?: ProviderSupervisorLifetime /** The process the supervisor serves; it must be the supervisor's parent. */ ownerPid?: number stdinEndGraceMs?: number @@ -144,6 +173,7 @@ export function supervisedPosixLaunch( { cwd = launch.cwd ?? process.cwd(), ownerPid = process.pid, + lifetime = 'session', stdinEndGraceMs = PROVIDER_STDIN_END_GRACE_MS, sigtermGraceMs = PROVIDER_SIGTERM_GRACE_MS }: ProviderSupervisorOptions = {} @@ -156,6 +186,7 @@ export function supervisedPosixLaunch( args: launch.args, cwd, ownerPid, + lifetime, stdinEndGraceMs, sigtermGraceMs }) @@ -176,7 +207,8 @@ export function supervisedPosixLaunch( export function createProviderSpawnSpec( launch: CodexAppServerLaunch, childEnv: NodeJS.ProcessEnv, - platform: NodeJS.Platform + platform: NodeJS.Platform, + { lifetime }: Pick = {} ): { program: string args: string[] @@ -186,7 +218,8 @@ export function createProviderSpawnSpec( /** The child is the supervisor, whose SIGTERM stops the provider and then itself. */ supervised: boolean } { - const supervisor = platform === 'win32' ? null : supervisedPosixLaunch(launch, childEnv) + const supervisor = + platform === 'win32' ? null : supervisedPosixLaunch(launch, childEnv, { lifetime }) return { program: supervisor?.command ?? launch.command, args: supervisor?.args ?? launch.args, From 7b44d9fa077925eb7ecfc8c905cffa293aae58aa Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:51:06 -0700 Subject: [PATCH 02/38] fix(text-generation): run agent one-shots under the provider supervisor on POSIX The Claude model-list probe, model discovery for every agent, and commit message, pull request and branch name generation all spawned the agent CLI as a plain child of Orca. If Orca quit, crashed or was killed while one was running, nothing stopped it, and a CLI that hung kept running after Orca was gone. They now run under the provider supervisor that native chat already uses, in its one-shot lifetime, so Orca's exit stops the agent's whole process group however Orca exits. A timeout or cancel asks the supervisor to stop (SIGTERM) and only tears the tree down once it has had its full stop time; killing the supervisor first would orphan the agent's group. A missing binary now reaches Orca as the supervisor's exit 127, which is mapped back to the existing not-found message. Windows and WSL keep spawning the agent directly. --- .../claude/claude-model-catalog-probe.test.ts | 45 +++++- .../commit-message-model-discovery.ts | 8 +- ...ssage-text-generation-cancellation.test.ts | 6 + ...e-text-generation-local-subprocess.test.ts | 6 + ...ge-text-generation-model-discovery.test.ts | 33 +++++ ...it-message-text-generation-test-harness.ts | 16 +++ .../source-control-agent-launch.test.ts | 129 ++++++++++++++++++ .../source-control-agent-launch.ts | 55 ++++++-- .../source-control-local-process.test.ts | 119 ++++++++++++++++ .../source-control-local-process.ts | 38 +++++- .../source-control-text-generation-types.ts | 13 +- 11 files changed, 449 insertions(+), 19 deletions(-) create mode 100644 src/main/text-generation/source-control-agent-launch.test.ts create mode 100644 src/main/text-generation/source-control-local-process.test.ts diff --git a/src/main/claude/claude-model-catalog-probe.test.ts b/src/main/claude/claude-model-catalog-probe.test.ts index 6e1bd13e1b58..db204a2bd946 100644 --- a/src/main/claude/claude-model-catalog-probe.test.ts +++ b/src/main/claude/claude-model-catalog-probe.test.ts @@ -1,6 +1,8 @@ -import { homedir } from 'node:os' +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { homedir, tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' +import { PROVIDER_STDIN_END_GRACE_MS } from '../codex/codex-app-server-posix-supervisor' import { createClaudeModelCatalogProbe } from './claude-model-catalog-probe' import { resolveClaudeStructuredInvocation } from './claude-structured-launch-resolution' import type { discoverModelsLocal } from '../text-generation/commit-message-model-discovery' @@ -124,4 +126,45 @@ describe('claude model catalog probe', () => { }) await expect(probe('/homes/a')).rejects.toThrow(/listed no models/) }) + + it.runIf(process.platform !== 'win32')( + 'lists through a supervised one-shot that answers after its input ends', + async () => { + const folder = mkdtempSync(join(tmpdir(), 'orca-claude-probe-')) + try { + const parentFile = join(folder, 'parent-pid') + const standIn = join(folder, 'claude') + // Answers only after the session stdin-end grace, the way a slow `claude -p` does. + writeFileSync( + standIn, + `#!${process.execPath} +let input = '' +process.stdin.on('data', (chunk) => (input += chunk)) +process.stdin.on('end', () => setTimeout(() => { + require('node:fs').writeFileSync(${JSON.stringify(parentFile)}, String(process.ppid)) + const request = JSON.parse(input.trim().split('\\n').at(-1)) + if (request.request.subtype !== 'list_models') process.exit(2) + console.log(JSON.stringify({ type: 'control_response', response: { subtype: 'success', + request_id: request.request_id, response: { models: [{ value: 'sonnet', displayName: 'Sonnet' }] } } })) +}, ${PROVIDER_STDIN_END_GRACE_MS * 1.5})) +` + ) + chmodSync(standIn, 0o755) + const probe = createClaudeModelCatalogProbe({ + ...probeDeps(), + resolveCommand: () => standIn, + resolveInheritedEnv: async () => ({ PATH: process.env.PATH ?? '', HOME: folder }) + }) + + await expect(probe(join(folder, 'account'))).resolves.toMatchObject({ + origin: 'probe', + models: [{ id: 'sonnet', label: 'Sonnet' }] + }) + // The stand-in's parent is the supervisor, never Orca itself. + expect(Number(readFileSync(parentFile, 'utf8'))).not.toBe(process.pid) + } finally { + rmSync(folder, { recursive: true, force: true }) + } + } + ) }) diff --git a/src/main/text-generation/commit-message-model-discovery.ts b/src/main/text-generation/commit-message-model-discovery.ts index 80dbbb205a39..c4005992df99 100644 --- a/src/main/text-generation/commit-message-model-discovery.ts +++ b/src/main/text-generation/commit-message-model-discovery.ts @@ -4,6 +4,7 @@ import type { CommitMessagePlan } from '../../shared/commit-message-plan' import { getAgentModelProbeSpec } from '../../shared/agent-model-probe-spec' import type { TuiAgent } from '../../shared/tui-agent' import { resolveCodexHomeProcessLockKeyForSpawnEnv } from '../codex-cli/codex-home-process-lock' +import { isSupervisedProviderNotFound } from '../codex/codex-app-server-posix-supervisor' import { isSshRequestOutcomeUnverifiable } from '../ssh/ssh-channel-multiplexer' import { WINDOWS_BATCH_UNSAFE_ARGUMENTS_ERROR } from '../win32-utils' import { @@ -139,6 +140,7 @@ export async function discoverModelsLocal(input: { } const onStdoutData = (chunk: Buffer): void => onData(chunk, (text) => (stdout += text)) const onStderrData = (chunk: Buffer): void => onData(chunk, (text) => (stderr += text)) + const notFoundError = `${spec.modelDiscovery?.binary ?? spec.binary} not found on PATH. Install ${spec.label} to discover models.` const onError = (error: Error): void => { if (!child.pid) { markProcessClosed() @@ -147,7 +149,7 @@ export async function discoverModelsLocal(input: { success: false, error: (error as NodeJS.ErrnoException).code === 'ENOENT' - ? `${spec.modelDiscovery?.binary ?? spec.binary} not found on PATH. Install ${spec.label} to discover models.` + ? notFoundError : `${spec.label} model discovery failed to start. Check the agent CLI configuration and try again.` }) } @@ -156,7 +158,9 @@ export async function discoverModelsLocal(input: { finish( outputLimitExceeded ? { success: false, error: `${spec.label} returned too much model data.` } - : finalizeModelDiscoveryOutput(spec, stdout, stderr, code) + : isSupervisedProviderNotFound(code, stderr) + ? { success: false, error: notFoundError } + : finalizeModelDiscoveryOutput(spec, stdout, stderr, code) ) } child.stdout?.on('data', onStdoutData) diff --git a/src/main/text-generation/commit-message-text-generation-cancellation.test.ts b/src/main/text-generation/commit-message-text-generation-cancellation.test.ts index a40912290b74..c277d6d9b4f4 100644 --- a/src/main/text-generation/commit-message-text-generation-cancellation.test.ts +++ b/src/main/text-generation/commit-message-text-generation-cancellation.test.ts @@ -22,6 +22,12 @@ vi.mock('../windows-process-tree-kill', () => ({ terminateWindowsProcessTree: terminateWindowsProcessTreeMock })) +vi.mock('../codex/codex-app-server-posix-supervisor', async (importOriginal) => + (await import('./commit-message-text-generation-test-harness')).directSpawnProviderSupervisor( + importOriginal + ) +) + vi.mock('child_process', async (importOriginal) => { const actual = await importOriginal() return { diff --git a/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts b/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts index f73152255da6..767cfdc81ff4 100644 --- a/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts +++ b/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts @@ -15,6 +15,12 @@ vi.mock('../windows-process-tree-kill', () => ({ terminateWindowsProcessTree: terminateWindowsProcessTreeMock })) +vi.mock('../codex/codex-app-server-posix-supervisor', async (importOriginal) => + (await import('./commit-message-text-generation-test-harness')).directSpawnProviderSupervisor( + importOriginal + ) +) + vi.mock('child_process', async (importOriginal) => { const actual = await importOriginal() return { diff --git a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts index d935dc33c922..72651a3165a4 100644 --- a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts +++ b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts @@ -12,6 +12,7 @@ import { import { createChildTerminationExpectation, createMockDiscoveryChild, + type MockDiscoveryChild, withPlatform } from './commit-message-text-generation-test-harness' @@ -23,6 +24,12 @@ vi.mock('../windows-process-tree-kill', () => ({ terminateWindowsProcessTree: terminateWindowsProcessTreeMock })) +vi.mock('../codex/codex-app-server-posix-supervisor', async (importOriginal) => + (await import('./commit-message-text-generation-test-harness')).directSpawnProviderSupervisor( + importOriginal + ) +) + vi.mock('child_process', async (importOriginal) => { const actual = await importOriginal() return { @@ -33,6 +40,10 @@ vi.mock('child_process', async (importOriginal) => { const spawnMock = vi.mocked(spawn) +function enoent(): Error { + return Object.assign(new Error('spawn claude ENOENT'), { code: 'ENOENT' }) +} + const expectChildTerminated = createChildTerminationExpectation(terminateWindowsProcessTreeMock) beforeEach(() => { @@ -334,6 +345,28 @@ describe('discoverCommitMessageModelsLocal', () => { }) }) + it.each([ + ['a spawn error', (child: MockDiscoveryChild) => child.emit('error', enoent())], + [ + 'the supervisor exit for a missing binary', + (child: MockDiscoveryChild) => { + child.stderr.emit('data', Buffer.from('spawn claude ENOENT\n')) + child.emit('close', 127) + } + ] + ])('reports a missing CLI as not found on PATH from %s', async (_, fail) => { + const child = createMockDiscoveryChild() + spawnMock.mockReturnValue(child as never) + + const pending = discoverCommitMessageModelsLocal('claude', undefined) + fail(child) + + await expect(pending).resolves.toEqual({ + success: false, + error: 'claude not found on PATH. Install Claude to discover models.' + }) + }) + it('settles and detaches model discovery when timeout kill is ignored', async () => { vi.useFakeTimers() const child = createMockDiscoveryChild() diff --git a/src/main/text-generation/commit-message-text-generation-test-harness.ts b/src/main/text-generation/commit-message-text-generation-test-harness.ts index 8ba925ef0878..963d1a3d1389 100644 --- a/src/main/text-generation/commit-message-text-generation-test-harness.ts +++ b/src/main/text-generation/commit-message-text-generation-test-harness.ts @@ -1,5 +1,6 @@ import { EventEmitter } from 'node:events' import { expect, vi } from 'vitest' +import type * as ProviderSupervisor from '../codex/codex-app-server-posix-supervisor' export type MockDiscoveryChild = EventEmitter & { pid: number @@ -47,3 +48,18 @@ export function createChildTerminationExpectation( await vi.waitFor(() => expect(child.kill).toHaveBeenCalledWith('SIGKILL')) } } + +/** + * The supervisor module with agents spawned directly, as on Windows, for suites that drive fake + * children; the supervised launch and stop have their own tests. + */ +export async function directSpawnProviderSupervisor( + importOriginal: () => Promise +): Promise { + const actual = await importOriginal() + return { + ...actual, + createProviderSpawnSpec: (launch, env, _platform, options) => + actual.createProviderSpawnSpec(launch, env, 'win32', options) + } +} diff --git a/src/main/text-generation/source-control-agent-launch.test.ts b/src/main/text-generation/source-control-agent-launch.test.ts new file mode 100644 index 000000000000..d60b80737adb --- /dev/null +++ b/src/main/text-generation/source-control-agent-launch.test.ts @@ -0,0 +1,129 @@ +import { EventEmitter } from 'node:events' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import type * as RunProcess from '../../shared/child-process/run-process' +import type { ProcessSpec } from '../../shared/child-process/process-spec' +import { POSIX_PROVIDER_SUPERVISOR_SCRIPT } from '../codex/codex-app-server-posix-supervisor' +import { generateCommitMessageFromContext } from './commit-message-text-generation' +import { withPlatform } from './commit-message-text-generation-test-harness' +import { spawnSourceControlAgent } from './source-control-agent-launch' + +const { spawnProcessMock } = vi.hoisted(() => ({ spawnProcessMock: vi.fn() })) + +vi.mock('../../shared/child-process/run-process', async (importOriginal) => { + const actual = await importOriginal() + spawnProcessMock.mockImplementation(actual.spawnProcess) + return { ...actual, spawnProcess: spawnProcessMock } +}) + +function fakeChild(): EventEmitter & { stdin: { on: () => void; end: () => void } } { + return Object.assign(new EventEmitter(), { stdin: { on: vi.fn(), end: vi.fn() } }) +} + +function lastSpawnSpec(): ProcessSpec { + return spawnProcessMock.mock.calls.at(-1)![0] +} + +function decodedSupervisorSpec(spec: ProcessSpec): Record { + return JSON.parse(Buffer.from(spec.env!.ORCA_PROVIDER_SUPERVISOR_SPEC!, 'base64').toString()) +} + +const folder = mkdtempSync(join(tmpdir(), 'orca-agent-launch-')) +afterAll(() => rmSync(folder, { recursive: true, force: true })) + +beforeEach(() => { + spawnProcessMock.mockClear() +}) + +describe('spawnSourceControlAgent', () => { + it.each([ + ['in its own cwd', true, '/work/repo'], + ['in the process cwd', false, process.cwd()] + ])('runs a POSIX agent one-shot under the provider supervisor %s', (_, useCwd, cwd) => { + spawnProcessMock.mockReturnValueOnce(fakeChild()) + const child = withPlatform('linux', () => + spawnSourceControlAgent({ + binary: '/opt/agent/claude', + args: ['-p', '--verbose'], + cwd: '/work/repo', + env: { PATH: '/usr/bin', AGENT_TOKEN: 'kept' }, + stdinMode: 'pipe', + useCwdForNative: useCwd + }) + ) + + const spec = lastSpawnSpec() + expect(child.supervised).toBe(true) + expect(spec).toMatchObject({ + program: process.execPath, + args: ['-e', POSIX_PROVIDER_SUPERVISOR_SCRIPT], + cwd, + detached: true, + stdio: ['pipe', 'pipe', 'pipe'] + }) + expect(spec.env).toMatchObject({ PATH: '/usr/bin', AGENT_TOKEN: 'kept' }) + expect(decodedSupervisorSpec(spec)).toMatchObject({ + command: '/opt/agent/claude', + args: ['-p', '--verbose'], + cwd, + lifetime: 'one-shot' + }) + }) + + it('spawns a Windows agent directly, without a supervisor', () => { + spawnProcessMock.mockReturnValueOnce(fakeChild()) + const child = withPlatform('win32', () => + spawnSourceControlAgent({ + binary: 'C:/tools/agent.exe', + args: ['--print'], + env: { PATH: 'C:/tools' }, + stdinMode: 'pipe', + useCwdForNative: false + }) + ) + + expect(child.supervised).toBeUndefined() + expect(lastSpawnSpec()).toMatchObject({ program: 'C:/tools/agent.exe', args: ['--print'] }) + expect(lastSpawnSpec().env).not.toHaveProperty('ORCA_PROVIDER_SUPERVISOR_SPEC') + }) +}) + +describe.runIf(process.platform !== 'win32')('supervised agent processes', () => { + it('reports a missing agent binary as not found, as a direct spawn does', async () => { + const missing = join(folder, 'missing-agent') + + await expect( + generateCommitMessageFromContext( + { branch: 'main', stagedSummary: 'M README.md', stagedPatch: '+test' }, + { agentId: 'custom', model: '', customAgentCommand: `"${missing}"` }, + { kind: 'local', cwd: folder } + ) + ).resolves.toEqual({ + success: false, + error: `${missing} not found on PATH. Install ${missing} to use AI commit messages.` + }) + }) + + it('passes the stdin end through so the agent can answer its request', async () => { + const agent = join(folder, 'echo-agent.cjs') + writeFileSync( + agent, + `let prompt = '' +process.stdin.on('data', (chunk) => (prompt += chunk)) +process.stdin.on('end', () => setTimeout(() => console.log(prompt.includes('README') ? 'Update README' : 'no prompt'), 1200)) +` + ) + + await expect( + generateCommitMessageFromContext( + { branch: 'main', stagedSummary: 'M README.md', stagedPatch: '+test' }, + { agentId: 'custom', model: '', customAgentCommand: `"${process.execPath}" "${agent}"` }, + { kind: 'local', cwd: folder } + ) + ).resolves.toMatchObject({ success: true, message: 'Update README' }) + expect(lastSpawnSpec().program).toBe(process.execPath) + expect(decodedSupervisorSpec(lastSpawnSpec())).toMatchObject({ lifetime: 'one-shot' }) + }) +}) diff --git a/src/main/text-generation/source-control-agent-launch.ts b/src/main/text-generation/source-control-agent-launch.ts index 709c04813903..163d6f61daff 100644 --- a/src/main/text-generation/source-control-agent-launch.ts +++ b/src/main/text-generation/source-control-agent-launch.ts @@ -1,9 +1,11 @@ import { spawnProcess } from '../../shared/child-process/run-process' import { withCliRuntimeOnPath } from '../../shared/node-cli-command-resolution' import { resolveCliCommand } from '../codex-cli/command' +import { createProviderSpawnSpec } from '../codex/codex-app-server-posix-supervisor' import { wslAwareSpawn } from '../git/runner' import { getSpawnArgsForWindows } from '../win32-utils' import type { + SourceControlAgentSpawnInput, SpawnedSourceControlAgentProcess, SpawnSourceControlAgent } from './source-control-text-generation-types' @@ -57,20 +59,55 @@ export const spawnSourceControlAgent: SpawnSourceControlAgent = (input) => { } ) as SpawnedSourceControlAgentProcess } - const resolvedBinary = + const child = process.platform === 'win32' - ? resolveCliCommand(input.binary, { pathEnv: spawnEnv.PATH ?? spawnEnv.Path ?? null }) - : input.binary + ? spawnWindowsAgent(input, spawnEnv) + : spawnSupervisedAgent(input, spawnEnv) + if (input.stdinMode === 'ignore') { + child.stdin?.on?.('error', () => {}) + child.stdin?.end() + } + return child +} + +function spawnWindowsAgent( + input: SourceControlAgentSpawnInput, + spawnEnv: NodeJS.ProcessEnv +): SpawnedSourceControlAgentProcess { + const resolvedBinary = resolveCliCommand(input.binary, { + pathEnv: spawnEnv.PATH ?? spawnEnv.Path ?? null + }) const { spawnCmd, spawnArgs } = getSpawnArgsForWindows(resolvedBinary, input.args) - const child = spawnProcess({ + return spawnProcess({ program: spawnCmd, args: spawnArgs, env: withCliRuntimeOnPath(resolvedBinary, spawnEnv), ...(input.useCwdForNative ? { cwd: input.cwd } : {}) }) - if (input.stdinMode === 'ignore') { - child.stdin?.on?.('error', () => {}) - child.stdin?.end() - } - return child +} + +// Under the provider supervisor, an Orca that quits or dies mid-run still stops the agent's group. +function spawnSupervisedAgent( + input: SourceControlAgentSpawnInput, + spawnEnv: NodeJS.ProcessEnv +): SpawnedSourceControlAgentProcess { + const spec = createProviderSpawnSpec( + { + command: input.binary, + args: input.args, + ...(input.useCwdForNative && input.cwd !== undefined ? { cwd: input.cwd } : {}) + }, + withCliRuntimeOnPath(input.binary, spawnEnv), + process.platform, + { lifetime: 'one-shot' } + ) + const child = spawnProcess({ + program: spec.program, + args: spec.args, + env: spec.env, + cwd: spec.cwd, + detached: spec.detached, + stdio: ['pipe', 'pipe', 'pipe'] + }) + return Object.assign(child, { supervised: spec.supervised }) } diff --git a/src/main/text-generation/source-control-local-process.test.ts b/src/main/text-generation/source-control-local-process.test.ts new file mode 100644 index 000000000000..dec32178cb9e --- /dev/null +++ b/src/main/text-generation/source-control-local-process.test.ts @@ -0,0 +1,119 @@ +import { EventEmitter } from 'node:events' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../codex/codex-app-server-posix-supervisor' +import { spawnSourceControlAgent } from './source-control-agent-launch' +import { killSourceControlAgentProcess } from './source-control-local-process' +import type { SpawnedSourceControlAgentProcess } from './source-control-text-generation-types' + +const { terminateTreeMock } = vi.hoisted(() => ({ + terminateTreeMock: vi.fn(async () => true) +})) + +vi.mock('../codex/codex-app-server-process-teardown', () => ({ + terminateCodexAppServerProcessTree: terminateTreeMock +})) + +type FakeSupervisor = EventEmitter & { + pid: number + exitCode: number | null + signalCode: NodeJS.Signals | null + supervised: true + kill: ReturnType boolean>> +} + +function fakeSupervisor(exitsOn: NodeJS.Signals | null): FakeSupervisor { + const child: FakeSupervisor = Object.assign(new EventEmitter(), { + pid: 4242, + exitCode: null, + signalCode: null, + supervised: true as const, + kill: vi.fn((signal?: NodeJS.Signals) => { + if (signal === exitsOn) { + child.signalCode = signal + child.emit('exit', null, signal) + } + return true + }) + }) + return child +} + +function asSpawned(child: FakeSupervisor): SpawnedSourceControlAgentProcess { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the stop reads only pid, exit state, kill and the exit event, which the fake implements. + return child as unknown as SpawnedSourceControlAgentProcess +} + +afterEach(() => { + vi.useRealTimers() + terminateTreeMock.mockClear() +}) + +describe('killSourceControlAgentProcess for a supervised agent', () => { + it('asks the supervisor to stop and leaves its group to it', async () => { + const child = fakeSupervisor('SIGTERM') + + await killSourceControlAgentProcess(asSpawned(child)) + + expect(child.kill.mock.calls).toEqual([['SIGTERM']]) + expect(terminateTreeMock).not.toHaveBeenCalled() + }) + + it('tears the tree down only once the supervisor has had its full stop time', async () => { + vi.useFakeTimers() + const child = fakeSupervisor(null) + + const stopped = killSourceControlAgentProcess(asSpawned(child)) + await vi.advanceTimersByTimeAsync(PROVIDER_SUPERVISOR_MAX_STOP_MS - 1) + expect(child.kill.mock.calls).toEqual([['SIGTERM']]) + expect(terminateTreeMock).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(1) + await stopped + expect(terminateTreeMock).toHaveBeenCalledWith(child) + expect(child.kill).not.toHaveBeenCalledWith('SIGKILL') + }) + + it('signals nothing once the supervisor has exited', async () => { + const child = fakeSupervisor(null) + child.exitCode = 0 + + await killSourceControlAgentProcess(asSpawned(child)) + + expect(child.kill).not.toHaveBeenCalled() + expect(terminateTreeMock).not.toHaveBeenCalled() + }) +}) + +describe.runIf(process.platform !== 'win32')( + 'killSourceControlAgentProcess on a real agent', + () => { + it('stops an agent that ignores its stdin end through its supervisor', async () => { + const child = spawnSourceControlAgent({ + binary: process.execPath, + args: ['-e', 'console.log(process.pid); setInterval(() => {}, 60000)'], + env: process.env, + stdinMode: 'ignore', + useCwdForNative: false + }) + const agentPid = await new Promise((resolve) => + child.stdout.once('data', (chunk: Buffer) => resolve(Number(chunk.toString().trim()))) + ) + + try { + await killSourceControlAgentProcess(child) + + expect(child.signalCode ?? child.exitCode).not.toBeNull() + expect(() => process.kill(agentPid, 0)).toThrow(expect.objectContaining({ code: 'ESRCH' })) + expect(terminateTreeMock).not.toHaveBeenCalled() + } finally { + for (const pid of [agentPid, child.pid!]) { + try { + process.kill(pid, 'SIGKILL') + } catch { + // Already gone, as the test expects. + } + } + } + }) + } +) diff --git a/src/main/text-generation/source-control-local-process.ts b/src/main/text-generation/source-control-local-process.ts index 1a2d6c52b82e..01ca3c9f42dd 100644 --- a/src/main/text-generation/source-control-local-process.ts +++ b/src/main/text-generation/source-control-local-process.ts @@ -1,4 +1,10 @@ import type { CommitMessagePlan } from '../../shared/commit-message-plan' +import { + isSupervisedProviderNotFound, + PROVIDER_SUPERVISOR_MAX_STOP_MS +} from '../codex/codex-app-server-posix-supervisor' +import { terminateCodexAppServerProcessTree } from '../codex/codex-app-server-process-teardown' +import { waitForProcessExitUntil } from '../codex/codex-process-exit-deadline' import { UnsafeWindowsBatchArgumentsError } from '../win32-utils' import { terminateWindowsProcessTree } from '../windows-process-tree-kill' import { @@ -29,6 +35,10 @@ export async function killSourceControlAgentProcess( if (!pid) { return } + if (child.supervised) { + await stopSupervisedAgent(child) + return + } if (process.platform === 'win32') { // taskkill owns the tree, but the own-Chromium gate can refuse the // pid-addressed walk; the handle-addressed root kill below cannot reach the @@ -43,6 +53,24 @@ export async function killSourceControlAgentProcess( } } +// The supervisor stops the agent's group itself; a SIGKILL any sooner orphans that group. +async function stopSupervisedAgent(child: SpawnedSourceControlAgentProcess): Promise { + const exited = (): boolean => child.exitCode !== null || child.signalCode !== null + if (exited()) { + return + } + const exit = new Promise((resolve) => child.once('exit', () => resolve())) + try { + child.kill('SIGTERM') + } catch { + // The process may exit between the exit check and kill. + } + await waitForProcessExitUntil(exit, PROVIDER_SUPERVISOR_MAX_STOP_MS) + if (!exited()) { + await terminateCodexAppServerProcessTree(child) + } +} + // Why: Windows caps the CreateProcess command line at 32,767 UTF-16 code units, // including the executable path, per-arg quoting, and separators. The budget // leaves headroom for cmd.exe `/d /c` shim wrappers. @@ -193,15 +221,13 @@ export function runLocalSourceControlPlan(input: { } stderr += chunk.toString('utf-8') } + const notFoundError = `${plan.binary} not found on PATH. Install ${plan.label} to use AI commit messages.` const onError = (error: Error): void => { if (!child.pid) { markProcessClosed() } if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - finalize({ - success: false, - error: `${plan.binary} not found on PATH. Install ${plan.label} to use AI commit messages.` - }) + finalize({ success: false, error: notFoundError }) return } console.error('[commit-message] Local generator failed after spawn:', error) @@ -223,6 +249,10 @@ export function runLocalSourceControlPlan(input: { }) return } + if (isSupervisedProviderNotFound(code, stderr)) { + finalize({ success: false, error: notFoundError }) + return + } finalize( finalizeFromAgentOutput({ code, diff --git a/src/main/text-generation/source-control-text-generation-types.ts b/src/main/text-generation/source-control-text-generation-types.ts index a7b74c35d3a3..20b0e0d3e929 100644 --- a/src/main/text-generation/source-control-text-generation-types.ts +++ b/src/main/text-generation/source-control-text-generation-types.ts @@ -77,12 +77,15 @@ export type LocalProcessExecution = { processClosed: Promise } -export type SpawnedSourceControlAgentProcess = ReturnType +export type SpawnedSourceControlAgentProcess = ReturnType & { + /** True when the child is the POSIX provider supervisor: SIGTERM stops the agent's group, then itself. */ + readonly supervised?: boolean +} export type LocalGenerationTarget = Extract export type RemoteGenerationTarget = Extract -export type SpawnSourceControlAgent = (input: { +export type SourceControlAgentSpawnInput = { binary: string args: string[] cwd?: string @@ -92,4 +95,8 @@ export type SpawnSourceControlAgent = (input: { commandEnv?: Record stdinMode: 'ignore' | 'pipe' useCwdForNative: boolean -}) => SpawnedSourceControlAgentProcess +} + +export type SpawnSourceControlAgent = ( + input: SourceControlAgentSpawnInput +) => SpawnedSourceControlAgentProcess From ef72be7a0d4b5922176505e79b62a772e4144093 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:14:41 -0700 Subject: [PATCH 03/38] fix(supervisor): carry the provider argv on the supervisor's argv, map every spawn error back, and share one stop ladder - The supervisor read the provider's command and arguments from one base64 JSON env string. Linux caps a single env string at 128 KiB, so an argv prompt of about 90-120 KiB, which passes the 120 KiB per-argument guard, failed execve with E2BIG. The provider argv now follows the supervisor script's '--' as real arguments; the env keeps only small fields. - A supervisor that cannot start its provider reports Node's spawn error line and exits 127. That line now becomes the same error a direct spawn emits, so ENOENT still reads as 'not found on PATH' and EACCES or any other spawn error reads as 'failed to start'. - stopSupervisedProvider is the one ask, wait and force ladder: it gives a supervisor its full stop time before forcing. Agent one-shots, the Codex app-server close and the Claude child exit proof now share it, with the same requests, bounds and forced steps as before. --- .../claude-agent-sdk-process-spawn.test.ts | 8 +- .../claude/claude-child-exit-proof-ladder.ts | 50 +++++---- .../claude-stream-json-connection.test.ts | 12 +- src/main/codex/codex-app-server-connection.ts | 31 ++--- ...erver-posix-supervisor.integration.test.ts | 23 +++- .../codex-app-server-posix-supervisor.test.ts | 106 ++++++++++++++++-- .../codex-app-server-posix-supervisor.ts | 59 ++++++++-- .../commit-message-model-discovery.ts | 14 ++- ...ge-text-generation-model-discovery.test.ts | 34 +++--- .../source-control-agent-launch.test.ts | 25 +++-- .../source-control-local-process.test.ts | 84 ++++++++++++++ .../source-control-local-process.ts | 40 ++++--- 12 files changed, 368 insertions(+), 118 deletions(-) diff --git a/src/main/claude/claude-agent-sdk-process-spawn.test.ts b/src/main/claude/claude-agent-sdk-process-spawn.test.ts index 1b6c192d433a..ad1c90859acf 100644 --- a/src/main/claude/claude-agent-sdk-process-spawn.test.ts +++ b/src/main/claude/claude-agent-sdk-process-spawn.test.ts @@ -87,14 +87,18 @@ describe('claude agent SDK process spawn', () => { expect(spawn.pid).toBe(4321) expect(spec.program).toBe(globalThis.process.execPath) expect(spec.args?.[0]).toBe('-e') + expect(spec.args?.slice(2)).toEqual([ + '--', + '/usr/local/bin/claude', + '--output-format', + 'stream-json' + ]) expect(spec.detached).toBe(true) expect(spec.cwd).toBe('/work/repo') const supervisorSpec = JSON.parse( Buffer.from(String(spec.env?.ORCA_PROVIDER_SUPERVISOR_SPEC), 'base64').toString() ) expect(supervisorSpec).toMatchObject({ - command: '/usr/local/bin/claude', - args: ['--output-format', 'stream-json'], cwd: '/work/repo', ownerPid: globalThis.process.pid }) diff --git a/src/main/claude/claude-child-exit-proof-ladder.ts b/src/main/claude/claude-child-exit-proof-ladder.ts index 37d4e47508d7..841958b17b5b 100644 --- a/src/main/claude/claude-child-exit-proof-ladder.ts +++ b/src/main/claude/claude-child-exit-proof-ladder.ts @@ -1,11 +1,16 @@ import type { SpawnedProcess } from '../../shared/child-process/run-process' import { waitForProcessExitUntil } from '../codex/codex-process-exit-deadline' -import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../codex/codex-app-server-posix-supervisor' +import { + PROVIDER_SUPERVISOR_MAX_STOP_MS, + stopSupervisedProvider +} from '../codex/codex-app-server-posix-supervisor' import type { ClaudeChildTreeReaper } from './claude-agent-sdk-exit-proof' export const GRACEFUL_EXIT_MS = 1_500 +const SUPERVISED_EXIT_SLACK_MS = 500 // A signalled supervisor escalates on its own; forcing it sooner kills it and orphans Claude. -export const SUPERVISED_GRACEFUL_EXIT_MS = PROVIDER_SUPERVISOR_MAX_STOP_MS + 500 +export const SUPERVISED_GRACEFUL_EXIT_MS = + PROVIDER_SUPERVISOR_MAX_STOP_MS + SUPERVISED_EXIT_SLACK_MS const FORCED_EXIT_MS = 1_000 export type ClaudeChildExitProofInput = { @@ -24,29 +29,30 @@ export async function proveClaudeChildExitWithReaper( const tree = input.tree ?? createTree() // Arm before the stop: only a live root can identify its descendants. await tree.capture() - try { - input.child.stdin?.end() - } catch { - // The reap below still owns the process. - } - // Stdin end alone lets Claude finish its turn, tools and edits included; a close is a stop. - // Windows has no supervisor, and a direct SIGTERM there is TerminateProcess. - if (input.supervised && !input.exited()) { - input.child.kill('SIGTERM') - } - let reaped = false - if (!input.exited()) { - await waitForProcessExitUntil( - input.exitPromise, - input.supervised ? SUPERVISED_GRACEFUL_EXIT_MS : GRACEFUL_EXIT_MS - ) - if (!input.exited()) { - reaped = true + const reaped = await stopSupervisedProvider({ + request: () => { + try { + input.child.stdin?.end() + } catch { + // The reap below still owns the process. + } + // Stdin end alone lets Claude finish its turn, tools and edits included; a close is a stop. + // Windows has no supervisor, and a direct SIGTERM there is TerminateProcess. + if (input.supervised && !input.exited()) { + input.child.kill('SIGTERM') + } + }, + exitPromise: input.exitPromise, + exited: input.exited, + force: async () => { await tree.refresh?.() await tree.reap() await waitForProcessExitUntil(input.exitPromise, FORCED_EXIT_MS) - } - } + }, + supervised: input.supervised === true, + directWaitMs: GRACEFUL_EXIT_MS, + slackMs: SUPERVISED_EXIT_SLACK_MS + }) if (!reaped && input.exited() && tree.treeVerdict !== 'exited') { await tree.reap() } diff --git a/src/main/claude/claude-stream-json-connection.test.ts b/src/main/claude/claude-stream-json-connection.test.ts index 082ed4c12ce7..f7ecad5f8ea1 100644 --- a/src/main/claude/claude-stream-json-connection.test.ts +++ b/src/main/claude/claude-stream-json-connection.test.ts @@ -110,15 +110,9 @@ async function open( } function launchedArgv(spec: ProcessSpec | undefined): string[] { - const supervised = spec?.env?.ORCA_PROVIDER_SUPERVISOR_SPEC - if (!supervised) { - return [spec?.program ?? '', ...(spec?.args ?? [])] - } - const launch: unknown = JSON.parse(Buffer.from(supervised, 'base64').toString()) - if (!launch || typeof launch !== 'object' || !('command' in launch) || !('args' in launch)) { - return [] - } - return [String(launch.command), ...(Array.isArray(launch.args) ? launch.args.map(String) : [])] + const argv = [spec?.program ?? '', ...(spec?.args ?? [])] + // A supervised launch carries the provider's argv after the supervisor script's '--'. + return spec?.env?.ORCA_PROVIDER_SUPERVISOR_SPEC ? argv.slice(argv.indexOf('--') + 1) : argv } function childEnv(): Record { diff --git a/src/main/codex/codex-app-server-connection.ts b/src/main/codex/codex-app-server-connection.ts index 070d3e3c301c..8552619b7e02 100644 --- a/src/main/codex/codex-app-server-connection.ts +++ b/src/main/codex/codex-app-server-connection.ts @@ -2,7 +2,7 @@ import { spawnProcess } from '../../shared/child-process/run-process' import { RetryableProcessExitProof } from '../../shared/child-process/retryable-process-exit-proof' import { createProviderSpawnSpec, - PROVIDER_SUPERVISOR_MAX_STOP_MS + stopSupervisedProvider } from './codex-app-server-posix-supervisor' import { buildCodexAppServerExitError } from './codex-app-server-exit-error' import { initializeCodexAppServerConnection } from './codex-app-server-handshake' @@ -245,25 +245,26 @@ export async function openCodexAppServerConnection( } closing = true return exitProof.run(async () => { - try { - child.stdin.end() - } catch { - // Already destroyed; the reap below still runs. - } - if (!exited) { - // The POSIX supervisor stops its own provider group; forcing it any sooner can orphan it. - await waitForProcessExitUntil( - exitPromise, - process.platform === 'win32' ? GRACEFUL_EXIT_MS : PROVIDER_SUPERVISOR_MAX_STOP_MS - ) - if (!exited) { + await stopSupervisedProvider({ + request: () => { + try { + child.stdin.end() + } catch { + // Already destroyed; the reap below still runs. + } + }, + exitPromise, + exited: () => exited, + force: async () => { const treeExited = await terminateProcessTree() await waitForProcessExitUntil(exitPromise, FORCED_EXIT_MS) // The lease follows the root, which is gone: a child left behind is reported by the // owner, and blocks nothing. processTreeUnproven = !treeExited && exitObserved - } - } + }, + supervised: spawnSpec.supervised, + directWaitMs: GRACEFUL_EXIT_MS + }) dispatcher.failPending(new Error('codex app-server connection closed')) return exitObserved }) diff --git a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts index 2efcdbc5ef99..919004c43f03 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts @@ -4,7 +4,6 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' import { - POSIX_PROVIDER_SUPERVISOR_SCRIPT, PROVIDER_SIGTERM_GRACE_MS, PROVIDER_STDIN_END_GRACE_MS, PROVIDER_SUPERVISOR_MAX_STOP_MS, @@ -74,7 +73,7 @@ const OWNER = String.raw` if (quitsCleanly) process.on('SIGUSR2', () => process.exit(0)) const spec = JSON.parse(Buffer.from(process.env.ORCA_PROVIDER_SUPERVISOR_SPEC, 'base64').toString()) spec.ownerPid = process.pid - const supervisor = spawn(process.execPath, ['-e', process.env.ORCA_TEST_SUPERVISOR_SCRIPT], { + const supervisor = spawn(process.execPath, JSON.parse(process.env.ORCA_TEST_SUPERVISOR_ARGS), { env: { ...process.env, ORCA_PROVIDER_SUPERVISOR_SPEC: Buffer.from(JSON.stringify(spec)).toString('base64') }, stdio: ['pipe', 'pipe', 'ignore'], detached: true @@ -192,7 +191,7 @@ async function launchUnderOwner( options ) const owner = spawn(process.execPath, ['-e', OWNER], { - env: { ...launch.env, ORCA_TEST_SUPERVISOR_SCRIPT: POSIX_PROVIDER_SUPERVISOR_SCRIPT }, + env: { ...launch.env, ORCA_TEST_SUPERVISOR_ARGS: JSON.stringify(launch.args) }, stdio: ['ignore', 'pipe', 'ignore'] }) recordedPids.add(owner.pid!) @@ -433,6 +432,24 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe expect(alive(grandchild)).toBe(false) }) + it('hands the provider a near-cap argv prompt intact', async () => { + const prompt = 'x'.repeat(110 * 1024) + const { supervisor, exit } = launchSupervisor( + { lifetime: 'one-shot' }, + {}, + { + command: process.execPath, + args: ['-e', 'process.stdout.write(String(process.argv[1].length))', prompt] + } + ) + let stdout = '' + supervisor.stdout!.on('data', (chunk: Buffer) => (stdout += chunk.toString())) + supervisor.stdin!.end() + + await expect(exit).resolves.toEqual({ code: 0, signal: null }) + expect(stdout).toBe(String(prompt.length)) + }) + it('relays all of the provider output to a slow owner before exiting', async () => { const bytes = 4 * 1024 * 1024 const { supervisor, exit } = launchSupervisor( diff --git a/src/main/codex/codex-app-server-posix-supervisor.test.ts b/src/main/codex/codex-app-server-posix-supervisor.test.ts index 1d01e6120ada..d7b3a2541752 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.test.ts @@ -1,12 +1,14 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import type { CodexAppServerLaunch } from './codex-app-server-connection' import { createProviderSpawnSpec, - isSupervisedProviderNotFound, POSIX_PROVIDER_SUPERVISOR_SCRIPT, PROVIDER_SIGTERM_GRACE_MS, PROVIDER_STDIN_END_GRACE_MS, - supervisedPosixLaunch + PROVIDER_SUPERVISOR_MAX_STOP_MS, + stopSupervisedProvider, + supervisedPosixLaunch, + supervisedProviderSpawnError } from './codex-app-server-posix-supervisor' const launch: CodexAppServerLaunch = { @@ -22,14 +24,19 @@ describe('structured provider supervision', () => { const spec = supervisedPosixLaunch(launch, childEnv) expect(spec.command).toBe(process.execPath) - expect(spec.args).toEqual(['-e', POSIX_PROVIDER_SUPERVISOR_SCRIPT]) + expect(spec.args).toEqual([ + '-e', + POSIX_PROVIDER_SUPERVISOR_SCRIPT, + '--', + '/opt/codex', + 'app-server', + '--flag' + ]) expect(spec.env.PATH).toBe('/bin') expect( JSON.parse(Buffer.from(spec.env.ORCA_PROVIDER_SUPERVISOR_SPEC!, 'base64').toString()) ).toEqual( expect.objectContaining({ - command: '/opt/codex', - args: ['app-server', '--flag'], cwd: '/work/repo', ownerPid: process.pid, lifetime: 'session', @@ -59,14 +66,89 @@ describe('structured provider supervision', () => { expect(spec).toMatchObject({ program: process.execPath, detached: true, supervised: true }) expect( JSON.parse(Buffer.from(spec.env.ORCA_PROVIDER_SUPERVISOR_SPEC!, 'base64').toString()) - ).toMatchObject({ command: '/opt/codex', args: ['app-server', '--flag'], lifetime: 'one-shot' }) + ).toMatchObject({ lifetime: 'one-shot' }) }) - it('reads only the supervisor own missing-binary exit as not found', () => { - expect(isSupervisedProviderNotFound(127, 'spawn /opt/my tools/claude ENOENT\n')).toBe(true) - expect(isSupervisedProviderNotFound(1, 'spawn claude ENOENT\n')).toBe(false) - expect(isSupervisedProviderNotFound(127, 'sh: claude: command not found\n')).toBe(false) - expect(isSupervisedProviderNotFound(127, 'warning\nspawn claude ENOENT\n')).toBe(false) + it('keeps every argv and env string of a 120 KiB argv prompt under the Linux 128 KiB cap', () => { + const prompt = 'x'.repeat(120 * 1024) + const spec = createProviderSpawnSpec( + { command: '/opt/agent', args: ['--print', prompt] }, + { PATH: '/bin' }, + 'linux', + { lifetime: 'one-shot' } + ) + const strings = [ + ...spec.args, + ...Object.entries(spec.env).map(([key, value]) => `${key}=${value}`) + ] + + expect(spec.args.slice(-2)).toEqual(['--print', prompt]) + for (const value of strings) { + expect(Buffer.byteLength(value)).toBeLessThan(128 * 1024) + } + }) + + it.each([ + ['ENOENT', 127, 'spawn /opt/my tools/claude ENOENT\n'], + ['EACCES', 127, 'spawn /opt/claude EACCES\n'] + ])('reads the supervisor own %s spawn failure as that spawn error', (errno, code, stderr) => { + expect(supervisedProviderSpawnError(code, stderr)).toMatchObject({ + code: errno, + message: stderr.trim() + }) + }) + + it.each([ + [1, 'spawn claude ENOENT\n'], + [127, 'sh: claude: command not found\n'], + [127, 'warning\nspawn claude ENOENT\n'], + [127, 'spawn claude ENOENT: no such file\n'] + ])('reads no spawn failure from exit %s with %j', (code, stderr) => { + expect(supervisedProviderSpawnError(code, stderr)).toBeNull() + }) + + it.each([ + [true, PROVIDER_SUPERVISOR_MAX_STOP_MS + 500], + [false, 1_500] + ])('forces a provider (supervised %s) only after %s ms', async (supervised, waitMs) => { + vi.useFakeTimers() + try { + const request = vi.fn() + const force = vi.fn(async () => {}) + const stopped = stopSupervisedProvider({ + request, + exitPromise: new Promise(() => {}), + exited: () => false, + force, + supervised, + directWaitMs: 1_500, + slackMs: 500 + }) + expect(request).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(waitMs - 1) + expect(force).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await expect(stopped).resolves.toBe(true) + expect(force).toHaveBeenCalledTimes(1) + } finally { + vi.useRealTimers() + } + }) + + it('never forces a provider that exits within its wait', async () => { + let exited = false + let markExited!: () => void + const force = vi.fn(async () => {}) + const stopped = stopSupervisedProvider({ + request: () => setTimeout(() => ((exited = true), markExited()), 10), + exitPromise: new Promise((resolve) => (markExited = resolve)), + exited: () => exited, + force, + supervised: true + }) + + await expect(stopped).resolves.toBe(false) + expect(force).not.toHaveBeenCalled() }) it('refuses a grace longer than recovery waits before SIGKILL', () => { diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index aea2100919e3..6d8d87c898b0 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -1,4 +1,5 @@ import type { CodexAppServerLaunch } from './codex-app-server-connection' +import { waitForProcessExitUntil } from './codex-process-exit-deadline' /** Time the provider gets to exit on its own after its stdin ends, before SIGTERM. */ export const PROVIDER_STDIN_END_GRACE_MS = 1_000 @@ -23,6 +24,8 @@ export const PROVIDER_SUPERVISOR_MAX_STOP_MS = export const POSIX_PROVIDER_SUPERVISOR_SCRIPT = ` const { spawn } = require('node:child_process') const spec = JSON.parse(Buffer.from(process.env.ORCA_PROVIDER_SUPERVISOR_SPEC, 'base64').toString()) +// The provider's own argv, after this script's '--'. +const [providerCommand, ...providerArgs] = process.argv.slice(1) // A detached supervisor is reparented when its owner exits. The new parent may // be PID 1 or a platform subreaper, so any other parent means no live owner. const ownerGone = () => process.ppid !== spec.ownerPid @@ -33,7 +36,7 @@ if (ownerGone()) process.exit(1) const childEnv = { ...process.env } delete childEnv.ORCA_PROVIDER_SUPERVISOR_SPEC delete childEnv.ELECTRON_RUN_AS_NODE -const child = spawn(spec.command, spec.args, { +const child = spawn(providerCommand, providerArgs, { cwd: spec.cwd, env: childEnv, stdio: ['pipe', 'pipe', 'pipe'], @@ -146,9 +149,52 @@ child.once('exit', (code, signal) => { */ export type ProviderSupervisorLifetime = 'session' | 'one-shot' -/** The supervisor could not start its provider: it relays Node's spawn ENOENT and exits 127. */ -export function isSupervisedProviderNotFound(code: number | null, stderr: string): boolean { - return code === 127 && /^spawn .+ ENOENT$/.test(stderr.trim()) +/** + * The error a direct spawn would have emitted, from a supervisor that could not start its provider + * (it relays Node's spawn error line and exits 127); null for any other exit. + */ +export function supervisedProviderSpawnError( + code: number | null, + stderr: string +): NodeJS.ErrnoException | null { + const line = stderr.trim() + const errno = code === 127 ? /^spawn .+ (E[A-Z0-9]+)$/.exec(line)?.[1] : undefined + return errno ? Object.assign(new Error(line), { code: errno }) : null +} + +export type ProviderStopInput = { + /** Asks the child to stop: a SIGTERM, or the stdin end that closes a session. */ + request: () => void + exitPromise: Promise + exited: () => boolean + /** Runs only if the child outlives its wait. */ + force: () => Promise + /** The child is the supervisor; a direct (Windows) child is waited on for `directWaitMs`. */ + supervised: boolean + directWaitMs?: number + slackMs?: number +} + +/** + * Stops a provider child, forcing it only after its wait. A supervisor gets its full stop time: + * forcing it any sooner kills it mid-ladder and orphans the provider group. True when it forced. + */ +export async function stopSupervisedProvider(input: ProviderStopInput): Promise { + input.request() + if (input.exited()) { + return false + } + await waitForProcessExitUntil( + input.exitPromise, + input.supervised + ? PROVIDER_SUPERVISOR_MAX_STOP_MS + (input.slackMs ?? 0) + : (input.directWaitMs ?? 0) + ) + if (input.exited()) { + return false + } + await input.force() + return true } export type ProviderSupervisorOptions = { @@ -180,10 +226,9 @@ export function supervisedPosixLaunch( ): { command: string; args: string[]; env: NodeJS.ProcessEnv } { assertGraceWithin('stdin-end', stdinEndGraceMs, PROVIDER_STDIN_END_GRACE_MS) assertGraceWithin('SIGTERM', sigtermGraceMs, PROVIDER_SIGTERM_GRACE_MS) + // Only small fields ride in the env: Linux caps one env string at 128 KiB, and argv prompts near it. const supervisorSpec = Buffer.from( JSON.stringify({ - command: launch.command, - args: launch.args, cwd, ownerPid, lifetime, @@ -193,7 +238,7 @@ export function supervisedPosixLaunch( ).toString('base64') return { command: process.execPath, - args: ['-e', POSIX_PROVIDER_SUPERVISOR_SCRIPT], + args: ['-e', POSIX_PROVIDER_SUPERVISOR_SCRIPT, '--', launch.command, ...launch.args], // Electron's executable needs Node mode for the inline supervisor. The // marker is removed above so providers never inherit Electron semantics. env: { diff --git a/src/main/text-generation/commit-message-model-discovery.ts b/src/main/text-generation/commit-message-model-discovery.ts index c4005992df99..0411415c65ab 100644 --- a/src/main/text-generation/commit-message-model-discovery.ts +++ b/src/main/text-generation/commit-message-model-discovery.ts @@ -4,7 +4,7 @@ import type { CommitMessagePlan } from '../../shared/commit-message-plan' import { getAgentModelProbeSpec } from '../../shared/agent-model-probe-spec' import type { TuiAgent } from '../../shared/tui-agent' import { resolveCodexHomeProcessLockKeyForSpawnEnv } from '../codex-cli/codex-home-process-lock' -import { isSupervisedProviderNotFound } from '../codex/codex-app-server-posix-supervisor' +import { supervisedProviderSpawnError } from '../codex/codex-app-server-posix-supervisor' import { isSshRequestOutcomeUnverifiable } from '../ssh/ssh-channel-multiplexer' import { WINDOWS_BATCH_UNSAFE_ARGUMENTS_ERROR } from '../win32-utils' import { @@ -140,7 +140,6 @@ export async function discoverModelsLocal(input: { } const onStdoutData = (chunk: Buffer): void => onData(chunk, (text) => (stdout += text)) const onStderrData = (chunk: Buffer): void => onData(chunk, (text) => (stderr += text)) - const notFoundError = `${spec.modelDiscovery?.binary ?? spec.binary} not found on PATH. Install ${spec.label} to discover models.` const onError = (error: Error): void => { if (!child.pid) { markProcessClosed() @@ -149,18 +148,21 @@ export async function discoverModelsLocal(input: { success: false, error: (error as NodeJS.ErrnoException).code === 'ENOENT' - ? notFoundError + ? `${spec.modelDiscovery?.binary ?? spec.binary} not found on PATH. Install ${spec.label} to discover models.` : `${spec.label} model discovery failed to start. Check the agent CLI configuration and try again.` }) } const onClose = (code: number | null): void => { markClosedAfterTermination() + const spawnError = outputLimitExceeded ? null : supervisedProviderSpawnError(code, stderr) + if (spawnError) { + onError(spawnError) + return + } finish( outputLimitExceeded ? { success: false, error: `${spec.label} returned too much model data.` } - : isSupervisedProviderNotFound(code, stderr) - ? { success: false, error: notFoundError } - : finalizeModelDiscoveryOutput(spec, stdout, stderr, code) + : finalizeModelDiscoveryOutput(spec, stdout, stderr, code) ) } child.stdout?.on('data', onStdoutData) diff --git a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts index 72651a3165a4..cc8d14a5495b 100644 --- a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts +++ b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts @@ -12,7 +12,6 @@ import { import { createChildTerminationExpectation, createMockDiscoveryChild, - type MockDiscoveryChild, withPlatform } from './commit-message-text-generation-test-harness' @@ -40,8 +39,8 @@ vi.mock('child_process', async (importOriginal) => { const spawnMock = vi.mocked(spawn) -function enoent(): Error { - return Object.assign(new Error('spawn claude ENOENT'), { code: 'ENOENT' }) +function spawnError(errno: string): Error { + return Object.assign(new Error(`spawn claude ${errno}`), { code: errno }) } const expectChildTerminated = createChildTerminationExpectation(terminateWindowsProcessTreeMock) @@ -345,26 +344,27 @@ describe('discoverCommitMessageModelsLocal', () => { }) }) + const notFound = 'claude not found on PATH. Install Claude to discover models.' + const failedToStart = + 'Claude model discovery failed to start. Check the agent CLI configuration and try again.' it.each([ - ['a spawn error', (child: MockDiscoveryChild) => child.emit('error', enoent())], - [ - 'the supervisor exit for a missing binary', - (child: MockDiscoveryChild) => { - child.stderr.emit('data', Buffer.from('spawn claude ENOENT\n')) - child.emit('close', 127) - } - ] - ])('reports a missing CLI as not found on PATH from %s', async (_, fail) => { + ['a spawn error', 'ENOENT', notFound], + ['the supervisor exit for a spawn error', 'ENOENT', notFound], + ['a spawn error', 'EACCES', failedToStart], + ['the supervisor exit for a spawn error', 'EACCES', failedToStart] + ])('reports %s %s as a direct spawn does', async (source, errno, error) => { const child = createMockDiscoveryChild() spawnMock.mockReturnValue(child as never) const pending = discoverCommitMessageModelsLocal('claude', undefined) - fail(child) + if (source === 'a spawn error') { + child.emit('error', spawnError(errno)) + } else { + child.stderr.emit('data', Buffer.from(`spawn claude ${errno}\n`)) + child.emit('close', 127) + } - await expect(pending).resolves.toEqual({ - success: false, - error: 'claude not found on PATH. Install Claude to discover models.' - }) + await expect(pending).resolves.toEqual({ success: false, error }) }) it('settles and detaches model discovery when timeout kill is ignored', async () => { diff --git a/src/main/text-generation/source-control-agent-launch.test.ts b/src/main/text-generation/source-control-agent-launch.test.ts index d60b80737adb..a85ec1905e77 100644 --- a/src/main/text-generation/source-control-agent-launch.test.ts +++ b/src/main/text-generation/source-control-agent-launch.test.ts @@ -58,18 +58,13 @@ describe('spawnSourceControlAgent', () => { expect(child.supervised).toBe(true) expect(spec).toMatchObject({ program: process.execPath, - args: ['-e', POSIX_PROVIDER_SUPERVISOR_SCRIPT], + args: ['-e', POSIX_PROVIDER_SUPERVISOR_SCRIPT, '--', '/opt/agent/claude', '-p', '--verbose'], cwd, detached: true, stdio: ['pipe', 'pipe', 'pipe'] }) expect(spec.env).toMatchObject({ PATH: '/usr/bin', AGENT_TOKEN: 'kept' }) - expect(decodedSupervisorSpec(spec)).toMatchObject({ - command: '/opt/agent/claude', - args: ['-p', '--verbose'], - cwd, - lifetime: 'one-shot' - }) + expect(decodedSupervisorSpec(spec)).toMatchObject({ cwd, lifetime: 'one-shot' }) }) it('spawns a Windows agent directly, without a supervisor', () => { @@ -106,6 +101,22 @@ describe.runIf(process.platform !== 'win32')('supervised agent processes', () => }) }) + it('reports an agent that cannot be executed as failing to start, as a direct spawn does', async () => { + const agent = join(folder, 'not-executable-agent') + writeFileSync(agent, '#!/bin/sh\necho never\n', { mode: 0o644 }) + + await expect( + generateCommitMessageFromContext( + { branch: 'main', stagedSummary: 'M README.md', stagedPatch: '+test' }, + { agentId: 'custom', model: '', customAgentCommand: `"${agent}"` }, + { kind: 'local', cwd: folder } + ) + ).resolves.toEqual({ + success: false, + error: `${agent} failed to start. Check the agent command in Settings and try again.` + }) + }) + it('passes the stdin end through so the agent can answer its request', async () => { const agent = join(folder, 'echo-agent.cjs') writeFileSync( diff --git a/src/main/text-generation/source-control-local-process.test.ts b/src/main/text-generation/source-control-local-process.test.ts index dec32178cb9e..713783906473 100644 --- a/src/main/text-generation/source-control-local-process.test.ts +++ b/src/main/text-generation/source-control-local-process.test.ts @@ -1,8 +1,13 @@ import { EventEmitter } from 'node:events' +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../codex/codex-app-server-posix-supervisor' +import { cancelGenerateCommitMessageLocal } from './commit-message-text-generation' import { spawnSourceControlAgent } from './source-control-agent-launch' import { killSourceControlAgentProcess } from './source-control-local-process' +import { generateCommitMessage } from './source-control-text-generation-requests' import type { SpawnedSourceControlAgentProcess } from './source-control-text-generation-types' const { terminateTreeMock } = vi.hoisted(() => ({ @@ -43,6 +48,15 @@ function asSpawned(child: FakeSupervisor): SpawnedSourceControlAgentProcess { return child as unknown as SpawnedSourceControlAgentProcess } +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch { + return false + } +} + afterEach(() => { vi.useRealTimers() terminateTreeMock.mockClear() @@ -115,5 +129,75 @@ describe.runIf(process.platform !== 'win32')( } } }) + + it('holds the Codex home until a canceled agent has stopped through its supervisor', async () => { + const folder = mkdtempSync(join(tmpdir(), 'orca-supervised-cancel-')) + const pids: number[] = [] + try { + const home = join(folder, 'codex-home') + for (const repo of ['first', 'second']) { + mkdirSync(join(folder, repo)) + } + const pidFile = join(folder, 'first-pid') + const stoppedFile = join(folder, 'first-stopped') + // Ignores its stdin end, then takes a moment to stop on SIGTERM, as a CLI flushing state does. + const slowStop = join(folder, 'slow-stop.cjs') + writeFileSync( + slowStop, + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)) +process.stdin.resume() +process.on('SIGTERM', () => setTimeout(() => { + require('node:fs').writeFileSync(${JSON.stringify(stoppedFile)}, 'stopped') + process.exit(0) +}, 300)) +setInterval(() => {}, 60000) +` + ) + const answers = join(folder, 'answers.cjs') + writeFileSync( + answers, + "process.stdin.resume().on('end', () => console.log('Update README'))\n" + ) + const firstAliveAtSecondSpawn: boolean[] = [] + const spawnAgent = vi.fn((input: Parameters[0]) => { + if (pids.length > 0) { + firstAliveAtSecondSpawn.push(isAlive(pids[0]!)) + } + return spawnSourceControlAgent(input) + }) + const request = (cwd: string, script: string): ReturnType => + generateCommitMessage({ + context: { branch: 'main', stagedSummary: 'M README.md', stagedPatch: '+test' }, + params: { + agentId: 'codex', + model: 'gpt-5.5', + agentCommandOverride: `CODEX_HOME="${home}" "${process.execPath}" "${script}"` + }, + target: { kind: 'local', cwd, env: process.env }, + spawnAgent + }) + + const first = request(join(folder, 'first'), slowStop) + await vi.waitFor(() => expect(existsSync(pidFile)).toBe(true), { timeout: 10_000 }) + pids.push(Number(readFileSync(pidFile, 'utf8'))) + cancelGenerateCommitMessageLocal(join(folder, 'first')) + await expect(first).resolves.toMatchObject({ canceled: true }) + + await expect(request(join(folder, 'second'), answers)).resolves.toMatchObject({ + success: true, + message: 'Update README' + }) + expect(firstAliveAtSecondSpawn).toEqual([false]) + expect(readFileSync(stoppedFile, 'utf8')).toBe('stopped') + expect(terminateTreeMock).not.toHaveBeenCalled() + } finally { + for (const pid of pids) { + if (isAlive(pid)) { + process.kill(pid, 'SIGKILL') + } + } + rmSync(folder, { recursive: true, force: true }) + } + }) } ) diff --git a/src/main/text-generation/source-control-local-process.ts b/src/main/text-generation/source-control-local-process.ts index 01ca3c9f42dd..9f0b2cbd5a08 100644 --- a/src/main/text-generation/source-control-local-process.ts +++ b/src/main/text-generation/source-control-local-process.ts @@ -1,10 +1,9 @@ import type { CommitMessagePlan } from '../../shared/commit-message-plan' import { - isSupervisedProviderNotFound, - PROVIDER_SUPERVISOR_MAX_STOP_MS + stopSupervisedProvider, + supervisedProviderSpawnError } from '../codex/codex-app-server-posix-supervisor' import { terminateCodexAppServerProcessTree } from '../codex/codex-app-server-process-teardown' -import { waitForProcessExitUntil } from '../codex/codex-process-exit-deadline' import { UnsafeWindowsBatchArgumentsError } from '../win32-utils' import { terminateWindowsProcessTree } from '../windows-process-tree-kill' import { @@ -53,22 +52,24 @@ export async function killSourceControlAgentProcess( } } -// The supervisor stops the agent's group itself; a SIGKILL any sooner orphans that group. async function stopSupervisedAgent(child: SpawnedSourceControlAgentProcess): Promise { const exited = (): boolean => child.exitCode !== null || child.signalCode !== null if (exited()) { return } - const exit = new Promise((resolve) => child.once('exit', () => resolve())) - try { - child.kill('SIGTERM') - } catch { - // The process may exit between the exit check and kill. - } - await waitForProcessExitUntil(exit, PROVIDER_SUPERVISOR_MAX_STOP_MS) - if (!exited()) { - await terminateCodexAppServerProcessTree(child) - } + await stopSupervisedProvider({ + request: () => { + try { + child.kill('SIGTERM') + } catch { + // The process may exit between the exit check and kill. + } + }, + exitPromise: new Promise((resolve) => child.once('exit', () => resolve())), + exited, + force: () => terminateCodexAppServerProcessTree(child), + supervised: true + }) } // Why: Windows caps the CreateProcess command line at 32,767 UTF-16 code units, @@ -221,13 +222,15 @@ export function runLocalSourceControlPlan(input: { } stderr += chunk.toString('utf-8') } - const notFoundError = `${plan.binary} not found on PATH. Install ${plan.label} to use AI commit messages.` const onError = (error: Error): void => { if (!child.pid) { markProcessClosed() } if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - finalize({ success: false, error: notFoundError }) + finalize({ + success: false, + error: `${plan.binary} not found on PATH. Install ${plan.label} to use AI commit messages.` + }) return } console.error('[commit-message] Local generator failed after spawn:', error) @@ -249,8 +252,9 @@ export function runLocalSourceControlPlan(input: { }) return } - if (isSupervisedProviderNotFound(code, stderr)) { - finalize({ success: false, error: notFoundError }) + const spawnError = supervisedProviderSpawnError(code, stderr) + if (spawnError) { + onError(spawnError) return } finalize( From 24b29ab0a37e1f4bf3e712f11a3dc40187fa81c8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:31:57 -0700 Subject: [PATCH 04/38] fix(supervisor): report every provider spawn failure on one marked stderr line Node throws most spawn failures (ENOEXEC, ENOTDIR, ELOOP, EPERM, ...) instead of emitting them, and the supervisor had no catch, so it died with exit 1 and a stack trace that the user saw as the agent's failure. A thrown or emitted spawn failure now exits 127 with one marked line carrying whether it was thrown, its code and its message. Orca reads only the last stderr line, so a runtime warning printed earlier cannot hide it, and maps it to the message a direct spawn gave: thrown is 'could not be started', ENOENT is 'not found on PATH', and any other emitted error is 'failed to start'. --- .../codex-app-server-posix-supervisor.test.ts | 38 +++++++---- .../codex-app-server-posix-supervisor.ts | 68 ++++++++++++++----- .../commit-message-model-discovery.ts | 22 +++--- ...ge-text-generation-model-discovery.test.ts | 45 +++++++++--- .../source-control-agent-launch.test.ts | 40 +++++++++++ .../source-control-local-process.ts | 20 +++--- 6 files changed, 175 insertions(+), 58 deletions(-) diff --git a/src/main/codex/codex-app-server-posix-supervisor.test.ts b/src/main/codex/codex-app-server-posix-supervisor.test.ts index d7b3a2541752..25fba52e5420 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.test.ts @@ -7,8 +7,9 @@ import { PROVIDER_STDIN_END_GRACE_MS, PROVIDER_SUPERVISOR_MAX_STOP_MS, stopSupervisedProvider, + PROVIDER_SPAWN_FAILURE_MARKER, supervisedPosixLaunch, - supervisedProviderSpawnError + supervisedProviderSpawnFailure } from './codex-app-server-posix-supervisor' const launch: CodexAppServerLaunch = { @@ -88,23 +89,32 @@ describe('structured provider supervision', () => { } }) + const report = (thrown: boolean, code: string, message: string): string => + `${PROVIDER_SPAWN_FAILURE_MARKER}${JSON.stringify({ thrown, code, message })}\n` + it.each([ - ['ENOENT', 127, 'spawn /opt/my tools/claude ENOENT\n'], - ['EACCES', 127, 'spawn /opt/claude EACCES\n'] - ])('reads the supervisor own %s spawn failure as that spawn error', (errno, code, stderr) => { - expect(supervisedProviderSpawnError(code, stderr)).toMatchObject({ - code: errno, - message: stderr.trim() - }) + ['an emitted ENOENT', report(false, 'ENOENT', 'spawn /opt/my tools/claude ENOENT'), false], + ['a thrown ENOTDIR', report(true, 'ENOTDIR', 'spawn ENOTDIR'), true], + [ + 'a report after a runtime warning', + `Warning: Ignoring extra certs from \`/missing.pem\`, load failed\n${report(false, 'EACCES', 'spawn /opt/claude EACCES')}`, + false + ] + ])('reads %s from the supervisor last stderr line', (_, stderr, thrown) => { + const failure = supervisedProviderSpawnFailure(127, stderr) + + expect(failure?.thrown).toBe(thrown) + expect(failure?.error.code).toMatch(/^E[A-Z]+$/) + expect(failure?.error.message).toMatch(/^spawn /) }) it.each([ - [1, 'spawn claude ENOENT\n'], - [127, 'sh: claude: command not found\n'], - [127, 'warning\nspawn claude ENOENT\n'], - [127, 'spawn claude ENOENT: no such file\n'] - ])('reads no spawn failure from exit %s with %j', (code, stderr) => { - expect(supervisedProviderSpawnError(code, stderr)).toBeNull() + ['another exit code', 1, report(false, 'ENOENT', 'spawn claude ENOENT')], + ['a provider line after the report', 127, `${report(true, 'ENOEXEC', 'spawn ENOEXEC')}more\n`], + ['an unmarked spawn line', 127, 'spawn claude ENOENT\n'], + ['a malformed report', 127, `${PROVIDER_SPAWN_FAILURE_MARKER}{"thrown":true}\n`] + ])('reads no spawn failure from %s', (_, code, stderr) => { + expect(supervisedProviderSpawnFailure(code, stderr)).toBeNull() }) it.each([ diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index 6d8d87c898b0..85201d7bea6a 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -20,6 +20,9 @@ export const PROVIDER_OUTPUT_DRAIN_TIMEOUT_MS = 1_000 export const PROVIDER_SUPERVISOR_MAX_STOP_MS = PROVIDER_STDIN_END_GRACE_MS + PROVIDER_SIGTERM_GRACE_MS + PROVIDER_GROUP_REAP_TIMEOUT_MS +/** Starts the one stderr line that reports a provider the supervisor could not start. */ +export const PROVIDER_SPAWN_FAILURE_MARKER = '[orca-provider-supervisor] spawn failed: ' + /** Inline supervisor source kept dependency-free for the spawned Node child. */ export const POSIX_PROVIDER_SUPERVISOR_SCRIPT = ` const { spawn } = require('node:child_process') @@ -36,12 +39,24 @@ if (ownerGone()) process.exit(1) const childEnv = { ...process.env } delete childEnv.ORCA_PROVIDER_SUPERVISOR_SPEC delete childEnv.ELECTRON_RUN_AS_NODE -const child = spawn(providerCommand, providerArgs, { - cwd: spec.cwd, - env: childEnv, - stdio: ['pipe', 'pipe', 'pipe'], - detached: true -}) +// The owner sees only this pid's exit; this marked last stderr line says the provider never started. +const exitWithSpawnFailure = (error, thrown) => { + const report = { thrown, code: (error && error.code) || 'UNKNOWN', message: String(error && error.message) } + try { require('node:fs').writeSync(2, ${JSON.stringify(PROVIDER_SPAWN_FAILURE_MARKER)} + JSON.stringify(report) + '\\n') } catch {} + process.exit(127) +} +let child +try { + child = spawn(providerCommand, providerArgs, { + cwd: spec.cwd, + env: childEnv, + stdio: ['pipe', 'pipe', 'pipe'], + detached: true + }) +} catch (error) { + // Node throws most spawn failures (ENOEXEC, ENOTDIR, ...) rather than emitting them. + exitWithSpawnFailure(error, true) +} let timer let ownerShutdownTimer let settling = false @@ -135,8 +150,7 @@ timer = setInterval(() => { timer.unref() child.once('error', (error) => { clearInterval(timer) - // The owner sees only this pid's exit; stderr is where a missing provider binary can say so. - process.stderr.write(String(error && error.message) + '\\n', () => process.exit(127)) + exitWithSpawnFailure(error, false) }) child.once('exit', (code, signal) => { void reapProviderExit(code, signal) @@ -149,17 +163,37 @@ child.once('exit', (code, signal) => { */ export type ProviderSupervisorLifetime = 'session' | 'one-shot' -/** - * The error a direct spawn would have emitted, from a supervisor that could not start its provider - * (it relays Node's spawn error line and exits 127); null for any other exit. - */ -export function supervisedProviderSpawnError( +/** A provider the supervisor could not start: `thrown` when a direct spawn would have thrown. */ +export type SupervisedProviderSpawnFailure = { thrown: boolean; error: NodeJS.ErrnoException } + +/** The failure a supervisor reported on its last stderr line before exiting 127; null otherwise. */ +export function supervisedProviderSpawnFailure( code: number | null, stderr: string -): NodeJS.ErrnoException | null { - const line = stderr.trim() - const errno = code === 127 ? /^spawn .+ (E[A-Z0-9]+)$/.exec(line)?.[1] : undefined - return errno ? Object.assign(new Error(line), { code: errno }) : null +): SupervisedProviderSpawnFailure | null { + const line = stderr.trimEnd().split('\n').at(-1) ?? '' + if (code !== 127 || !line.startsWith(PROVIDER_SPAWN_FAILURE_MARKER)) { + return null + } + let report: unknown + try { + report = JSON.parse(line.slice(PROVIDER_SPAWN_FAILURE_MARKER.length)) + } catch { + return null + } + if ( + !report || + typeof report !== 'object' || + !('thrown' in report && typeof report.thrown === 'boolean') || + !('code' in report && typeof report.code === 'string') || + !('message' in report && typeof report.message === 'string') + ) { + return null + } + return { + thrown: report.thrown, + error: Object.assign(new Error(report.message), { code: report.code }) + } } export type ProviderStopInput = { diff --git a/src/main/text-generation/commit-message-model-discovery.ts b/src/main/text-generation/commit-message-model-discovery.ts index 0411415c65ab..7b7fa6f3f796 100644 --- a/src/main/text-generation/commit-message-model-discovery.ts +++ b/src/main/text-generation/commit-message-model-discovery.ts @@ -4,7 +4,7 @@ import type { CommitMessagePlan } from '../../shared/commit-message-plan' import { getAgentModelProbeSpec } from '../../shared/agent-model-probe-spec' import type { TuiAgent } from '../../shared/tui-agent' import { resolveCodexHomeProcessLockKeyForSpawnEnv } from '../codex-cli/codex-home-process-lock' -import { supervisedProviderSpawnError } from '../codex/codex-app-server-posix-supervisor' +import { supervisedProviderSpawnFailure } from '../codex/codex-app-server-posix-supervisor' import { isSshRequestOutcomeUnverifiable } from '../ssh/ssh-channel-multiplexer' import { WINDOWS_BATCH_UNSAFE_ARGUMENTS_ERROR } from '../win32-utils' import { @@ -58,6 +58,7 @@ export async function discoverModelsLocal(input: { input.options.wslDistro ? 'linux' : process.platform ) + const couldNotStart = `${spec.label} model discovery could not be started. Check the agent CLI configuration and try again.` const startDiscovery = (): LocalProcessExecution => { let markProcessClosed!: () => void const processClosed = new Promise((resolve) => { @@ -83,10 +84,7 @@ export async function discoverModelsLocal(input: { } catch (error) { markProcessClosed() console.error('[commit-message] Failed to spawn model discovery:', error) - resolve({ - success: false, - error: `${spec.label} model discovery could not be started. Check the agent CLI configuration and try again.` - }) + resolve({ success: false, error: couldNotStart }) return } @@ -154,9 +152,17 @@ export async function discoverModelsLocal(input: { } const onClose = (code: number | null): void => { markClosedAfterTermination() - const spawnError = outputLimitExceeded ? null : supervisedProviderSpawnError(code, stderr) - if (spawnError) { - onError(spawnError) + // A supervised spawn failure reads as the same failure a direct spawn reports. + const spawnFailure = outputLimitExceeded + ? null + : supervisedProviderSpawnFailure(code, stderr) + if (spawnFailure?.thrown) { + console.error('[commit-message] Failed to spawn model discovery:', spawnFailure.error) + finish({ success: false, error: couldNotStart }) + return + } + if (spawnFailure) { + onError(spawnFailure.error) return } finish( diff --git a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts index cc8d14a5495b..099f4e8e1e4d 100644 --- a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts +++ b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts @@ -9,6 +9,7 @@ import { discoverCommitMessageModelsLocal, discoverCommitMessageModelsRemote } from './commit-message-text-generation' +import { PROVIDER_SPAWN_FAILURE_MARKER } from '../codex/codex-app-server-posix-supervisor' import { createChildTerminationExpectation, createMockDiscoveryChild, @@ -347,22 +348,44 @@ describe('discoverCommitMessageModelsLocal', () => { const notFound = 'claude not found on PATH. Install Claude to discover models.' const failedToStart = 'Claude model discovery failed to start. Check the agent CLI configuration and try again.' + const couldNotStart = + 'Claude model discovery could not be started. Check the agent CLI configuration and try again.' it.each([ - ['a spawn error', 'ENOENT', notFound], - ['the supervisor exit for a spawn error', 'ENOENT', notFound], - ['a spawn error', 'EACCES', failedToStart], - ['the supervisor exit for a spawn error', 'EACCES', failedToStart] - ])('reports %s %s as a direct spawn does', async (source, errno, error) => { + ['ENOENT', false, notFound], + ['EACCES', false, failedToStart], + ['ENOTDIR', true, couldNotStart] + ])( + 'reports a supervisor %s spawn failure as a direct spawn does', + async (errno, thrown, error) => { + const child = createMockDiscoveryChild() + spawnMock.mockReturnValue(child as never) + + const pending = discoverCommitMessageModelsLocal('claude', undefined) + child.stderr.emit( + 'data', + Buffer.from( + `Warning: an Electron startup notice\n${PROVIDER_SPAWN_FAILURE_MARKER}${JSON.stringify({ + thrown, + code: errno, + message: `spawn claude ${errno}` + })}\n` + ) + ) + child.emit('close', 127) + + await expect(pending).resolves.toEqual({ success: false, error }) + } + ) + + it.each([ + ['ENOENT', notFound], + ['EACCES', failedToStart] + ])('reports an emitted %s spawn error as before', async (errno, error) => { const child = createMockDiscoveryChild() spawnMock.mockReturnValue(child as never) const pending = discoverCommitMessageModelsLocal('claude', undefined) - if (source === 'a spawn error') { - child.emit('error', spawnError(errno)) - } else { - child.stderr.emit('data', Buffer.from(`spawn claude ${errno}\n`)) - child.emit('close', 127) - } + child.emit('error', spawnError(errno)) await expect(pending).resolves.toEqual({ success: false, error }) }) diff --git a/src/main/text-generation/source-control-agent-launch.test.ts b/src/main/text-generation/source-control-agent-launch.test.ts index a85ec1905e77..5a1f7c9df490 100644 --- a/src/main/text-generation/source-control-agent-launch.test.ts +++ b/src/main/text-generation/source-control-agent-launch.test.ts @@ -117,6 +117,46 @@ describe.runIf(process.platform !== 'win32')('supervised agent processes', () => }) }) + it.each([ + ['an executable that is not a program', 'garbage', (path: string) => path], + ['a path through a file', 'file', (path: string) => join(path, 'agent')] + ])('reports %s as not startable, as a direct spawn does', async (_, name, commandFor) => { + const file = join(folder, name) + writeFileSync(file, Buffer.from([0, 1, 2, 3]), { mode: 0o755 }) + const command = commandFor(file) + + await expect( + generateCommitMessageFromContext( + { branch: 'main', stagedSummary: 'M README.md', stagedPatch: '+test' }, + { agentId: 'custom', model: '', customAgentCommand: `"${command}"` }, + { kind: 'local', cwd: folder } + ) + ).resolves.toEqual({ + success: false, + error: `${command} could not be started. Check the agent command in Settings and try again.` + }) + }) + + // Electron prints startup warnings (a bad NODE_EXTRA_CA_CERTS); Node's debug log stands in here. + it('reads a missing binary as not found past the runtime own stderr output', async () => { + const missing = join(folder, 'missing-behind-warning') + + await expect( + generateCommitMessageFromContext( + { branch: 'main', stagedSummary: 'M README.md', stagedPatch: '+test' }, + { agentId: 'custom', model: '', customAgentCommand: `"${missing}"` }, + { + kind: 'local', + cwd: folder, + env: { ...process.env, NODE_DEBUG: 'child_process' } + } + ) + ).resolves.toEqual({ + success: false, + error: `${missing} not found on PATH. Install ${missing} to use AI commit messages.` + }) + }) + it('passes the stdin end through so the agent can answer its request', async () => { const agent = join(folder, 'echo-agent.cjs') writeFileSync( diff --git a/src/main/text-generation/source-control-local-process.ts b/src/main/text-generation/source-control-local-process.ts index 9f0b2cbd5a08..a627ec74886b 100644 --- a/src/main/text-generation/source-control-local-process.ts +++ b/src/main/text-generation/source-control-local-process.ts @@ -1,7 +1,7 @@ import type { CommitMessagePlan } from '../../shared/commit-message-plan' import { stopSupervisedProvider, - supervisedProviderSpawnError + supervisedProviderSpawnFailure } from '../codex/codex-app-server-posix-supervisor' import { terminateCodexAppServerProcessTree } from '../codex/codex-app-server-process-teardown' import { UnsafeWindowsBatchArgumentsError } from '../win32-utils' @@ -122,6 +122,7 @@ export function runLocalSourceControlPlan(input: { const processClosed = new Promise((resolve) => { markProcessClosed = resolve }) + const couldNotStart = `${plan.label} could not be started. Check the agent command in Settings and try again.` const result = new Promise((resolve) => { let child: SpawnedSourceControlAgentProcess try { @@ -150,10 +151,7 @@ export function runLocalSourceControlPlan(input: { return } console.error('[commit-message] Failed to spawn local generator:', error) - resolve({ - success: false, - error: `${plan.label} could not be started. Check the agent command in Settings and try again.` - }) + resolve({ success: false, error: couldNotStart }) return } @@ -252,9 +250,15 @@ export function runLocalSourceControlPlan(input: { }) return } - const spawnError = supervisedProviderSpawnError(code, stderr) - if (spawnError) { - onError(spawnError) + // A supervised spawn failure reads as the same failure a direct spawn reports. + const spawnFailure = supervisedProviderSpawnFailure(code, stderr) + if (spawnFailure?.thrown) { + console.error('[commit-message] Failed to spawn local generator:', spawnFailure.error) + finalize({ success: false, error: couldNotStart }) + return + } + if (spawnFailure) { + onError(spawnFailure.error) return } finalize( From 904043f69bd6b19a5c0d72acc932f3cb0db5512f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:33:46 -0700 Subject: [PATCH 05/38] fix(supervisor): keep the user's Node options away from the supervisor and hand them to the provider The supervisor runs Electron in Node mode, which honours NODE_OPTIONS and NODE_REPL_EXTERNAL_MODULE. A user value such as a --require of a missing file stopped the supervisor from starting, breaking even native CLIs like Codex that never load it. The launch now takes both out of the supervisor's environment, carries them in its spec, and restores them for the provider only, so a Node-based CLI still gets them. --- ...erver-posix-supervisor.integration.test.ts | 15 ++++++++++++++ .../codex-app-server-posix-supervisor.test.ts | 14 +++++++++++++ .../codex-app-server-posix-supervisor.ts | 20 ++++++++++++++++--- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts index 919004c43f03..be96ba07b1e7 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts @@ -432,6 +432,21 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe expect(alive(grandchild)).toBe(false) }) + it('keeps the user Node options from the supervisor and hands them to the provider', async () => { + const nodeOptions = `--require ${join(tempDir(), 'missing-preload.js')}` + const { supervisor, exit } = launchSupervisor( + { lifetime: 'one-shot' }, + { NODE_OPTIONS: nodeOptions }, + { command: '/bin/sh', args: ['-c', 'printf %s "$NODE_OPTIONS"'] } + ) + let stdout = '' + supervisor.stdout!.on('data', (chunk: Buffer) => (stdout += chunk.toString())) + supervisor.stdin!.end() + + await expect(exit).resolves.toEqual({ code: 0, signal: null }) + expect(stdout).toBe(nodeOptions) + }) + it('hands the provider a near-cap argv prompt intact', async () => { const prompt = 'x'.repeat(110 * 1024) const { supervisor, exit } = launchSupervisor( diff --git a/src/main/codex/codex-app-server-posix-supervisor.test.ts b/src/main/codex/codex-app-server-posix-supervisor.test.ts index 25fba52e5420..735db7ef11c5 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.test.ts @@ -70,6 +70,20 @@ describe('structured provider supervision', () => { ).toMatchObject({ lifetime: 'one-shot' }) }) + it('keeps the user Node options out of the supervisor env and in the provider spec', () => { + const spec = supervisedPosixLaunch(launch, { + PATH: '/bin', + NODE_OPTIONS: '--require /missing.js', + NODE_REPL_EXTERNAL_MODULE: '/repl.js' + }) + + expect(spec.env).not.toHaveProperty('NODE_OPTIONS') + expect(spec.env).not.toHaveProperty('NODE_REPL_EXTERNAL_MODULE') + expect( + JSON.parse(Buffer.from(spec.env.ORCA_PROVIDER_SUPERVISOR_SPEC!, 'base64').toString()).nodeEnv + ).toEqual({ NODE_OPTIONS: '--require /missing.js', NODE_REPL_EXTERNAL_MODULE: '/repl.js' }) + }) + it('keeps every argv and env string of a 120 KiB argv prompt under the Linux 128 KiB cap', () => { const prompt = 'x'.repeat(120 * 1024) const spec = createProviderSpawnSpec( diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index 85201d7bea6a..697623498836 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -36,7 +36,7 @@ const ownerGone = () => process.ppid !== spec.ownerPid for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) process.on(signal, () => stopProviderGroup(signal)) // Orca can die before this runs; spawning then would start a provider nothing watches. if (ownerGone()) process.exit(1) -const childEnv = { ...process.env } +const childEnv = { ...process.env, ...spec.nodeEnv } delete childEnv.ORCA_PROVIDER_SUPERVISOR_SPEC delete childEnv.ELECTRON_RUN_AS_NODE // The owner sees only this pid's exit; this marked last stderr line says the provider never started. @@ -240,6 +240,9 @@ export type ProviderSupervisorOptions = { sigtermGraceMs?: number } +// The user's Node startup options, as the CLI launchers stash them away from Electron's bootstrap. +const PROVIDER_ONLY_NODE_ENV_KEYS = ['NODE_OPTIONS', 'NODE_REPL_EXTERNAL_MODULE'] as const + // A longer grace than the max stop allows would let recovery or close SIGKILL mid-stop. function assertGraceWithin(name: string, graceMs: number, maxMs: number): void { if (!(graceMs >= 0 && graceMs <= maxMs)) { @@ -261,13 +264,24 @@ export function supervisedPosixLaunch( assertGraceWithin('stdin-end', stdinEndGraceMs, PROVIDER_STDIN_END_GRACE_MS) assertGraceWithin('SIGTERM', sigtermGraceMs, PROVIDER_SIGTERM_GRACE_MS) // Only small fields ride in the env: Linux caps one env string at 128 KiB, and argv prompts near it. + // Electron's Node bootstrap honours these too; held in the spec, they reach only the provider. + const supervisorEnv = { ...childEnv } + const nodeEnv: Record = {} + for (const key of PROVIDER_ONLY_NODE_ENV_KEYS) { + const value = supervisorEnv[key] + delete supervisorEnv[key] + if (value !== undefined) { + nodeEnv[key] = value + } + } const supervisorSpec = Buffer.from( JSON.stringify({ cwd, ownerPid, lifetime, stdinEndGraceMs, - sigtermGraceMs + sigtermGraceMs, + nodeEnv }) ).toString('base64') return { @@ -276,7 +290,7 @@ export function supervisedPosixLaunch( // Electron's executable needs Node mode for the inline supervisor. The // marker is removed above so providers never inherit Electron semantics. env: { - ...childEnv, + ...supervisorEnv, ELECTRON_RUN_AS_NODE: '1', ORCA_PROVIDER_SUPERVISOR_SPEC: supervisorSpec } From 7bef5010e8844858aed4f8042cfa7d23f8632fa2 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:35:20 -0700 Subject: [PATCH 06/38] fix(text-generation): file a forced agent one-shot teardown under its own breadcrumb site The forced tree teardown recorded every self-initiated kill as the Codex app-server's, so a forced commit-message or model-discovery stop read as a Codex teardown in crash breadcrumbs. The teardown now takes the caller's site; source-control stops pass the site their Windows tree kill already uses. --- .../codex-app-server-process-teardown.test.ts | 14 ++++++++++++++ .../codex/codex-app-server-process-teardown.ts | 10 +++++++--- .../source-control-local-process.test.ts | 4 +++- .../source-control-local-process.ts | 6 ++++-- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/main/codex/codex-app-server-process-teardown.test.ts b/src/main/codex/codex-app-server-process-teardown.test.ts index 295d780718d0..2b89af5722cb 100644 --- a/src/main/codex/codex-app-server-process-teardown.test.ts +++ b/src/main/codex/codex-app-server-process-teardown.test.ts @@ -140,6 +140,20 @@ describe('terminateCodexAppServerProcessTree', () => { ]) }) + it('files the group kill under the site that forced it', async () => { + await terminateCodexAppServerProcessTree(child(), { + platform: 'darwin', + captureDescendants: async () => ({ rootPgid: 1234, descendants: [], capturedAtMs: 1 }), + terminateDescendants: async () => true, + signalProcessGroup: vi.fn(), + site: 'source-control-text-generation' + }) + + expect(findSelfInitiatedTreeKills(Date.now())).toEqual([ + expect.objectContaining({ pid: 1234, site: 'source-control-text-generation' }) + ]) + }) + it('tears down 40 dedicated groups without process-table scans or cross-group fanout', async () => { const killMocks = Array.from({ length: 40 }, () => vi.fn(() => true)) const targets = killMocks.map((kill, index) => ({ diff --git a/src/main/codex/codex-app-server-process-teardown.ts b/src/main/codex/codex-app-server-process-teardown.ts index 10e625bef7e8..0bd8b028b440 100644 --- a/src/main/codex/codex-app-server-process-teardown.ts +++ b/src/main/codex/codex-app-server-process-teardown.ts @@ -15,8 +15,12 @@ export type CodexAppServerProcessTeardownDeps = { terminateDescendants?: (snapshot: DescendantSnapshot) => Promise terminateWindowsTree?: (rootPid: number, deps?: { site?: string }) => Promise signalProcessGroup?: (pgid: number, signal: NodeJS.Signals) => void + /** Who forced the teardown, for the self-initiated kill breadcrumbs. */ + site?: string } +const DEFAULT_TEARDOWN_SITE = 'codex-app-server-teardown' + function terminateDedicatedPosixGroup( rootPid: number, deps: CodexAppServerProcessTeardownDeps @@ -32,7 +36,7 @@ function terminateDedicatedPosixGroup( // Outside the try: that catch is the ESRCH contract, not a breadcrumb handler. recordSelfInitiatedTreeKill({ pid: rootPid, - site: 'codex-app-server-teardown', + site: deps.site ?? DEFAULT_TEARDOWN_SITE, scope: 'posix-process-group' }) return true @@ -74,7 +78,7 @@ async function terminatePosixTree( // already-gone contract, not a breadcrumb handler. recordSelfInitiatedTreeKill({ pid: snapshot.rootPgid, - site: 'codex-app-server-teardown', + site: deps.site ?? DEFAULT_TEARDOWN_SITE, scope: 'posix-process-group' }) } @@ -99,7 +103,7 @@ async function terminateOnce( } if ((deps.platform ?? process.platform) === 'win32') { const terminate = deps.terminateWindowsTree ?? terminateWindowsProcessTree - await terminate(rootPid, { site: 'codex-app-server-teardown' }) + await terminate(rootPid, { site: deps.site ?? DEFAULT_TEARDOWN_SITE }) // taskkill owns the tree; this preserves the prior direct-child fallback when it fails. child.kill('SIGKILL') return true diff --git a/src/main/text-generation/source-control-local-process.test.ts b/src/main/text-generation/source-control-local-process.test.ts index 713783906473..415a26178670 100644 --- a/src/main/text-generation/source-control-local-process.test.ts +++ b/src/main/text-generation/source-control-local-process.test.ts @@ -83,7 +83,9 @@ describe('killSourceControlAgentProcess for a supervised agent', () => { await vi.advanceTimersByTimeAsync(1) await stopped - expect(terminateTreeMock).toHaveBeenCalledWith(child) + expect(terminateTreeMock).toHaveBeenCalledWith(child, { + site: 'source-control-text-generation' + }) expect(child.kill).not.toHaveBeenCalledWith('SIGKILL') }) diff --git a/src/main/text-generation/source-control-local-process.ts b/src/main/text-generation/source-control-local-process.ts index a627ec74886b..13c95d2b3c0d 100644 --- a/src/main/text-generation/source-control-local-process.ts +++ b/src/main/text-generation/source-control-local-process.ts @@ -27,6 +27,8 @@ import type { TextGenerationOperation } from './source-control-text-generation-types' +const SOURCE_CONTROL_KILL_SITE = 'source-control-text-generation' + export async function killSourceControlAgentProcess( child: SpawnedSourceControlAgentProcess ): Promise { @@ -43,7 +45,7 @@ export async function killSourceControlAgentProcess( // pid-addressed walk; the handle-addressed root kill below cannot reach the // recycled pid it refused, and callers release the managed-home lock on this // promise, so it must not resolve having killed nothing. - await terminateWindowsProcessTree(pid, { site: 'source-control-text-generation' }) + await terminateWindowsProcessTree(pid, { site: SOURCE_CONTROL_KILL_SITE }) } try { child.kill('SIGKILL') @@ -67,7 +69,7 @@ async function stopSupervisedAgent(child: SpawnedSourceControlAgentProcess): Pro }, exitPromise: new Promise((resolve) => child.once('exit', () => resolve())), exited, - force: () => terminateCodexAppServerProcessTree(child), + force: () => terminateCodexAppServerProcessTree(child, { site: SOURCE_CONTROL_KILL_SITE }), supervised: true }) } From f95551dfddce80b1a136380872b7cc598ac67f4a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:38:43 -0700 Subject: [PATCH 07/38] test(text-generation): cover the supervised stop under timeout and output limit; name the direct-child suites The commit-message suites that drive fake children spawn them directly, the unsupervised shape Windows and WSL use, so they now say so. The supervised POSIX stop gets its own compositions: a timed-out Codex generation settles at once but holds the Codex home until its supervisor has stopped (faithful fake, fake timers), and an agent that floods past the output limit is stopped through its real supervisor with no process left behind. --- ...ssage-text-generation-cancellation.test.ts | 7 +- ...e-text-generation-local-subprocess.test.ts | 7 +- ...ge-text-generation-model-discovery.test.ts | 9 +- ...it-message-text-generation-test-harness.ts | 9 +- .../source-control-local-process.test.ts | 107 +++++++++++++++++- 5 files changed, 127 insertions(+), 12 deletions(-) diff --git a/src/main/text-generation/commit-message-text-generation-cancellation.test.ts b/src/main/text-generation/commit-message-text-generation-cancellation.test.ts index c277d6d9b4f4..a644c0e138fc 100644 --- a/src/main/text-generation/commit-message-text-generation-cancellation.test.ts +++ b/src/main/text-generation/commit-message-text-generation-cancellation.test.ts @@ -22,8 +22,11 @@ vi.mock('../windows-process-tree-kill', () => ({ terminateWindowsProcessTree: terminateWindowsProcessTreeMock })) +// Generation over a direct agent child, the unsupervised shape Windows and WSL spawn. The POSIX +// supervised stop composed with timeout, cancel, output limit and the Codex home lock is covered +// in source-control-local-process.test.ts. vi.mock('../codex/codex-app-server-posix-supervisor', async (importOriginal) => - (await import('./commit-message-text-generation-test-harness')).directSpawnProviderSupervisor( + (await import('./commit-message-text-generation-test-harness')).directAgentChildSupervisorModule( importOriginal ) ) @@ -46,7 +49,7 @@ beforeEach(() => { spawnMock.mockClear() }) -describe('generateCommitMessageFromContext', () => { +describe('generateCommitMessageFromContext over a direct agent child', () => { it('fails clearly before spawning when a jcode argv prompt exceeds the Windows command line', async () => { await withPlatform('win32', async () => { const pending = generateCommitMessageFromContext( diff --git a/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts b/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts index 767cfdc81ff4..d46193051197 100644 --- a/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts +++ b/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts @@ -15,8 +15,11 @@ vi.mock('../windows-process-tree-kill', () => ({ terminateWindowsProcessTree: terminateWindowsProcessTreeMock })) +// Generation over a direct agent child, the unsupervised shape Windows and WSL spawn. The POSIX +// supervised stop composed with timeout, cancel, output limit and the Codex home lock is covered +// in source-control-local-process.test.ts. vi.mock('../codex/codex-app-server-posix-supervisor', async (importOriginal) => - (await import('./commit-message-text-generation-test-harness')).directSpawnProviderSupervisor( + (await import('./commit-message-text-generation-test-harness')).directAgentChildSupervisorModule( importOriginal ) ) @@ -39,7 +42,7 @@ beforeEach(() => { spawnMock.mockClear() }) -describe('generateCommitMessageFromContext', () => { +describe('generateCommitMessageFromContext over a direct agent child', () => { it('caps local agent output before buffering unbounded data', async () => { const listeners = new Map void>() const child = { diff --git a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts index 099f4e8e1e4d..37c171c7b327 100644 --- a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts +++ b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts @@ -24,8 +24,11 @@ vi.mock('../windows-process-tree-kill', () => ({ terminateWindowsProcessTree: terminateWindowsProcessTreeMock })) +// Generation over a direct agent child, the unsupervised shape Windows and WSL spawn. The POSIX +// supervised stop composed with timeout, cancel, output limit and the Codex home lock is covered +// in source-control-local-process.test.ts. vi.mock('../codex/codex-app-server-posix-supervisor', async (importOriginal) => - (await import('./commit-message-text-generation-test-harness')).directSpawnProviderSupervisor( + (await import('./commit-message-text-generation-test-harness')).directAgentChildSupervisorModule( importOriginal ) ) @@ -52,7 +55,7 @@ beforeEach(() => { spawnMock.mockClear() }) -describe('discoverCommitMessageModelsLocal', () => { +describe('discoverCommitMessageModelsLocal over a direct agent child', () => { it('returns static catalog models without spawning for static agents', async () => { const result = await discoverCommitMessageModelsLocal('amp', undefined) @@ -501,7 +504,7 @@ describe('discoverCommitMessageModelsLocal', () => { }) }) -describe('generateCommitMessageFromContext', () => { +describe('generateCommitMessageFromContext over a direct agent child', () => { it('discovers dynamic models through a remote execution plan', async () => { const execute = vi.fn(async (plan, cwd, timeoutMs) => { expect(plan).toEqual({ diff --git a/src/main/text-generation/commit-message-text-generation-test-harness.ts b/src/main/text-generation/commit-message-text-generation-test-harness.ts index 963d1a3d1389..62875b3b3345 100644 --- a/src/main/text-generation/commit-message-text-generation-test-harness.ts +++ b/src/main/text-generation/commit-message-text-generation-test-harness.ts @@ -31,7 +31,8 @@ export function withPlatform(platform: NodeJS.Platform, fn: () => T): T { } // Binds the caller's hoisted tree-kill mock so test bodies keep calling -// expectChildTerminated(child) with no extra argument. +// expectChildTerminated(child) with no extra argument. Asserts the direct-child kill: a +// supervised child is stopped with SIGTERM instead (source-control-local-process.test.ts). export function createChildTerminationExpectation( terminateWindowsProcessTreeMock: ReturnType ): (child: { pid: number; kill: ReturnType }) => Promise { @@ -50,10 +51,10 @@ export function createChildTerminationExpectation( } /** - * The supervisor module with agents spawned directly, as on Windows, for suites that drive fake - * children; the supervised launch and stop have their own tests. + * The supervisor module with every agent spawned as a direct child, the unsupervised shape Windows + * and WSL use, for suites that drive fake children through generation. */ -export async function directSpawnProviderSupervisor( +export async function directAgentChildSupervisorModule( importOriginal: () => Promise ): Promise { const actual = await importOriginal() diff --git a/src/main/text-generation/source-control-local-process.test.ts b/src/main/text-generation/source-control-local-process.test.ts index 415a26178670..55863f4f4077 100644 --- a/src/main/text-generation/source-control-local-process.test.ts +++ b/src/main/text-generation/source-control-local-process.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../codex/codex-app-server-posix-supervisor' +import { SOURCE_CONTROL_GENERATION_TIMEOUT_MS } from './source-control-generation-limits' import { cancelGenerateCommitMessageLocal } from './commit-message-text-generation' import { spawnSourceControlAgent } from './source-control-agent-launch' import { killSourceControlAgentProcess } from './source-control-local-process' @@ -43,8 +44,32 @@ function fakeSupervisor(exitsOn: NodeJS.Signals | null): FakeSupervisor { return child } +// Behaves as the supervisor does: a SIGTERM ends it, after the time its provider takes to stop. +function fakeSupervisedAgent(stopMs: number): FakeSupervisor & { + stdout: EventEmitter + stderr: EventEmitter + stdin: { on: () => void; end: () => void } +} { + const child = Object.assign(fakeSupervisor(null), { + stdout: new EventEmitter(), + stderr: new EventEmitter(), + stdin: { on: vi.fn(), end: vi.fn() } + }) + child.kill.mockImplementation((signal?: NodeJS.Signals) => { + if (signal === 'SIGTERM') { + setTimeout(() => { + child.signalCode = 'SIGTERM' + child.emit('exit', null, 'SIGTERM') + child.emit('close', null, 'SIGTERM') + }, stopMs) + } + return true + }) + return child +} + function asSpawned(child: FakeSupervisor): SpawnedSourceControlAgentProcess { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the stop reads only pid, exit state, kill and the exit event, which the fake implements. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the stop and the generation read only pid, exit state, kill, the stdio streams and the child events, which the fakes implement. return child as unknown as SpawnedSourceControlAgentProcess } @@ -100,6 +125,42 @@ describe('killSourceControlAgentProcess for a supervised agent', () => { }) }) +describe('a supervised agent one-shot that runs out of time', () => { + it('times out at once but holds the Codex home until the supervisor has stopped', async () => { + vi.useFakeTimers() + const stopMs = 2_000 + const agents = [fakeSupervisedAgent(stopMs), fakeSupervisedAgent(stopMs)] + const spawnAgent = vi.fn(() => asSpawned(agents[spawnAgent.mock.calls.length - 1]!)) + const request = (cwd: string): ReturnType => + generateCommitMessage({ + context: { branch: 'main', stagedSummary: 'M README.md', stagedPatch: '+test' }, + params: { agentId: 'codex', model: 'gpt-5.5' }, + target: { kind: 'local', cwd, env: { CODEX_HOME: '/codex/timeout-home' } }, + spawnAgent + }) + + const first = request('/repo/first') + await vi.advanceTimersByTimeAsync(SOURCE_CONTROL_GENERATION_TIMEOUT_MS) + await expect(first).resolves.toMatchObject({ + success: false, + error: expect.stringMatching(/timed out/) + }) + expect(agents[0]!.kill.mock.calls).toEqual([['SIGTERM']]) + + const second = request('/repo/second') + await vi.advanceTimersByTimeAsync(stopMs - 1) + expect(spawnAgent).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + expect(spawnAgent).toHaveBeenCalledTimes(2) + + agents[1]!.stdout.emit('data', Buffer.from('Update README\n')) + agents[1]!.emit('close', 0) + await expect(second).resolves.toMatchObject({ success: true, message: 'Update README' }) + expect(agents[0]!.kill).not.toHaveBeenCalledWith('SIGKILL') + expect(terminateTreeMock).not.toHaveBeenCalled() + }) +}) + describe.runIf(process.platform !== 'win32')( 'killSourceControlAgentProcess on a real agent', () => { @@ -201,5 +262,49 @@ setInterval(() => {}, 60000) rmSync(folder, { recursive: true, force: true }) } }) + + it('stops an agent through its supervisor once its output passes the limit', async () => { + const folder = mkdtempSync(join(tmpdir(), 'orca-supervised-output-limit-')) + const pids: number[] = [] + try { + const pidFile = join(folder, 'agent-pid') + const agent = join(folder, 'floods.cjs') + // Floods stdout past the limit, then waits on its never-answered request. + writeFileSync( + agent, + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)) +process.stdin.resume() +process.stdout.write('x'.repeat(5 * 1024 * 1024)) +setInterval(() => {}, 60000) +` + ) + + await expect( + generateCommitMessage({ + context: { branch: 'main', stagedSummary: 'M README.md', stagedPatch: '+test' }, + params: { + agentId: 'custom', + model: '', + customAgentCommand: `"${process.execPath}" "${agent}"` + }, + target: { kind: 'local', cwd: folder, env: process.env }, + spawnAgent: spawnSourceControlAgent + }) + ).resolves.toMatchObject({ + success: false, + error: expect.stringMatching(/too much output/) + }) + pids.push(Number(readFileSync(pidFile, 'utf8'))) + expect(isAlive(pids[0]!)).toBe(false) + expect(terminateTreeMock).not.toHaveBeenCalled() + } finally { + for (const pid of pids) { + if (isAlive(pid)) { + process.kill(pid, 'SIGKILL') + } + } + rmSync(folder, { recursive: true, force: true }) + } + }) } ) From 5cd8c8ea70e32f36ada594eda903075d7fc14db9 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:39:50 -0700 Subject: [PATCH 08/38] fix(codex): run short-lived app-server sessions under the provider supervisor on POSIX The Codex model-list probe, the hook trust grant and the session index heal each start a short-lived `codex app-server` as a plain child of Orca, and counted on it exiting when its input closes. A wedged Codex (a cold model/list waiting on the network, say) kept running after Orca quit, crashed or was killed. These sessions now run under the provider supervisor that native chat's Codex connection already uses, in its session lifetime, so Orca's exit stops the server's whole process group. The session's end and its deadline share the one stop ladder: end its input (and SIGTERM a session past its deadline), give the supervisor its full stop time, and only then tear the tree down. A missing binary reaches Orca as the supervisor's exit 127 and is mapped back to the spawn error, so the trust-grant telemetry still reads it as a missing binary. Windows and WSL keep spawning the server directly, with the same timings as before. The supervisor now takes only the command line it starts, so the CLI build, which also runs these sessions, no longer pulls in the native chat connection's types. --- config/tsconfig.cli.json | 17 ++ .../codex-app-server-posix-supervisor.ts | 12 +- ...dex-app-server-session-supervision.test.ts | 170 ++++++++++++++++++ src/main/codex/codex-app-server-session.ts | 79 +++++--- .../supervised-probe-owner.test-fixture.ts | 130 ++++++++++++++ 5 files changed, 385 insertions(+), 23 deletions(-) create mode 100644 src/main/codex/codex-app-server-session-supervision.test.ts create mode 100644 src/main/codex/supervised-probe-owner.test-fixture.ts diff --git a/config/tsconfig.cli.json b/config/tsconfig.cli.json index 6afcf4f142f8..ce04eeeaa8bb 100644 --- a/config/tsconfig.cli.json +++ b/config/tsconfig.cli.json @@ -85,6 +85,21 @@ "../src/main/windows-native-registry.ts", "../src/main/windows/windows-command-line-recovery-health.ts", "../src/main/windows/windows-process-table.ts", + "../src/main/windows/windows-pty-job.ts", + "../src/main/crash-reporting/crash-breadcrumb-store.ts", + "../src/main/crash-reporting/durable-crash-breadcrumb.ts", + "../src/main/crash-reporting/main-process-lifecycle-identity.ts", + "../src/main/crash-reporting/self-initiated-tree-kill-log.ts", + "../src/main/observability/redactor.ts", + "../src/main/observability/tracer.ts", + "../src/main/orca-chromium-process-pids.ts", + "../src/main/own-chromium-tree-kill-guard.ts", + "../src/main/providers/windows-foreground-process-rows.ts", + "../src/main/pty-descendant-exit-verification.ts", + "../src/main/pty-descendant-termination.ts", + "../src/main/pty-process-table-parser.ts", + "../src/main/windows-process-tree-kill.ts", + "../src/main/windows-pty-root-identity.ts", "../src/main/windows/windows-process-table-cim-scan.ts", "../src/main/daemon/daemon-process-start-time.ts", "../src/main/daemon/daemon-process-identity-query.ts", @@ -117,6 +132,8 @@ "../src/main/codex/codex-app-server-capability-cache.ts", "../src/main/codex/codex-app-server-capability-signal.ts", "../src/main/codex/codex-app-server-client.ts", + "../src/main/codex/codex-app-server-posix-supervisor.ts", + "../src/main/codex/codex-app-server-process-teardown.ts", "../src/main/codex/codex-app-server-process-tree-kill.ts", "../src/main/codex/codex-app-server-record-reader.ts", "../src/main/codex/codex-app-server-session.ts", diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index 6d8d87c898b0..af33a1b4f5f8 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -1,4 +1,3 @@ -import type { CodexAppServerLaunch } from './codex-app-server-connection' import { waitForProcessExitUntil } from './codex-process-exit-deadline' /** Time the provider gets to exit on its own after its stdin ends, before SIGTERM. */ @@ -197,6 +196,13 @@ export async function stopSupervisedProvider(input: ProviderStopInput): Promise< return true } +/** The provider command line a supervisor starts. */ +export type ProviderLaunch = { + command: string + args: string[] + cwd?: string +} + export type ProviderSupervisorOptions = { cwd?: string lifetime?: ProviderSupervisorLifetime @@ -214,7 +220,7 @@ function assertGraceWithin(name: string, graceMs: number, maxMs: number): void { } export function supervisedPosixLaunch( - launch: CodexAppServerLaunch, + launch: ProviderLaunch, childEnv: NodeJS.ProcessEnv, { cwd = launch.cwd ?? process.cwd(), @@ -250,7 +256,7 @@ export function supervisedPosixLaunch( } export function createProviderSpawnSpec( - launch: CodexAppServerLaunch, + launch: ProviderLaunch, childEnv: NodeJS.ProcessEnv, platform: NodeJS.Platform, { lifetime }: Pick = {} diff --git a/src/main/codex/codex-app-server-session-supervision.test.ts b/src/main/codex/codex-app-server-session-supervision.test.ts new file mode 100644 index 000000000000..79c4d7c1c829 --- /dev/null +++ b/src/main/codex/codex-app-server-session-supervision.test.ts @@ -0,0 +1,170 @@ +import { existsSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from './codex-app-server-posix-supervisor' +import type { CodexAppServerSpawn } from './codex-app-server-process-tree-kill' +import { CodexAppServerTimeoutError, runCodexAppServerSession } from './codex-app-server-session' +import { classifyCodexTrustGrantError } from './codex-trust-grant-telemetry' +import { + alive, + createSupervisedProbeRig, + waitFor, + type SupervisedProbeRig +} from './supervised-probe-owner.test-fixture' + +// Answers initialize, then nothing; it ignores its stdin end, as a wedged Codex can. +const WEDGED_APP_SERVER = String.raw` +require('node:readline').createInterface({ input: process.stdin }).on('line', (line) => { + const message = JSON.parse(line) + if (message.method === 'initialize') { + process.stdout.write(JSON.stringify({ id: message.id, result: {} }) + '\n') + } +}) +` + +const OWNER = String.raw` +import { runCodexAppServerSession } from './codex-app-server-session' +void runCodexAppServerSession( + { command: process.env.ORCA_TEST_STAND_IN!, cliPath: null, args: ['app-server'], timeoutMs: 120_000 }, + () => new Promise(() => {}) +).catch(() => {}) +setInterval(() => {}, 60_000) +` + +const originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform') +let rig: SupervisedProbeRig | null = null + +afterEach(() => { + if (originalPlatform) { + Object.defineProperty(process, 'platform', originalPlatform) + } + rig?.cleanup() + rig = null +}) + +type CapturedSpawn = { program: string; args: string[]; options: Record } + +function captureSpawn(): { spawnImpl: CodexAppServerSpawn; calls: CapturedSpawn[] } { + const calls: CapturedSpawn[] = [] + return { + calls, + spawnImpl: (program, args, options) => { + calls.push({ program, args, options }) + throw new Error('spawn captured') + } + } +} + +function decodedSupervisorSpec(env: unknown): unknown { + const encoded = + typeof env === 'object' && env !== null && 'ORCA_PROVIDER_SUPERVISOR_SPEC' in env + ? String(env.ORCA_PROVIDER_SUPERVISOR_SPEC) + : '' + return JSON.parse(Buffer.from(encoded, 'base64').toString() || 'null') +} + +describe('short-lived Codex app-server session spawn', () => { + it('runs a POSIX session under the provider supervisor in session lifetime', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'darwin' }) + const { spawnImpl, calls } = captureSpawn() + + await expect( + runCodexAppServerSession( + { command: '/bin/codex', cliPath: null, args: ['app-server'], timeoutMs: 1_000 }, + async () => null, + spawnImpl + ) + ).rejects.toThrow('spawn captured') + + const [{ program, args, options }] = calls + expect(program).toBe(process.execPath) + expect(args.slice(args.indexOf('--') + 1)).toEqual(['/bin/codex', 'app-server']) + expect(options.detached).toBe(true) + expect(decodedSupervisorSpec(options.env)).toMatchObject({ + lifetime: 'session', + ownerPid: process.pid + }) + }) + + it('spawns a Windows session directly, as before', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const { spawnImpl, calls } = captureSpawn() + + await expect( + runCodexAppServerSession( + { command: 'C:\\codex.exe', cliPath: null, args: ['app-server'], timeoutMs: 1_000 }, + async () => null, + spawnImpl + ) + ).rejects.toThrow('spawn captured') + + const [{ program, args, options }] = calls + expect(decodedSupervisorSpec(options.env)).toBeNull() + expect(program).toBe('C:\\codex.exe') + expect(args).toEqual(['app-server']) + expect(options).not.toHaveProperty('detached') + }) +}) + +describe.runIf(process.platform !== 'win32')('supervised Codex app-server sessions', () => { + it('stops a wedged session at its deadline with SIGTERM to its whole group', async () => { + rig = createSupervisedProbeRig() + const standIn = rig.writeStandIn('codex', WEDGED_APP_SERVER) + + const readPids = rig.readPids + const reported: { pids?: { provider: number; grandchild: number } } = {} + // The deadline must find the stand-in armed, however slowly a loaded host starts it. + const session = runCodexAppServerSession( + { command: standIn, cliPath: null, args: ['app-server'], env: rig.env, timeoutMs: 4_000 }, + async () => { + reported.pids = await readPids() + return new Promise(() => {}) + } + ) + + await expect(session).rejects.toBeInstanceOf(CodexAppServerTimeoutError) + const { pids } = reported + expect(pids).toBeDefined() + if (!pids) { + return + } + // Graceful first: the old deadline SIGKILLed the root, which a Codex mid-write cannot survive. + expect(existsSync(rig.signalFile) && readFileSync(rig.signalFile, 'utf8')).toBe('SIGTERM') + expect(alive(pids.provider)).toBe(false) + // The group ladder reaches a descendant that ignores SIGTERM; a root-only kill misses it. + expect(alive(pids.grandchild)).toBe(false) + }) + + it('stops the session group when its owner is SIGKILLed mid-session', async () => { + rig = createSupervisedProbeRig() + const standIn = rig.writeStandIn('codex', WEDGED_APP_SERVER) + const bundle = await rig.bundleOwner(OWNER, __dirname) + const owner = rig.launchOwner(bundle, { ORCA_TEST_STAND_IN: standIn }) + const pids = await rig.readPids() + + owner.kill('SIGKILL') + + expect( + await waitFor( + () => !alive(pids.provider) && !alive(pids.grandchild), + PROVIDER_SUPERVISOR_MAX_STOP_MS + 1_000 + ) + ).toBe(true) + }) + + it('reports a missing Codex binary as the spawn error, not an early exit', async () => { + rig = createSupervisedProbeRig() + const error = await runCodexAppServerSession( + { + command: join(rig.dir, 'missing', 'codex'), + cliPath: null, + args: ['app-server'], + timeoutMs: 5_000 + }, + async () => null + ).catch((caught: unknown) => caught) + + expect(error).toMatchObject({ code: 'ENOENT' }) + expect(classifyCodexTrustGrantError(error)).toBe('binary-missing') + }) +}) diff --git a/src/main/codex/codex-app-server-session.ts b/src/main/codex/codex-app-server-session.ts index 389cd20516cb..5d140ff9e859 100644 --- a/src/main/codex/codex-app-server-session.ts +++ b/src/main/codex/codex-app-server-session.ts @@ -2,6 +2,12 @@ import type { ChildProcessWithoutNullStreams } from 'node:child_process' import { waitForProcessExitUntil } from './codex-process-exit-deadline' import { stderrIndicatesMissingAppServer } from './codex-app-server-capability-signal' import { withCliRuntimeOnPath } from '../../shared/node-cli-command-resolution' +import { + createProviderSpawnSpec, + stopSupervisedProvider, + supervisedProviderSpawnError +} from './codex-app-server-posix-supervisor' +import { terminateCodexAppServerProcessTree } from './codex-app-server-process-teardown' import { killCodexAppServerProcessTree, spawnCodexAppServerProcess, @@ -33,7 +39,7 @@ export type CodexAppServerInvocation = { /** Env keys stripped from the inherited environment before spawn (e.g. an * inherited CODEX_HOME, so a default-home grant runs against the real ~/.codex). */ envToDelete?: readonly string[] - /** Whole-session deadline. The codex child is SIGKILLed when it lapses. */ + /** Whole-session deadline. The codex child is stopped when it lapses. */ timeoutMs: number } @@ -93,7 +99,8 @@ export function isCodexMethodNotFoundError(error: unknown): boolean { /** * Runs one short-lived `codex app-server` session over stdio JSON-RPC (JSONL): * spawn → initialize → initialized → body(rpc) → EOF/reap. The child is reaped - * on every path; the session deadline SIGKILLs it. + * on every path; the session deadline stops it. On POSIX it runs under the provider + * supervisor, so an Orca that quits or dies mid-session still stops its group. */ export async function runCodexAppServerSession( invocation: CodexAppServerInvocation, @@ -109,14 +116,23 @@ export async function runCodexAppServerSession( const pairedEnv = invocation.cliPath ? withCliRuntimeOnPath(invocation.cliPath, childEnv) : childEnv - const child = spawnImpl(invocation.command, invocation.args, { - env: pairedEnv, + const spawnSpec = createProviderSpawnSpec( + { command: invocation.command, args: invocation.args }, + pairedEnv, + process.platform, + { lifetime: 'session' } + ) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: every stdio slot is 'pipe', so stdin, stdout and stderr exist. + const child = spawnImpl(spawnSpec.program, spawnSpec.args, { + env: spawnSpec.env, stdio: ['pipe', 'pipe', 'pipe'], - windowsHide: true + windowsHide: true, + ...(spawnSpec.supervised ? { detached: true } : {}) }) as ChildProcessWithoutNullStreams let stderrTail = '' let exited = false + let exitCode: number | null = null let nextRequestId = 1 let timedOut = false const pending = new Map< @@ -125,8 +141,9 @@ export async function runCodexAppServerSession( >() const exitPromise = new Promise((resolve) => { - child.on('exit', () => { + child.on('exit', (code: number | null) => { exited = true + exitCode = code resolve() }) }) @@ -190,7 +207,10 @@ export async function runCodexAppServerSession( const error = new CodexAppServerTimeoutError( `codex app-server session exceeded ${invocation.timeoutMs}ms (${invocation.command})` ) - killCodexAppServerProcessTree(child) + // A supervised child is stopped by the finally below, which this rejection runs at once. + if (!spawnSpec.supervised) { + killCodexAppServerProcessTree(child) + } failPending(error) rejectDeadline(error) }, invocation.timeoutMs) @@ -263,6 +283,13 @@ export async function runCodexAppServerSession( } function buildEarlyExitError(): Error { + // A supervisor reports a missing binary as exit 127; callers classify the spawn error. + const providerSpawnError = spawnSpec.supervised + ? supervisedProviderSpawnError(exitCode, stderrTail) + : null + if (providerSpawnError) { + return providerSpawnError + } if (stderrIndicatesMissingAppServer(stderrTail)) { return new CodexAppServerUnsupportedError( `codex CLI does not support the app-server subcommand: ${stderrTail.trim().slice(0, 400)}` @@ -297,20 +324,32 @@ export async function runCodexAppServerSession( } throw error } finally { - try { - child.stdin.end() - } catch { - // stdin may already be destroyed after a kill; reaping below still runs. - } - if (!exited) { - // Why: the server exits promptly on stdin EOF; the grace period only - // bounds a wedged child before the guaranteed SIGKILL reap. - await waitForProcessExitUntil(exitPromise, 1500) - if (!exited) { - killCodexAppServerProcessTree(child) + await stopSupervisedProvider({ + request: () => { + try { + child.stdin.end() + } catch { + // stdin may already be destroyed after a kill; reaping below still runs. + } + // A session past its deadline is wedged; its stdin end would only add a grace. + if (timedOut && spawnSpec.supervised && !exited) { + child.kill('SIGTERM') + } + }, + exitPromise, + exited: () => exited, + force: async () => { + if (spawnSpec.supervised) { + await terminateCodexAppServerProcessTree(child) + } else { + killCodexAppServerProcessTree(child) + } await waitForProcessExitUntil(exitPromise, 1000) - } - } + }, + supervised: spawnSpec.supervised, + // Why: the server exits promptly on stdin EOF; the wait only bounds a wedged child. + directWaitMs: 1500 + }) clearTimeout(deadline) } } diff --git a/src/main/codex/supervised-probe-owner.test-fixture.ts b/src/main/codex/supervised-probe-owner.test-fixture.ts new file mode 100644 index 000000000000..5d3a75827732 --- /dev/null +++ b/src/main/codex/supervised-probe-owner.test-fixture.ts @@ -0,0 +1,130 @@ +import { spawn, type ChildProcess } from 'node:child_process' +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { build } from 'esbuild' + +// A stand-in agent CLI that, like `opencode serve` or a wedged Codex, ignores its stdin end. It +// leaves a SIGTERM-ignoring grandchild in its group, so only a group SIGKILL ends everything, and +// records a SIGTERM before exiting on it. `body` adds the caller's protocol. +function standInSource(body: string): string { + return String.raw` +const { spawn } = require('node:child_process') +const fs = require('node:fs') +process.on('SIGTERM', () => { + fs.writeFileSync(process.env.ORCA_TEST_SIGNAL_FILE, 'SIGTERM') + process.exit(0) +}) +process.stdin.on('error', () => {}).resume() +const grandchild = spawn( + process.execPath, + ['-e', "process.on('SIGTERM', () => {}); process.stdout.write('armed'); setInterval(() => {}, 60000)"], + { stdio: ['ignore', 'pipe', 'ignore'] } +) +grandchild.stdout.once('data', () => { + fs.writeFileSync( + process.env.ORCA_TEST_PID_FILE, + JSON.stringify({ provider: process.pid, grandchild: grandchild.pid }) + ) +}) +setInterval(() => {}, 60000) +${body} +` +} + +export type SupervisedProbeRig = { + dir: string + pidFile: string + signalFile: string + env: Record + /** An executable stand-in, run as ` ` like a real agent CLI. */ + writeStandIn: (name: string, body?: string) => string + /** Bundles a TypeScript owner entry, resolved from `resolveDir`, into one plain-Node file. */ + bundleOwner: (source: string, resolveDir: string) => Promise + launchOwner: (bundle: string, env?: Record) => ChildProcess + readPids: (timeoutMs?: number) => Promise<{ provider: number; grandchild: number }> + cleanup: () => void +} + +export function alive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch (error) { + return !(error instanceof Error && 'code' in error && error.code === 'ESRCH') + } +} + +export async function waitFor(predicate: () => boolean, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs + while (!predicate()) { + if (Date.now() >= deadline) { + return false + } + await new Promise((resolve) => setTimeout(resolve, 25)) + } + return true +} + +export function createSupervisedProbeRig(): SupervisedProbeRig { + const dir = mkdtempSync(join(tmpdir(), 'orca-supervised-probe-')) + const pidFile = join(dir, 'pids.json') + const signalFile = join(dir, 'signal') + const recordedPids = new Set() + const env = { ORCA_TEST_PID_FILE: pidFile, ORCA_TEST_SIGNAL_FILE: signalFile } + return { + dir, + pidFile, + signalFile, + env, + writeStandIn: (name, body = '') => { + const script = join(dir, `${name}.cjs`) + writeFileSync(script, standInSource(body)) + const launcher = join(dir, name) + writeFileSync(launcher, `#!/bin/sh\nexec "${process.execPath}" "${script}" "$@"\n`) + chmodSync(launcher, 0o755) + return launcher + }, + bundleOwner: async (source, resolveDir) => { + const outfile = join(dir, 'owner.cjs') + await build({ + stdin: { contents: source, resolveDir, loader: 'ts', sourcefile: 'owner.ts' }, + bundle: true, + platform: 'node', + format: 'cjs', + target: 'node22', + outfile, + logLevel: 'silent' + }) + return outfile + }, + launchOwner: (bundle, ownerEnv = {}) => { + const owner = spawn(process.execPath, [bundle], { + env: { ...process.env, ...env, ...ownerEnv }, + stdio: 'ignore' + }) + recordedPids.add(owner.pid!) + return owner + }, + readPids: async (timeoutMs = 10_000) => { + if (!(await waitFor(() => existsSync(pidFile), timeoutMs))) { + throw new Error('the stand-in never reported its pids') + } + const pids: { provider: number; grandchild: number } = JSON.parse( + readFileSync(pidFile, 'utf8') + ) + recordedPids.add(pids.provider) + recordedPids.add(pids.grandchild) + return pids + }, + cleanup: () => { + // Only pids this rig recorded, so a failed run never leaves a stand-in behind. + for (const pid of recordedPids) { + if (alive(pid)) { + process.kill(pid, 'SIGKILL') + } + } + rmSync(dir, { recursive: true, force: true }) + } + } +} From a3d89b4f80697d6ff9c05c41d060a298cdc264a4 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:40:08 -0700 Subject: [PATCH 09/38] refactor(supervisor): share the stop of a supervised child process Agent one-shots stop their supervisor with SIGTERM through the shared stop ladder, watching the child's own exit. That adapter moves into one helper so the Codex backfill recovery can use it with its own stop request, instead of a copy. --- .../codex/supervised-child-process-stop.ts | 32 +++++++++++++++++++ .../source-control-local-process.ts | 29 ++--------------- 2 files changed, 35 insertions(+), 26 deletions(-) create mode 100644 src/main/codex/supervised-child-process-stop.ts diff --git a/src/main/codex/supervised-child-process-stop.ts b/src/main/codex/supervised-child-process-stop.ts new file mode 100644 index 000000000000..335219bd8952 --- /dev/null +++ b/src/main/codex/supervised-child-process-stop.ts @@ -0,0 +1,32 @@ +import type { ChildProcessHandle } from '../../shared/child-process/process-spec' +import { stopSupervisedProvider } from './codex-app-server-posix-supervisor' +import { terminateCodexAppServerProcessTree } from './codex-app-server-process-teardown' + +type SupervisedChild = Pick + +/** + * Stops a provider supervisor child: `request` asks it to stop (SIGTERM unless given), and its + * tree is forced only after the supervisor's full stop time. True when it had to force. + */ +export async function stopSupervisedChildProcess( + child: SupervisedChild, + request: () => void = () => child.kill('SIGTERM') +): Promise { + const exited = (): boolean => child.exitCode !== null || child.signalCode !== null + if (exited()) { + return false + } + return stopSupervisedProvider({ + request: () => { + try { + request() + } catch { + // The process may exit between the exit check and the request. + } + }, + exitPromise: new Promise((resolve) => child.once('exit', () => resolve())), + exited, + force: () => terminateCodexAppServerProcessTree(child), + supervised: true + }) +} diff --git a/src/main/text-generation/source-control-local-process.ts b/src/main/text-generation/source-control-local-process.ts index 9f0b2cbd5a08..7a881b225c58 100644 --- a/src/main/text-generation/source-control-local-process.ts +++ b/src/main/text-generation/source-control-local-process.ts @@ -1,9 +1,6 @@ import type { CommitMessagePlan } from '../../shared/commit-message-plan' -import { - stopSupervisedProvider, - supervisedProviderSpawnError -} from '../codex/codex-app-server-posix-supervisor' -import { terminateCodexAppServerProcessTree } from '../codex/codex-app-server-process-teardown' +import { supervisedProviderSpawnError } from '../codex/codex-app-server-posix-supervisor' +import { stopSupervisedChildProcess } from '../codex/supervised-child-process-stop' import { UnsafeWindowsBatchArgumentsError } from '../win32-utils' import { terminateWindowsProcessTree } from '../windows-process-tree-kill' import { @@ -35,7 +32,7 @@ export async function killSourceControlAgentProcess( return } if (child.supervised) { - await stopSupervisedAgent(child) + await stopSupervisedChildProcess(child) return } if (process.platform === 'win32') { @@ -52,26 +49,6 @@ export async function killSourceControlAgentProcess( } } -async function stopSupervisedAgent(child: SpawnedSourceControlAgentProcess): Promise { - const exited = (): boolean => child.exitCode !== null || child.signalCode !== null - if (exited()) { - return - } - await stopSupervisedProvider({ - request: () => { - try { - child.kill('SIGTERM') - } catch { - // The process may exit between the exit check and kill. - } - }, - exitPromise: new Promise((resolve) => child.once('exit', () => resolve())), - exited, - force: () => terminateCodexAppServerProcessTree(child), - supervised: true - }) -} - // Why: Windows caps the CreateProcess command line at 32,767 UTF-16 code units, // including the executable path, per-arg quoting, and separators. The budget // leaves headroom for cmd.exe `/d /c` shim wrappers. From 924c2f326c474f7d60cc88cb444c65467529acd2 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:40:08 -0700 Subject: [PATCH 10/38] fix(codex): supervise the app-server that keeps a Codex index backfill alive While Codex rebuilds its session index, Orca keeps a read-only `codex app-server` running for up to an hour so Codex can finish. It was a plain child of Orca with its input held open, so a quit, crash or kill left it running. On POSIX it now runs under the provider supervisor in its session lifetime, so Orca's exit stops its whole process group. Stopping it (done, aborted, or given up) ends its input, as a Codex connection close does; the supervisor then SIGTERMs the group and SIGKILLs it after the grace, and the tree is torn down only if the supervisor outlives its full stop time. Windows and WSL keep the direct spawn and the drain-first probe termination. The spawn and stop of that process move into their own module. --- ...odex-state-db-backfill-recovery-process.ts | 64 ++++++++ ...e-db-backfill-recovery-supervision.test.ts | 137 ++++++++++++++++++ .../codex-state-db-backfill-recovery.test.ts | 10 +- .../codex/codex-state-db-backfill-recovery.ts | 58 +++----- 4 files changed, 225 insertions(+), 44 deletions(-) create mode 100644 src/main/codex/codex-state-db-backfill-recovery-process.ts create mode 100644 src/main/codex/codex-state-db-backfill-recovery-supervision.test.ts diff --git a/src/main/codex/codex-state-db-backfill-recovery-process.ts b/src/main/codex/codex-state-db-backfill-recovery-process.ts new file mode 100644 index 000000000000..ac9aa0899a09 --- /dev/null +++ b/src/main/codex/codex-state-db-backfill-recovery-process.ts @@ -0,0 +1,64 @@ +import type { ChildProcessHandle } from '../../shared/child-process/process-spec' +import { withCliRuntimeOnPath } from '../../shared/node-cli-command-resolution' +import { parseWslUncPath } from '../../shared/wsl-paths' +import { buildWslCodexAppServerArgs } from '../codex-accounts/wsl-codex-command' +import { CODEX_READ_ONLY_APP_SERVER_ARGS } from '../codex-cli/codex-read-only-app-server-args' +import { terminateCodexProbeChild } from '../rate-limits/codex-probe-termination' +import { createProviderSpawnSpec } from './codex-app-server-posix-supervisor' +import type { CodexAppServerSpawn } from './codex-app-server-process-tree-kill' +import { stopSupervisedChildProcess } from './supervised-child-process-stop' + +export type CodexBackfillRecoveryProcess = { child: ChildProcessHandle; supervised: boolean } + +/** Starts the read-only app-server whose presence lets Codex finish its own backfill. */ +export function spawnCodexBackfillRecoveryProcess( + codexHomePath: string, + spawnProcess: CodexAppServerSpawn, + resolveCommand: () => string +): CodexBackfillRecoveryProcess { + const wslHome = process.platform === 'win32' ? parseWslUncPath(codexHomePath) : null + if (wslHome) { + const child = spawnProcess( + 'wsl.exe', + buildWslCodexAppServerArgs( + wslHome.distro, + wslHome.linuxPath, + CODEX_READ_ONLY_APP_SERVER_ARGS + ), + { + stdio: ['pipe', 'ignore', 'ignore'], + windowsHide: true, + env: process.env + } + ) + return { child, supervised: false } + } + const command = resolveCommand() + // Kept alive up to an hour; on POSIX the supervisor stops it with Orca, however Orca exits. + const spawnSpec = createProviderSpawnSpec( + { command, args: [...CODEX_READ_ONLY_APP_SERVER_ARGS], cwd: codexHomePath }, + withCliRuntimeOnPath(command, { ...process.env, CODEX_HOME: codexHomePath }), + process.platform, + { lifetime: 'session' } + ) + const child = spawnProcess(spawnSpec.program, spawnSpec.args, { + cwd: codexHomePath, + stdio: ['pipe', 'ignore', 'ignore'], + windowsHide: true, + env: spawnSpec.env, + ...(spawnSpec.supervised ? { detached: true } : {}) + }) + return { child, supervised: spawnSpec.supervised } +} + +export async function stopCodexBackfillRecoveryProcess( + child: ChildProcessHandle, + supervised: boolean +): Promise { + if (supervised) { + // A session supervisor turns stdin end into its group stop, as a Codex connection close does. + await stopSupervisedChildProcess(child, () => child.stdin?.end()) + return + } + await terminateCodexProbeChild(child) +} diff --git a/src/main/codex/codex-state-db-backfill-recovery-supervision.test.ts b/src/main/codex/codex-state-db-backfill-recovery-supervision.test.ts new file mode 100644 index 000000000000..af5f58c1f2e7 --- /dev/null +++ b/src/main/codex/codex-state-db-backfill-recovery-supervision.test.ts @@ -0,0 +1,137 @@ +import { ChildProcess } from 'node:child_process' +import { existsSync, readFileSync } from 'node:fs' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { CODEX_READ_ONLY_APP_SERVER_ARGS } from '../codex-cli/codex-read-only-app-server-args' +import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from './codex-app-server-posix-supervisor' +import type { CodexAppServerSpawn } from './codex-app-server-process-tree-kill' +import type { CodexStateDbBackfillStatus } from './codex-state-db' +import { runCodexStateDbBackfillRecovery } from './codex-state-db-backfill-recovery' +import { + alive, + createSupervisedProbeRig, + waitFor, + type SupervisedProbeRig +} from './supervised-probe-owner.test-fixture' + +const INCOMPLETE: CodexStateDbBackfillStatus = { + kind: 'incomplete', + stateDbPath: '/state.sqlite', + status: 'running' +} + +const OWNER = String.raw` +import { runCodexStateDbBackfillRecovery } from './codex-state-db-backfill-recovery' +void runCodexStateDbBackfillRecovery(process.env.ORCA_TEST_CODEX_HOME!, new AbortController().signal, { + resolveCommand: () => process.env.ORCA_TEST_STAND_IN!, + readStatus: () => ({ kind: 'incomplete', stateDbPath: '/state.sqlite', status: 'running' }) +}).catch(() => {}) +setInterval(() => {}, 60_000) +` + +const originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform') +let rig: SupervisedProbeRig | null = null + +afterEach(() => { + if (originalPlatform) { + Object.defineProperty(process, 'platform', originalPlatform) + } + vi.unstubAllEnvs() + rig?.cleanup() + rig = null +}) + +function decodedSupervisorSpec(env: unknown): unknown { + const encoded = + typeof env === 'object' && env !== null && 'ORCA_PROVIDER_SUPERVISOR_SPEC' in env + ? String(env.ORCA_PROVIDER_SUPERVISOR_SPEC) + : '' + return JSON.parse(Buffer.from(encoded, 'base64').toString() || 'null') +} + +describe('Codex backfill recovery app-server supervision', () => { + it('runs the POSIX recovery app-server under a session-lifetime supervisor', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'linux' }) + // Never signalled: terminate is a stub here, and a handle-less ChildProcess must not be killed. + const child = new ChildProcess() + const calls: { program: string; args: string[]; options: Record }[] = [] + const spawnProcess: CodexAppServerSpawn = (program, args, options) => { + calls.push({ program, args, options }) + return child + } + const terminate = vi.fn(async () => {}) + const readStatus = vi + .fn() + .mockReturnValueOnce(INCOMPLETE) + .mockReturnValue({ kind: 'complete', stateDbPath: '/state.sqlite' }) + + await expect( + runCodexStateDbBackfillRecovery('/managed-home', new AbortController().signal, { + spawnProcess, + resolveCommand: () => '/bin/codex', + readStatus, + terminate, + sleep: vi.fn(async () => {}), + now: vi.fn(() => 1_000) + }) + ).resolves.toEqual({ outcome: 'completed', spawnCount: 1 }) + + const [{ program, args, options }] = calls + expect(program).toBe(process.execPath) + expect(args.slice(args.indexOf('--') + 1)).toEqual([ + '/bin/codex', + ...CODEX_READ_ONLY_APP_SERVER_ARGS + ]) + expect(options).toMatchObject({ cwd: '/managed-home', detached: true }) + expect(options.env).toMatchObject({ CODEX_HOME: '/managed-home' }) + expect(decodedSupervisorSpec(options.env)).toMatchObject({ + lifetime: 'session', + cwd: '/managed-home' + }) + expect(terminate).toHaveBeenCalledWith(child, true) + }) +}) + +describe.runIf(process.platform !== 'win32')('supervised Codex backfill recovery processes', () => { + it('stops the whole recovery group when the recovery is aborted', async () => { + rig = createSupervisedProbeRig() + for (const [key, value] of Object.entries(rig.env)) { + vi.stubEnv(key, value) + } + const standIn = rig.writeStandIn('codex') + const controller = new AbortController() + + const recovery = runCodexStateDbBackfillRecovery(rig.dir, controller.signal, { + resolveCommand: () => standIn, + readStatus: () => INCOMPLETE + }) + const pids = await rig.readPids() + controller.abort() + + await expect(recovery).resolves.toEqual({ outcome: 'stopped', spawnCount: 1 }) + // Its stdin end comes first; the supervisor then SIGTERMs, and SIGKILLs the group after. + expect(existsSync(rig.signalFile) && readFileSync(rig.signalFile, 'utf8')).toBe('SIGTERM') + expect(alive(pids.provider)).toBe(false) + // A SIGTERM to the root alone left this descendant running for good. + expect(alive(pids.grandchild)).toBe(false) + }) + + it('stops the recovery group when its owner is SIGKILLed', async () => { + rig = createSupervisedProbeRig() + const standIn = rig.writeStandIn('codex') + const bundle = await rig.bundleOwner(OWNER, __dirname) + const owner = rig.launchOwner(bundle, { + ORCA_TEST_STAND_IN: standIn, + ORCA_TEST_CODEX_HOME: rig.dir + }) + const pids = await rig.readPids() + + owner.kill('SIGKILL') + + expect( + await waitFor( + () => !alive(pids.provider) && !alive(pids.grandchild), + PROVIDER_SUPERVISOR_MAX_STOP_MS + 1_000 + ) + ).toBe(true) + }) +}) diff --git a/src/main/codex/codex-state-db-backfill-recovery.test.ts b/src/main/codex/codex-state-db-backfill-recovery.test.ts index a76c5f65a5a6..6577bf3bd53e 100644 --- a/src/main/codex/codex-state-db-backfill-recovery.test.ts +++ b/src/main/codex/codex-state-db-backfill-recovery.test.ts @@ -357,7 +357,7 @@ describe('Codex state DB backfill recovery', () => { expect(spawnProcess).toHaveBeenCalledTimes(2) expect(timerDurations).toEqual([5_000, 2_000, 5_000]) expect(terminate).toHaveBeenCalledTimes(1) - expect(terminate).toHaveBeenCalledWith(children[1]) + expect(terminate).toHaveBeenCalledWith(children[1], process.platform !== 'win32') }) it('keeps the successful app-server claimant alive until Codex marks its DB complete', async () => { @@ -377,7 +377,7 @@ describe('Codex state DB backfill recovery', () => { now: vi.fn(() => 1_000) }) ).resolves.toEqual({ outcome: 'completed', spawnCount: 1 }) - expect(terminate).toHaveBeenCalledWith(child) + expect(terminate).toHaveBeenCalledWith(child, process.platform !== 'win32') }) it('hands the Codex shim itself to the spawn chokepoint for native Windows recovery', async () => { @@ -389,6 +389,7 @@ describe('Codex state DB backfill recovery', () => { .fn() .mockReturnValueOnce({ kind: 'incomplete', stateDbPath: 'state.sqlite', status: 'running' }) .mockReturnValue({ kind: 'complete', stateDbPath: 'state.sqlite' }) + const terminate = vi.fn(async () => {}) await runCodexStateDbBackfillRecovery( 'C:\\Users\\alice\\.codex', @@ -397,7 +398,7 @@ describe('Codex state DB backfill recovery', () => { spawnProcess: spawnProcess as never, resolveCommand: () => codexCommand, readStatus, - terminate: vi.fn(async () => {}), + terminate, sleep: vi.fn(async () => {}), now: vi.fn(() => 1_000) } @@ -421,6 +422,9 @@ describe('Codex state DB backfill recovery', () => { ]) expect(spawnOptions.cwd).toBe('C:\\Users\\alice\\.codex') expect(spawnOptions.env?.CODEX_HOME).toBe('C:\\Users\\alice\\.codex') + // Windows has no supervisor: the drain-first probe termination stays its stop. + expect(spawnOptions).not.toHaveProperty('detached') + expect(terminate).toHaveBeenCalledWith(child, false) }) it('retries a live foreign lease until one durable claimant can recover it', async () => { diff --git a/src/main/codex/codex-state-db-backfill-recovery.ts b/src/main/codex/codex-state-db-backfill-recovery.ts index b4acef3b3375..85635bf0e6ab 100644 --- a/src/main/codex/codex-state-db-backfill-recovery.ts +++ b/src/main/codex/codex-state-db-backfill-recovery.ts @@ -5,16 +5,16 @@ import { normalizeRuntimePathForComparison } from '../../shared/cross-platform-p import { parseWslUncPath } from '../../shared/wsl-paths' import { withManagedHookInstallLock } from '../agent-hooks/managed-hook-install-lock' import { readManagedHookHostIdentity } from '../agent-hooks/managed-hook-owner-identity' -import { buildWslCodexAppServerArgs } from '../codex-accounts/wsl-codex-command' import { resolveCodexCommand } from '../codex-cli/command' -import { withCliRuntimeOnPath } from '../../shared/node-cli-command-resolution' -import { CODEX_READ_ONLY_APP_SERVER_ARGS } from '../codex-cli/codex-read-only-app-server-args' -import { terminateCodexProbeChild } from '../rate-limits/codex-probe-termination' import type { ChildProcessHandle } from '../../shared/child-process/process-spec' import { spawnCodexAppServerProcess, type CodexAppServerSpawn } from './codex-app-server-process-tree-kill' +import { + spawnCodexBackfillRecoveryProcess, + stopCodexBackfillRecoveryProcess +} from './codex-state-db-backfill-recovery-process' import { getOrcaUserDataPath } from './codex-home-paths' import { BACKFILL_PENDING_MIN_SESSION_FILES, @@ -43,7 +43,7 @@ type RecoveryDependencies = { countSessions: (sessionsRoot: string, limit: number) => number now: () => number sleep: (ms: number, signal: AbortSignal) => Promise - terminate: (child: ChildProcessHandle) => Promise + terminate: (child: ChildProcessHandle, supervised: boolean) => Promise } const defaultDependencies: RecoveryDependencies = { @@ -53,7 +53,7 @@ const defaultDependencies: RecoveryDependencies = { countSessions: countCodexSessionFilesUpTo, now: Date.now, sleep: async (ms, signal) => await delay(ms, undefined, { signal }), - terminate: async (child) => await terminateCodexProbeChild(child) + terminate: stopCodexBackfillRecoveryProcess } function finish( @@ -86,35 +86,6 @@ function initialRecoveryDecision( return null } -function spawnRecoveryProcess( - codexHomePath: string, - dependencies: RecoveryDependencies -): ChildProcessHandle { - const wslHome = process.platform === 'win32' ? parseWslUncPath(codexHomePath) : null - if (wslHome) { - return dependencies.spawnProcess( - 'wsl.exe', - buildWslCodexAppServerArgs( - wslHome.distro, - wslHome.linuxPath, - CODEX_READ_ONLY_APP_SERVER_ARGS - ), - { - stdio: ['pipe', 'ignore', 'ignore'], - windowsHide: true, - env: process.env - } - ) - } - const command = dependencies.resolveCommand() - return dependencies.spawnProcess(command, [...CODEX_READ_ONLY_APP_SERVER_ARGS], { - cwd: codexHomePath, - stdio: ['pipe', 'ignore', 'ignore'], - windowsHide: true, - env: withCliRuntimeOnPath(command, { ...process.env, CODEX_HOME: codexHomePath }) - }) -} - /** Keeps a sanctioned app-server claimant alive until Codex completes its own backfill. */ export async function runCodexStateDbBackfillRecovery( codexHomePath: string, @@ -130,7 +101,12 @@ export async function runCodexStateDbBackfillRecovery( const deadline = dependencies.now() + RECOVERY_MAX_TOTAL_MS let spawnCount = 0 while (!signal.aborted && dependencies.now() < deadline && spawnCount < RECOVERY_MAX_SPAWNS) { - const child = spawnRecoveryProcess(codexHomePath, dependencies) + const { child, supervised } = spawnCodexBackfillRecoveryProcess( + codexHomePath, + dependencies.spawnProcess, + dependencies.resolveCommand + ) + const terminate = (): Promise => dependencies.terminate(child, supervised) spawnCount += 1 let childDown = false child.once('error', () => { @@ -145,27 +121,27 @@ export async function runCodexStateDbBackfillRecovery( await dependencies.sleep(RECOVERY_POLL_INTERVAL_MS, signal) const status = dependencies.readStatus(codexHomePath) if (status.kind === 'complete') { - await dependencies.terminate(child) + await terminate() return finish('completed', spawnCount) } if (status.kind === 'unreadable') { - await dependencies.terminate(child) + await terminate() return finish('unreadable', spawnCount) } } } catch (error) { if (!signal.aborted) { - await dependencies.terminate(child) + await terminate() throw error } } if (signal.aborted) { - await dependencies.terminate(child) + await terminate() return finish('stopped', spawnCount) } if (!childDown) { - await dependencies.terminate(child) + await terminate() return finish('gave-up', spawnCount) } if (spawnCount >= RECOVERY_MAX_SPAWNS) { From 216eabde0bd73e5ab5fd17aced4bc8aab43099a8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:40:08 -0700 Subject: [PATCH 11/38] fix(opencode): stop the launch model preflight server with Orca on POSIX Before an OpenCode launch, Orca starts `opencode serve` to read the configured agent and models, then stops it. The server was detached into its own process group and never exits when its input ends, so if Orca quit, crashed or was killed during that preflight (up to 10 s), nothing ever stopped it. On POSIX the server now runs under the provider supervisor in its one-shot lifetime: the preflight's closed input does not stop it, and Orca's exit does. The preflight's teardown asks the supervisor to stop (SIGTERM) and forces the tree only after the supervisor's full stop time; signalling or SIGKILLing the supervisor's own group would orphan the server's. A descendant that ignores SIGTERM is now killed with the group rather than left running once the pipes close. Windows keeps the direct spawn and its existing teardown. --- ...ncode-launch-model-context-cleanup.test.ts | 90 ++++++++++++++++-- ...e-launch-model-context-supervision.test.ts | 92 +++++++++++++++++++ .../opencode/opencode-launch-model-context.ts | 61 +++++++++--- 3 files changed, 223 insertions(+), 20 deletions(-) create mode 100644 src/main/opencode/opencode-launch-model-context-supervision.test.ts diff --git a/src/main/opencode/opencode-launch-model-context-cleanup.test.ts b/src/main/opencode/opencode-launch-model-context-cleanup.test.ts index 4acf1ba72de4..c1e12b0dc293 100644 --- a/src/main/opencode/opencode-launch-model-context-cleanup.test.ts +++ b/src/main/opencode/opencode-launch-model-context-cleanup.test.ts @@ -9,12 +9,17 @@ import { } from '../../shared/child-process/process-tree-termination' import { probeOpenCodeLaunchModelContext } from './opencode-launch-model-context' import { readFetchResponseJsonWithinLimit } from '../../shared/fetch-response-body' +import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../codex/codex-app-server-posix-supervisor' +import { terminateCodexAppServerProcessTree } from '../codex/codex-app-server-process-teardown' vi.mock('../../shared/child-process/run-process', () => ({ spawnProcess: vi.fn() })) vi.mock('../../shared/child-process/process-tree-termination', () => ({ signalProcessTree: vi.fn(), forceTerminateProcessTree: vi.fn() })) +vi.mock('../codex/codex-app-server-process-teardown', () => ({ + terminateCodexAppServerProcessTree: vi.fn() +})) vi.mock('../../shared/fetch-response-body', () => ({ readFetchResponseJsonWithinLimit: vi.fn() })) @@ -24,6 +29,8 @@ class ProbeChild extends ChildProcess { override stderr = new PassThrough() override pid = 123456 override exitCode: number | null = null + // A handle-less ChildProcess's own kill can signal this test's process group. + override kill = vi.fn((_signal?: NodeJS.Signals | number): boolean => true) override stdio: [PassThrough, PassThrough, PassThrough, undefined, undefined] = [ this.stdin, this.stdout, @@ -54,9 +61,24 @@ function close() { } const options = { executable: '/private/opencode', cwd: directory, env: {} } +function decodedSupervisorSpec(env: unknown): unknown { + const encoded = + typeof env === 'object' && env !== null && 'ORCA_PROVIDER_SUPERVISOR_SPEC' in env + ? String(env.ORCA_PROVIDER_SUPERVISOR_SPEC) + : '' + return JSON.parse(Buffer.from(encoded, 'base64').toString() || 'null') +} + +async function untilStopRequested(): Promise { + while (child.kill.mock.calls.length === 0) { + await new Promise((resolve) => setImmediate(resolve)) + } +} + describe('OpenCode model probe termination evidence', () => { beforeEach(() => { vi.resetAllMocks() + Object.defineProperty(process, 'platform', { value: 'darwin' }) vi.mocked(readFetchResponseJsonWithinLimit).mockImplementation((response) => response.json()) child = new ProbeChild() closeDuringFetch = false @@ -82,9 +104,16 @@ describe('OpenCode model probe termination evidence', () => { return true }) vi.mocked(forceTerminateProcessTree).mockResolvedValue(false) + child.kill.mockImplementation((signal) => { + if (signal === 'SIGTERM') { + close() + } + return true + }) }) afterEach(() => { + vi.useRealTimers() if (platform) { Object.defineProperty(process, 'platform', platform) } @@ -100,7 +129,7 @@ describe('OpenCode model probe termination evidence', () => { ) expect(await probeOpenCodeLaunchModelContext(options)).toBeNull() expect(cancel).toHaveBeenCalledTimes(4) - expect(signalProcessTree).toHaveBeenCalledOnce() + expect(child.kill).toHaveBeenCalledWith('SIGTERM') }) it('accepts an already-closed Windows probe without signaling its former pid', async () => { @@ -112,14 +141,62 @@ describe('OpenCode model probe termination evidence', () => { expect(forceTerminateProcessTree).not.toHaveBeenCalled() }) - it('does not force a POSIX group after the graceful close', async () => { - Object.defineProperty(process, 'platform', { value: 'darwin' }) + it('runs a POSIX probe under a one-shot supervisor and stops it through that supervisor', async () => { expect(await probeOpenCodeLaunchModelContext(options)).toMatchObject({ primaryAgent: 'build' }) - expect(signalProcessTree).toHaveBeenCalledOnce() + + const [spec] = vi.mocked(spawnProcess).mock.calls[0] + expect(spec).toMatchObject({ program: process.execPath, cwd: directory, detached: true }) + const args = spec.args ?? [] + expect(args.slice(args.indexOf('--') + 1)).toEqual([ + '/private/opencode', + 'serve', + '--hostname', + '127.0.0.1', + '--port', + '0' + ]) + // Its stdin ends at once, so a session lifetime would stop the server a second later. + expect(decodedSupervisorSpec(spec.env)).toMatchObject({ lifetime: 'one-shot', cwd: directory }) + expect(child.kill.mock.calls).toEqual([['SIGTERM']]) + // Signalling the supervisor's own group, or SIGKILLing it, would orphan the server's group. + expect(signalProcessTree).not.toHaveBeenCalled() expect(forceTerminateProcessTree).not.toHaveBeenCalled() + expect(terminateCodexAppServerProcessTree).not.toHaveBeenCalled() }) - it('keeps a root exit with inherited pipes unverified and avoids its former pid', async () => { + it('forces a POSIX supervisor tree only after its full stop time', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + child.kill.mockReturnValue(true) + vi.mocked(terminateCodexAppServerProcessTree).mockResolvedValue(true) + + const probe = probeOpenCodeLaunchModelContext(options) + await untilStopRequested() + await vi.advanceTimersByTimeAsync(PROVIDER_SUPERVISOR_MAX_STOP_MS - 1) + expect(terminateCodexAppServerProcessTree).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + + expect(await probe).toMatchObject({ primaryAgent: 'build' }) + expect(terminateCodexAppServerProcessTree).toHaveBeenCalledWith(child) + }) + + it('never forces a POSIX supervisor whose root exited without closing its pipes', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + child.kill.mockImplementation(() => { + child.exitCode = 0 + child.emit('exit', 0, null) + return true + }) + + const probe = probeOpenCodeLaunchModelContext(options) + await untilStopRequested() + await vi.advanceTimersByTimeAsync(PROVIDER_SUPERVISOR_MAX_STOP_MS) + + expect(await probe).toBeNull() + expect(terminateCodexAppServerProcessTree).not.toHaveBeenCalled() + }) + + it('keeps a Windows root exit with inherited pipes unverified and avoids its former pid', async () => { + Object.defineProperty(process, 'platform', { value: 'win32' }) vi.mocked(signalProcessTree).mockImplementation(async () => { child.exitCode = 0 child.emit('exit', 0, null) @@ -129,7 +206,8 @@ describe('OpenCode model probe termination evidence', () => { expect(forceTerminateProcessTree).not.toHaveBeenCalled() }) - it('requires verified termination when a still-live probe does not close', async () => { + it('requires verified termination when a still-live Windows probe does not close', async () => { + Object.defineProperty(process, 'platform', { value: 'win32' }) vi.mocked(signalProcessTree).mockResolvedValue(true) expect(await probeOpenCodeLaunchModelContext(options)).toBeNull() expect(forceTerminateProcessTree).toHaveBeenCalledOnce() diff --git a/src/main/opencode/opencode-launch-model-context-supervision.test.ts b/src/main/opencode/opencode-launch-model-context-supervision.test.ts new file mode 100644 index 000000000000..224911545930 --- /dev/null +++ b/src/main/opencode/opencode-launch-model-context-supervision.test.ts @@ -0,0 +1,92 @@ +import { existsSync, readFileSync } from 'node:fs' +import { afterEach, describe, expect, it } from 'vitest' +import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../codex/codex-app-server-posix-supervisor' +import { + alive, + createSupervisedProbeRig, + waitFor, + type SupervisedProbeRig +} from '../codex/supervised-probe-owner.test-fixture' +import { probeOpenCodeLaunchModelContext } from './opencode-launch-model-context' + +// Like `opencode serve`, it never exits on stdin end. It starts listening only after the session +// stdin-end grace, so a supervisor that treated the probe's stdin end as a stop would kill it first. +const SERVE = String.raw` +setTimeout(() => { + const server = require('node:http').createServer((request, response) => { + const url = new URL(request.url, 'http://127.0.0.1') + const location = { directory: url.searchParams.get('location[directory]') } + const model = { id: 'model', providerID: 'provider', enabled: true } + const bodies = { + '/api/model': { location, data: [model] }, + '/api/agent': { location, data: [{ id: 'build', mode: 'primary', hidden: false }] }, + '/api/model/default': { location, data: model }, + '/api/config': [{ type: 'document', info: { default_agent: 'build' } }] + } + response.setHeader('content-type', 'application/json') + response.end(JSON.stringify(bodies[url.pathname])) + }) + server.listen(0, '127.0.0.1', () => { + process.stdout.write('server listening on http://127.0.0.1:' + server.address().port + '\n') + }) +}, 1500) +` + +const OWNER = String.raw` +import { probeOpenCodeLaunchModelContext } from './opencode-launch-model-context' +void probeOpenCodeLaunchModelContext({ + executable: process.env.ORCA_TEST_STAND_IN!, + cwd: process.cwd(), + env: process.env +}).catch(() => {}) +setInterval(() => {}, 60_000) +` + +let rig: SupervisedProbeRig | null = null + +afterEach(() => { + rig?.cleanup() + rig = null +}) + +describe.runIf(process.platform !== 'win32')('supervised OpenCode launch-context probe', () => { + it('reads the context after its stdin end, then stops the whole server group', async () => { + rig = createSupervisedProbeRig() + const executable = rig.writeStandIn('opencode', SERVE) + + const context = await probeOpenCodeLaunchModelContext({ + executable, + cwd: rig.dir, + env: { ...process.env, ...rig.env } + }) + const pids = await rig.readPids() + + expect(context).toEqual({ + primaryAgent: 'build', + availableModels: ['provider/model'], + primaryModel: null + }) + expect(existsSync(rig.signalFile) && readFileSync(rig.signalFile, 'utf8')).toBe('SIGTERM') + expect(alive(pids.provider)).toBe(false) + // A descendant that ignores SIGTERM holds none of the probe's pipes, so a closed pipe never + // proved it gone; only the supervisor's group SIGKILL ends it. + expect(alive(pids.grandchild)).toBe(false) + }) + + it('stops the server group when its owner is SIGKILLed mid-probe', async () => { + rig = createSupervisedProbeRig() + const executable = rig.writeStandIn('opencode') + const bundle = await rig.bundleOwner(OWNER, __dirname) + const owner = rig.launchOwner(bundle, { ORCA_TEST_STAND_IN: executable }) + const pids = await rig.readPids() + + owner.kill('SIGKILL') + + expect( + await waitFor( + () => !alive(pids.provider) && !alive(pids.grandchild), + PROVIDER_SUPERVISOR_MAX_STOP_MS + 1_000 + ) + ).toBe(true) + }) +}) diff --git a/src/main/opencode/opencode-launch-model-context.ts b/src/main/opencode/opencode-launch-model-context.ts index ce1098d8acfb..e45eb32f9b5d 100644 --- a/src/main/opencode/opencode-launch-model-context.ts +++ b/src/main/opencode/opencode-launch-model-context.ts @@ -10,6 +10,11 @@ import { withTimeout } from '../../shared/promise-timeout-fallback' import { createOutputSink } from '../../shared/child-process/bounded-output-sink' import { readFetchResponseJsonWithinLimit } from '../../shared/fetch-response-body' import { cancelUnreadResponseBody } from '../lib/unread-response-body' +import { + createProviderSpawnSpec, + stopSupervisedProvider +} from '../codex/codex-app-server-posix-supervisor' +import { terminateCodexAppServerProcessTree } from '../codex/codex-app-server-process-teardown' const model = z.object({ id: z.string(), providerID: z.string() }) const availableModel = model.extend({ enabled: z.boolean() }) @@ -94,15 +99,22 @@ export async function probeOpenCodeLaunchModelContext(options: { expectedPrimaryModel?: string }): Promise { const password = randomBytes(32).toString('base64url') + const spawnSpec = createProviderSpawnSpec( + { + command: options.executable, + args: ['serve', '--hostname', '127.0.0.1', '--port', '0'], + cwd: options.cwd + }, + { ...options.env, OPENCODE_SERVER_USERNAME: 'opencode', OPENCODE_SERVER_PASSWORD: password }, + process.platform, + // `opencode serve` never exits on stdin end, so on POSIX only its supervisor stops it with Orca. + { lifetime: 'one-shot' } + ) const child = spawnProcess({ - program: options.executable, - args: ['serve', '--hostname', '127.0.0.1', '--port', '0'], + program: spawnSpec.program, + args: spawnSpec.args, cwd: options.cwd, - env: { - ...options.env, - OPENCODE_SERVER_USERNAME: 'opencode', - OPENCODE_SERVER_PASSWORD: password - }, + env: spawnSpec.env, detached: true }) const output = createOutputSink(4096) @@ -193,13 +205,34 @@ export async function probeOpenCodeLaunchModelContext(options: { } catch { context = null } finally { - if (!childClosed && !rootExited) { - await signalProcessTree(child, 'SIGTERM') - } - stopped = childClosed || (await withTimeout(closed, 1_500, false)) - // A reaped root's PID can be reused while descendants still hold its pipes. - if (!stopped && !rootExited) { - stopped = await forceTerminateProcessTree(child) + if (spawnSpec.supervised) { + let forcedStop = false + await stopSupervisedProvider({ + request: () => { + try { + child.kill('SIGTERM') + } catch { + // The supervisor may exit between the close check and kill. + } + }, + exitPromise: closed.then(() => undefined), + exited: () => childClosed, + force: async () => { + // A reaped root's PID can be reused; only a live supervisor's tree is ours to force. + forcedStop = !rootExited && (await terminateCodexAppServerProcessTree(child)) + }, + supervised: true + }) + stopped = childClosed || forcedStop + } else { + if (!childClosed && !rootExited) { + await signalProcessTree(child, 'SIGTERM') + } + stopped = childClosed || (await withTimeout(closed, 1_500, false)) + // A reaped root's PID can be reused while descendants still hold its pipes. + if (!stopped && !rootExited) { + stopped = await forceTerminateProcessTree(child) + } } } return stopped ? context : null From 4f1bc5416cbe718f862eb020904c4e4695298e89 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:42:02 -0700 Subject: [PATCH 12/38] test(codex): pin when supervised session and backfill stops escalate A session past its deadline is SIGTERMed through its supervisor rather than waiting out the stdin-end grace, and its tree is torn down only after the supervisor's full stop time; Windows keeps its deadline kill and 1.5 s close wait. A supervised backfill app-server is stopped by ending its input, with the same full stop time before any teardown. --- .../codex-app-server-session-stop.test.ts | 125 ++++++++++++++++++ ...state-db-backfill-recovery-process.test.ts | 74 +++++++++++ 2 files changed, 199 insertions(+) create mode 100644 src/main/codex/codex-app-server-session-stop.test.ts create mode 100644 src/main/codex/codex-state-db-backfill-recovery-process.test.ts diff --git a/src/main/codex/codex-app-server-session-stop.test.ts b/src/main/codex/codex-app-server-session-stop.test.ts new file mode 100644 index 000000000000..413526fbb759 --- /dev/null +++ b/src/main/codex/codex-app-server-session-stop.test.ts @@ -0,0 +1,125 @@ +import { EventEmitter } from 'node:events' +import { PassThrough } from 'node:stream' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from './codex-app-server-posix-supervisor' +import { terminateCodexAppServerProcessTree } from './codex-app-server-process-teardown' +import type * as ProcessTreeKill from './codex-app-server-process-tree-kill' +import { + killCodexAppServerProcessTree, + type CodexAppServerSpawn +} from './codex-app-server-process-tree-kill' +import { CodexAppServerTimeoutError, runCodexAppServerSession } from './codex-app-server-session' + +vi.mock('./codex-app-server-process-teardown', () => ({ + terminateCodexAppServerProcessTree: vi.fn(async () => true) +})) +vi.mock('./codex-app-server-process-tree-kill', async (importOriginal) => ({ + ...(await importOriginal()), + killCodexAppServerProcessTree: vi.fn() +})) + +type FakeChild = EventEmitter & { + pid: number + stdin: PassThrough + stdout: PassThrough + stderr: PassThrough + kill: ReturnType boolean>> +} + +/** A child that answers initialize and then nothing; it exits only on `exitsOn`. */ +function fakeChild(exitsOn: NodeJS.Signals | null): FakeChild { + const child: FakeChild = Object.assign(new EventEmitter(), { + pid: 4242, + stdin: new PassThrough(), + stdout: new PassThrough(), + stderr: new PassThrough(), + kill: vi.fn((signal?: NodeJS.Signals) => { + if (signal === exitsOn) { + child.emit('exit', null, signal) + child.emit('close', null, signal) + } + return true + }) + }) + child.stdin.on('data', (chunk: Buffer) => { + const message = JSON.parse(chunk.toString()) + if (message.method === 'initialize') { + child.stdout.write(`${JSON.stringify({ id: message.id, result: {} })}\n`) + } + }) + return child +} + +const originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform') + +afterEach(() => { + vi.useRealTimers() + vi.clearAllMocks() + if (originalPlatform) { + Object.defineProperty(process, 'platform', originalPlatform) + } +}) + +function startWedgedSession(child: FakeChild, platform: NodeJS.Platform): Promise { + Object.defineProperty(process, 'platform', { configurable: true, value: platform }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the session reads only pid, stdio, kill and the exit/close/error events, which the fake implements. + const spawnImpl: CodexAppServerSpawn = () => child as never + return runCodexAppServerSession( + { command: '/bin/codex', cliPath: null, args: ['app-server'], timeoutMs: 1_000 }, + () => new Promise(() => {}), + spawnImpl + ).catch((error: unknown) => error) +} + +describe('runCodexAppServerSession stop', () => { + it('SIGTERMs a supervised session at its deadline instead of waiting out its stdin grace', async () => { + vi.useFakeTimers() + const child = fakeChild('SIGTERM') + let stdinEnded = false + child.stdin.on('finish', () => (stdinEnded = true)) + + const outcome = startWedgedSession(child, 'darwin') + await vi.advanceTimersByTimeAsync(1_000) + + expect(child.kill.mock.calls).toEqual([['SIGTERM']]) + expect(stdinEnded).toBe(true) + expect(await outcome).toBeInstanceOf(CodexAppServerTimeoutError) + expect(terminateCodexAppServerProcessTree).not.toHaveBeenCalled() + expect(killCodexAppServerProcessTree).not.toHaveBeenCalled() + }) + + it('tears a supervised session down only once the supervisor has had its full stop time', async () => { + vi.useFakeTimers() + const child = fakeChild(null) + + const outcome = startWedgedSession(child, 'linux') + await vi.advanceTimersByTimeAsync(1_000 + PROVIDER_SUPERVISOR_MAX_STOP_MS - 1) + expect(terminateCodexAppServerProcessTree).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + + expect(terminateCodexAppServerProcessTree).toHaveBeenCalledWith(child) + await vi.advanceTimersByTimeAsync(1_000) + expect(await outcome).toBeInstanceOf(CodexAppServerTimeoutError) + // Its own group SIGKILL would kill the supervisor mid-ladder and orphan the server's group. + expect(child.kill).not.toHaveBeenCalledWith('SIGKILL') + expect(killCodexAppServerProcessTree).not.toHaveBeenCalled() + }) + + it('keeps the Windows deadline kill and its 1.5 s close wait', async () => { + vi.useFakeTimers() + const child = fakeChild(null) + + const outcome = startWedgedSession(child, 'win32') + await vi.advanceTimersByTimeAsync(1_000) + expect(killCodexAppServerProcessTree).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1_499) + expect(killCodexAppServerProcessTree).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + expect(killCodexAppServerProcessTree).toHaveBeenCalledTimes(2) + + await vi.advanceTimersByTimeAsync(1_000) + expect(await outcome).toBeInstanceOf(CodexAppServerTimeoutError) + expect(child.kill).not.toHaveBeenCalled() + expect(terminateCodexAppServerProcessTree).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/codex/codex-state-db-backfill-recovery-process.test.ts b/src/main/codex/codex-state-db-backfill-recovery-process.test.ts new file mode 100644 index 000000000000..ef25b1db1949 --- /dev/null +++ b/src/main/codex/codex-state-db-backfill-recovery-process.test.ts @@ -0,0 +1,74 @@ +import { EventEmitter } from 'node:events' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { ChildProcessHandle } from '../../shared/child-process/process-spec' +import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from './codex-app-server-posix-supervisor' +import { terminateCodexAppServerProcessTree } from './codex-app-server-process-teardown' +import { stopCodexBackfillRecoveryProcess } from './codex-state-db-backfill-recovery-process' + +vi.mock('./codex-app-server-process-teardown', () => ({ + terminateCodexAppServerProcessTree: vi.fn(async () => true) +})) + +type FakeSupervisor = EventEmitter & { + pid: number + exitCode: number | null + signalCode: NodeJS.Signals | null + stdin: { end: ReturnType } + kill: ReturnType boolean>> +} + +function fakeSupervisor(exitsOnStdinEnd: boolean): FakeSupervisor { + const child: FakeSupervisor = Object.assign(new EventEmitter(), { + pid: 4242, + exitCode: null, + signalCode: null, + stdin: { + end: vi.fn(() => { + if (exitsOnStdinEnd) { + child.exitCode = 0 + child.emit('exit', 0, null) + } + }) + }, + kill: vi.fn(() => true) + }) + return child +} + +function asHandle(child: FakeSupervisor): ChildProcessHandle { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the stop reads only pid, exit state, stdin, kill and the exit event, which the fake implements. + return child as unknown as ChildProcessHandle +} + +afterEach(() => { + vi.useRealTimers() + vi.clearAllMocks() +}) + +describe('stopCodexBackfillRecoveryProcess for a supervised app-server', () => { + it('ends its input and leaves the group stop to the supervisor', async () => { + const child = fakeSupervisor(true) + + await stopCodexBackfillRecoveryProcess(asHandle(child), true) + + expect(child.stdin.end).toHaveBeenCalledOnce() + // A direct SIGTERM would skip the stdin-end grace a Codex close gets. + expect(child.kill).not.toHaveBeenCalled() + expect(terminateCodexAppServerProcessTree).not.toHaveBeenCalled() + }) + + it('tears the tree down only once the supervisor has had its full stop time', async () => { + vi.useFakeTimers() + const child = fakeSupervisor(false) + + const stopped = stopCodexBackfillRecoveryProcess(asHandle(child), true) + await vi.advanceTimersByTimeAsync(PROVIDER_SUPERVISOR_MAX_STOP_MS - 1) + expect(terminateCodexAppServerProcessTree).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await stopped + + expect(terminateCodexAppServerProcessTree).toHaveBeenCalledWith(child) + // Its own SIGKILL would kill the supervisor mid-ladder and orphan the app-server's group. + expect(child.kill).not.toHaveBeenCalledWith('SIGKILL') + }) +}) From e7ecdd4e9403086eae6e77d7c210d8f900f29403 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:54:30 -0700 Subject: [PATCH 13/38] test(text-generation): run the direct-child suites on the Windows path and cover supervised discovery The commit-message suites that drive fake children mocked the supervisor away on POSIX, so they asserted a direct root SIGKILL that production no longer takes there. They now pin the platform to Windows (with an empty PATH, so host installs cannot answer a bare agent name) and assert the Windows kill, taskkill included. The three tests that check the host's own discovery spawn shape run on the host and read the agent argv past the supervisor's '--'. Model discovery gets its supervised composition: a timed-out Codex discovery settles at once but holds the Codex home until its supervisor has stopped. --- ...ssage-text-generation-cancellation.test.ts | 26 ++++---- ...e-text-generation-local-subprocess.test.ts | 26 ++++---- ...ge-text-generation-model-discovery.test.ts | 66 ++++++++++++------- ...it-message-text-generation-test-harness.ts | 24 +++---- .../source-control-local-process.test.ts | 41 ++++++++++++ 5 files changed, 120 insertions(+), 63 deletions(-) diff --git a/src/main/text-generation/commit-message-text-generation-cancellation.test.ts b/src/main/text-generation/commit-message-text-generation-cancellation.test.ts index a644c0e138fc..e0e83431337b 100644 --- a/src/main/text-generation/commit-message-text-generation-cancellation.test.ts +++ b/src/main/text-generation/commit-message-text-generation-cancellation.test.ts @@ -1,6 +1,6 @@ import { spawn } from 'node:child_process' import type * as ChildProcess from 'node:child_process' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { cancelGenerateCommitMessageLocal, cancelGeneratePullRequestFieldsLocal, @@ -22,15 +22,6 @@ vi.mock('../windows-process-tree-kill', () => ({ terminateWindowsProcessTree: terminateWindowsProcessTreeMock })) -// Generation over a direct agent child, the unsupervised shape Windows and WSL spawn. The POSIX -// supervised stop composed with timeout, cancel, output limit and the Codex home lock is covered -// in source-control-local-process.test.ts. -vi.mock('../codex/codex-app-server-posix-supervisor', async (importOriginal) => - (await import('./commit-message-text-generation-test-harness')).directAgentChildSupervisorModule( - importOriginal - ) -) - vi.mock('child_process', async (importOriginal) => { const actual = await importOriginal() return { @@ -43,13 +34,26 @@ const spawnMock = vi.mocked(spawn) const expectChildTerminated = createChildTerminationExpectation(terminateWindowsProcessTreeMock) +// These suites drive fake children down the Windows direct-child path, taskkill included. The POSIX +// supervised stop under timeout, cancel, output limit and the Codex home lock, for generation and +// discovery, is in source-control-local-process.test.ts. +const hostPlatform = process.platform + +afterEach(() => { + Object.defineProperty(process, 'platform', { configurable: true, value: hostPlatform }) + vi.unstubAllEnvs() +}) + beforeEach(() => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + // Windows resolves a bare agent name on PATH; the host's own installs must not answer it. + vi.stubEnv('PATH', '') terminateWindowsProcessTreeMock.mockClear() terminateWindowsProcessTreeMock.mockResolvedValue(undefined) spawnMock.mockClear() }) -describe('generateCommitMessageFromContext over a direct agent child', () => { +describe('generateCommitMessageFromContext on the Windows direct-child path', () => { it('fails clearly before spawning when a jcode argv prompt exceeds the Windows command line', async () => { await withPlatform('win32', async () => { const pending = generateCommitMessageFromContext( diff --git a/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts b/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts index d46193051197..804bcaed9420 100644 --- a/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts +++ b/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts @@ -1,6 +1,6 @@ import { spawn } from 'node:child_process' import type * as ChildProcess from 'node:child_process' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { generateCommitMessageFromContext } from './commit-message-text-generation' import { createChildTerminationExpectation, @@ -15,15 +15,6 @@ vi.mock('../windows-process-tree-kill', () => ({ terminateWindowsProcessTree: terminateWindowsProcessTreeMock })) -// Generation over a direct agent child, the unsupervised shape Windows and WSL spawn. The POSIX -// supervised stop composed with timeout, cancel, output limit and the Codex home lock is covered -// in source-control-local-process.test.ts. -vi.mock('../codex/codex-app-server-posix-supervisor', async (importOriginal) => - (await import('./commit-message-text-generation-test-harness')).directAgentChildSupervisorModule( - importOriginal - ) -) - vi.mock('child_process', async (importOriginal) => { const actual = await importOriginal() return { @@ -36,13 +27,26 @@ const spawnMock = vi.mocked(spawn) const expectChildTerminated = createChildTerminationExpectation(terminateWindowsProcessTreeMock) +// These suites drive fake children down the Windows direct-child path, taskkill included. The POSIX +// supervised stop under timeout, cancel, output limit and the Codex home lock, for generation and +// discovery, is in source-control-local-process.test.ts. +const hostPlatform = process.platform + +afterEach(() => { + Object.defineProperty(process, 'platform', { configurable: true, value: hostPlatform }) + vi.unstubAllEnvs() +}) + beforeEach(() => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + // Windows resolves a bare agent name on PATH; the host's own installs must not answer it. + vi.stubEnv('PATH', '') terminateWindowsProcessTreeMock.mockClear() terminateWindowsProcessTreeMock.mockResolvedValue(undefined) spawnMock.mockClear() }) -describe('generateCommitMessageFromContext over a direct agent child', () => { +describe('generateCommitMessageFromContext on the Windows direct-child path', () => { it('caps local agent output before buffering unbounded data', async () => { const listeners = new Map void>() const child = { diff --git a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts index 37c171c7b327..8fece13fb86d 100644 --- a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts +++ b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts @@ -1,6 +1,6 @@ import { spawn } from 'node:child_process' import type * as ChildProcess from 'node:child_process' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { createSshDisposalError, SSH_MUX_REQUEST_TIMEOUT_CODE @@ -13,6 +13,7 @@ import { PROVIDER_SPAWN_FAILURE_MARKER } from '../codex/codex-app-server-posix-s import { createChildTerminationExpectation, createMockDiscoveryChild, + spawnedAgentArgv, withPlatform } from './commit-message-text-generation-test-harness' @@ -24,15 +25,6 @@ vi.mock('../windows-process-tree-kill', () => ({ terminateWindowsProcessTree: terminateWindowsProcessTreeMock })) -// Generation over a direct agent child, the unsupervised shape Windows and WSL spawn. The POSIX -// supervised stop composed with timeout, cancel, output limit and the Codex home lock is covered -// in source-control-local-process.test.ts. -vi.mock('../codex/codex-app-server-posix-supervisor', async (importOriginal) => - (await import('./commit-message-text-generation-test-harness')).directAgentChildSupervisorModule( - importOriginal - ) -) - vi.mock('child_process', async (importOriginal) => { const actual = await importOriginal() return { @@ -49,13 +41,26 @@ function spawnError(errno: string): Error { const expectChildTerminated = createChildTerminationExpectation(terminateWindowsProcessTreeMock) +// These suites drive fake children down the Windows direct-child path, taskkill included. The POSIX +// supervised stop under timeout, cancel, output limit and the Codex home lock, for generation and +// discovery, is in source-control-local-process.test.ts. +const hostPlatform = process.platform + +afterEach(() => { + Object.defineProperty(process, 'platform', { configurable: true, value: hostPlatform }) + vi.unstubAllEnvs() +}) + beforeEach(() => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + // Windows resolves a bare agent name on PATH; the host's own installs must not answer it. + vi.stubEnv('PATH', '') terminateWindowsProcessTreeMock.mockClear() terminateWindowsProcessTreeMock.mockResolvedValue(undefined) spawnMock.mockClear() }) -describe('discoverCommitMessageModelsLocal over a direct agent child', () => { +describe('discoverCommitMessageModelsLocal on the Windows direct-child path', () => { it('returns static catalog models without spawning for static agents', async () => { const result = await discoverCommitMessageModelsLocal('amp', undefined) @@ -68,6 +73,8 @@ describe('discoverCommitMessageModelsLocal over a direct agent child', () => { }) it('discovers dynamic models through the agent CLI', async () => { + // The host's own spawn shape: supervised on POSIX, direct on Windows. + Object.defineProperty(process, 'platform', { configurable: true, value: hostPlatform }) const listeners = new Map void>() const child = { pid: 123, @@ -92,14 +99,13 @@ describe('discoverCommitMessageModelsLocal over a direct agent child', () => { { id: 'gpt-5.2', label: 'GPT-5.2' } ] }) - expect(spawnMock).toHaveBeenCalledWith( - 'cursor-agent', - ['--list-models'], - expect.objectContaining({ windowsHide: true }) - ) + expect(spawnedAgentArgv(spawnMock.mock.calls[0]!)).toEqual(['cursor-agent', '--list-models']) + expect(spawnMock.mock.calls[0]![2]).toMatchObject({ windowsHide: true }) }) it('writes the Claude list_models request to stdin and parses the control response', async () => { + // The host's own spawn shape: supervised on POSIX, direct on Windows. + Object.defineProperty(process, 'platform', { configurable: true, value: hostPlatform }) const listeners = new Map void>() const child = { pid: 123, @@ -149,11 +155,19 @@ describe('discoverCommitMessageModelsLocal over a direct agent child', () => { { id: 'haiku', label: 'Haiku' } ] }) - expect(spawnMock).toHaveBeenCalledWith( + expect(spawnedAgentArgv(spawnMock.mock.calls[0]!)).toEqual([ 'claude', - ['-p', '--input-format', 'stream-json', '--output-format', 'stream-json', '--verbose'], - expect.objectContaining({ windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] }) - ) + '-p', + '--input-format', + 'stream-json', + '--output-format', + 'stream-json', + '--verbose' + ]) + expect(spawnMock.mock.calls[0]![2]).toMatchObject({ + windowsHide: true, + stdio: ['pipe', 'pipe', 'pipe'] + }) expect(child.stdin.end).toHaveBeenCalledWith(expect.stringContaining('"list_models"')) }) @@ -188,6 +202,8 @@ describe('discoverCommitMessageModelsLocal over a direct agent child', () => { }) it('discovers dynamic models through the configured agent command override', async () => { + // The host's own spawn shape: supervised on POSIX, direct on Windows. + Object.defineProperty(process, 'platform', { configurable: true, value: hostPlatform }) const listeners = new Map void>() const child = { pid: 123, @@ -215,11 +231,11 @@ describe('discoverCommitMessageModelsLocal over a direct agent child', () => { expect.objectContaining({ windowsHide: true }) ) } else { - expect(spawnMock).toHaveBeenCalledWith( + expect(spawnedAgentArgv(spawnMock.mock.calls[0]!)).toEqual([ 'npx', - ['cursor-agent', '--list-models'], - expect.objectContaining({ windowsHide: true }) - ) + 'cursor-agent', + '--list-models' + ]) } }) @@ -504,7 +520,7 @@ describe('discoverCommitMessageModelsLocal over a direct agent child', () => { }) }) -describe('generateCommitMessageFromContext over a direct agent child', () => { +describe('generateCommitMessageFromContext on the Windows direct-child path', () => { it('discovers dynamic models through a remote execution plan', async () => { const execute = vi.fn(async (plan, cwd, timeoutMs) => { expect(plan).toEqual({ diff --git a/src/main/text-generation/commit-message-text-generation-test-harness.ts b/src/main/text-generation/commit-message-text-generation-test-harness.ts index 62875b3b3345..9c98bf34603d 100644 --- a/src/main/text-generation/commit-message-text-generation-test-harness.ts +++ b/src/main/text-generation/commit-message-text-generation-test-harness.ts @@ -1,6 +1,5 @@ import { EventEmitter } from 'node:events' import { expect, vi } from 'vitest' -import type * as ProviderSupervisor from '../codex/codex-app-server-posix-supervisor' export type MockDiscoveryChild = EventEmitter & { pid: number @@ -31,8 +30,8 @@ export function withPlatform(platform: NodeJS.Platform, fn: () => T): T { } // Binds the caller's hoisted tree-kill mock so test bodies keep calling -// expectChildTerminated(child) with no extra argument. Asserts the direct-child kill: a -// supervised child is stopped with SIGTERM instead (source-control-local-process.test.ts). +// expectChildTerminated(child) with no extra argument. Asserts the Windows direct-child kill; a +// supervised POSIX child is stopped with SIGTERM instead (source-control-local-process.test.ts). export function createChildTerminationExpectation( terminateWindowsProcessTreeMock: ReturnType ): (child: { pid: number; kill: ReturnType }) => Promise { @@ -50,17 +49,10 @@ export function createChildTerminationExpectation( } } -/** - * The supervisor module with every agent spawned as a direct child, the unsupervised shape Windows - * and WSL use, for suites that drive fake children through generation. - */ -export async function directAgentChildSupervisorModule( - importOriginal: () => Promise -): Promise { - const actual = await importOriginal() - return { - ...actual, - createProviderSpawnSpec: (launch, env, _platform, options) => - actual.createProviderSpawnSpec(launch, env, 'win32', options) - } +/** The argv a spawn started as the agent: past the supervisor script's '--' when supervised. */ +export function spawnedAgentArgv([file, args]: readonly unknown[]): unknown[] { + const argv: unknown[] = Array.isArray(args) ? args : [] + return file === process.execPath && argv[0] === '-e' + ? argv.slice(argv.indexOf('--') + 1) + : [file, ...argv] } diff --git a/src/main/text-generation/source-control-local-process.test.ts b/src/main/text-generation/source-control-local-process.test.ts index 55863f4f4077..013c502849b9 100644 --- a/src/main/text-generation/source-control-local-process.test.ts +++ b/src/main/text-generation/source-control-local-process.test.ts @@ -5,6 +5,7 @@ import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../codex/codex-app-server-posix-supervisor' import { SOURCE_CONTROL_GENERATION_TIMEOUT_MS } from './source-control-generation-limits' +import { discoverModelsLocal } from './commit-message-model-discovery' import { cancelGenerateCommitMessageLocal } from './commit-message-text-generation' import { spawnSourceControlAgent } from './source-control-agent-launch' import { killSourceControlAgentProcess } from './source-control-local-process' @@ -161,6 +162,46 @@ describe('a supervised agent one-shot that runs out of time', () => { }) }) +describe('a supervised Codex model discovery that runs out of time', () => { + it('times out at once but holds the Codex home until the supervisor has stopped', async () => { + vi.useFakeTimers() + const stopMs = 2_000 + const agents = [fakeSupervisedAgent(stopMs), fakeSupervisedAgent(stopMs)] + const spawnAgent = vi.fn(() => asSpawned(agents[spawnAgent.mock.calls.length - 1]!)) + const discover = (): ReturnType => + discoverModelsLocal({ + agentId: 'codex', + env: { CODEX_HOME: '/codex/discovery-timeout-home' }, + options: {}, + backslash: 'escape', + spawnAgent + }) + + const first = discover() + await vi.advanceTimersByTimeAsync(SOURCE_CONTROL_GENERATION_TIMEOUT_MS) + await expect(first).resolves.toMatchObject({ + success: false, + error: expect.stringMatching(/timed out/) + }) + expect(agents[0]!.kill.mock.calls).toEqual([['SIGTERM']]) + + const second = discover() + await vi.advanceTimersByTimeAsync(stopMs - 1) + expect(spawnAgent).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + expect(spawnAgent).toHaveBeenCalledTimes(2) + + agents[1]!.stdout.emit( + 'data', + Buffer.from(JSON.stringify({ models: [{ slug: 'gpt-5.5', display_name: 'GPT-5.5' }] })) + ) + agents[1]!.emit('close', 0) + await expect(second).resolves.toMatchObject({ success: true }) + expect(agents[0]!.kill).not.toHaveBeenCalledWith('SIGKILL') + expect(terminateTreeMock).not.toHaveBeenCalled() + }) +}) + describe.runIf(process.platform !== 'win32')( 'killSourceControlAgentProcess on a real agent', () => { From b2eced4a3a2c47389a525d00b290714cc82435cd Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:56:50 -0700 Subject: [PATCH 14/38] fix(supervisor): show a supervised spawn failure in native chat as the spawn error it was Native chat's exit errors carry the provider's stderr tail into Details. Under the supervisor a missing CLI left the supervisor's internal spawn-failure report there instead of Node's own 'spawn ENOENT'. The report, its parser and a display formatter now live in one module; the Codex app-server and Claude stream-json exit errors pass the tail through the formatter, which turns a report back into the spawn error and leaves any other stderr unchanged. --- .../claude-stream-json-connection.test.ts | 5 +- .../claude/claude-stream-json-connection.ts | 3 +- .../codex/codex-app-server-exit-error.test.ts | 13 +++++ src/main/codex/codex-app-server-exit-error.ts | 3 +- .../codex-app-server-posix-supervisor.test.ts | 32 +----------- .../codex-app-server-posix-supervisor.ts | 37 +------------ .../provider-spawn-failure-report.test.ts | 45 ++++++++++++++++ .../codex/provider-spawn-failure-report.ts | 52 +++++++++++++++++++ .../commit-message-model-discovery.ts | 2 +- ...ge-text-generation-model-discovery.test.ts | 2 +- .../source-control-local-process.ts | 6 +-- 11 files changed, 124 insertions(+), 76 deletions(-) create mode 100644 src/main/codex/provider-spawn-failure-report.test.ts create mode 100644 src/main/codex/provider-spawn-failure-report.ts diff --git a/src/main/claude/claude-stream-json-connection.test.ts b/src/main/claude/claude-stream-json-connection.test.ts index f7ecad5f8ea1..7540e1e8ae73 100644 --- a/src/main/claude/claude-stream-json-connection.test.ts +++ b/src/main/claude/claude-stream-json-connection.test.ts @@ -673,7 +673,10 @@ describe('Claude stream-json connection', () => { const reported = await until(() => exit, 'the supervised spawn failure') // The supervisor spawned, so this is its exit; only its stderr can say why. - expect(reported.message).toMatch(/\(code 127\).*ENOENT/s) + // Reads as the direct spawn's own error, never the supervisor's internal report. + expect(reported.message).toBe( + `claude stream-json exited (code 127): spawn ${missingCli} ENOENT` + ) // A first-hand root exit, which releases the lease like a processless start did. expect(connection.exitVerdict.root).toBe('exited') } diff --git a/src/main/claude/claude-stream-json-connection.ts b/src/main/claude/claude-stream-json-connection.ts index d6511a998630..d5ac1ede0935 100644 --- a/src/main/claude/claude-stream-json-connection.ts +++ b/src/main/claude/claude-stream-json-connection.ts @@ -23,6 +23,7 @@ import { createClaudeUserMessageQueue } from './claude-agent-sdk-user-message-queue' import type { ClaudeStructuredSdkOptions } from './claude-structured-launch-resolution' +import { providerStderrForDisplay } from '../codex/provider-spawn-failure-report' export { ClaudeControlRequestError } @@ -102,7 +103,7 @@ export type ClaudeStreamJsonConnection = ClaudeControlSurface & { type ExitStatus = { code: number | null; signal: NodeJS.Signals | null } function exitError(stderrTail: string, status: ExitStatus | null, cause?: Error): Error { - const detail = stderrTail.trim() + const detail = providerStderrForDisplay(stderrTail).trim() // The status is the diagnostic a signed-out or refused start leaves behind; // it has to survive every wrapper between here and the user. const how = diff --git a/src/main/codex/codex-app-server-exit-error.test.ts b/src/main/codex/codex-app-server-exit-error.test.ts index 63ec9a63743a..ed0447792bf5 100644 --- a/src/main/codex/codex-app-server-exit-error.test.ts +++ b/src/main/codex/codex-app-server-exit-error.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { providerDiagnosticOf } from '../../shared/agent-session-failure' import { buildCodexAppServerExitError } from './codex-app-server-exit-error' +import { PROVIDER_SPAWN_FAILURE_MARKER } from './provider-spawn-failure-report' describe('buildCodexAppServerExitError', () => { it("keeps the stderr tail apart from Orca's wording, as log text", () => { @@ -17,4 +18,16 @@ describe('buildCodexAppServerExitError', () => { expect(providerDiagnosticOf(error)).toBeUndefined() expect(providerDiagnosticOf(buildCodexAppServerExitError(''))).toBeUndefined() }) + + it('shows a supervisor spawn failure as the spawn error a direct spawn gives', () => { + const report = JSON.stringify({ + thrown: false, + code: 'ENOENT', + message: 'spawn /opt/codex ENOENT' + }) + const error = buildCodexAppServerExitError(`${PROVIDER_SPAWN_FAILURE_MARKER}${report}\n`) + + expect(error.message).toBe('codex app-server connection ended: spawn /opt/codex ENOENT') + expect(providerDiagnosticOf(error)?.text).toBe('spawn /opt/codex ENOENT') + }) }) diff --git a/src/main/codex/codex-app-server-exit-error.ts b/src/main/codex/codex-app-server-exit-error.ts index 8812e6698a08..ca95e1d6fe05 100644 --- a/src/main/codex/codex-app-server-exit-error.ts +++ b/src/main/codex/codex-app-server-exit-error.ts @@ -5,11 +5,12 @@ import { providerDiagnostic, withProviderDiagnostic } from '../../shared/agent-session-failure' import { stderrIndicatesMissingAppServer } from './codex-app-server-capability-signal' import { CodexAppServerUnsupportedError } from './codex-app-server-session' +import { providerStderrForDisplay } from './provider-spawn-failure-report' const EXIT_DETAIL_MAX_CHARS = 400 export function buildCodexAppServerExitError(stderrTail: string, cause?: Error): Error { - const tail = stderrTail.trim().slice(0, EXIT_DETAIL_MAX_CHARS) + const tail = providerStderrForDisplay(stderrTail).trim().slice(0, EXIT_DETAIL_MAX_CHARS) // The tail is Codex's own stderr: a log, kept behind Details. const diagnostic = providerDiagnostic(tail, 'log') if (stderrIndicatesMissingAppServer(stderrTail)) { diff --git a/src/main/codex/codex-app-server-posix-supervisor.test.ts b/src/main/codex/codex-app-server-posix-supervisor.test.ts index 735db7ef11c5..9987fa33bdb6 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.test.ts @@ -7,9 +7,7 @@ import { PROVIDER_STDIN_END_GRACE_MS, PROVIDER_SUPERVISOR_MAX_STOP_MS, stopSupervisedProvider, - PROVIDER_SPAWN_FAILURE_MARKER, - supervisedPosixLaunch, - supervisedProviderSpawnFailure + supervisedPosixLaunch } from './codex-app-server-posix-supervisor' const launch: CodexAppServerLaunch = { @@ -103,34 +101,6 @@ describe('structured provider supervision', () => { } }) - const report = (thrown: boolean, code: string, message: string): string => - `${PROVIDER_SPAWN_FAILURE_MARKER}${JSON.stringify({ thrown, code, message })}\n` - - it.each([ - ['an emitted ENOENT', report(false, 'ENOENT', 'spawn /opt/my tools/claude ENOENT'), false], - ['a thrown ENOTDIR', report(true, 'ENOTDIR', 'spawn ENOTDIR'), true], - [ - 'a report after a runtime warning', - `Warning: Ignoring extra certs from \`/missing.pem\`, load failed\n${report(false, 'EACCES', 'spawn /opt/claude EACCES')}`, - false - ] - ])('reads %s from the supervisor last stderr line', (_, stderr, thrown) => { - const failure = supervisedProviderSpawnFailure(127, stderr) - - expect(failure?.thrown).toBe(thrown) - expect(failure?.error.code).toMatch(/^E[A-Z]+$/) - expect(failure?.error.message).toMatch(/^spawn /) - }) - - it.each([ - ['another exit code', 1, report(false, 'ENOENT', 'spawn claude ENOENT')], - ['a provider line after the report', 127, `${report(true, 'ENOEXEC', 'spawn ENOEXEC')}more\n`], - ['an unmarked spawn line', 127, 'spawn claude ENOENT\n'], - ['a malformed report', 127, `${PROVIDER_SPAWN_FAILURE_MARKER}{"thrown":true}\n`] - ])('reads no spawn failure from %s', (_, code, stderr) => { - expect(supervisedProviderSpawnFailure(code, stderr)).toBeNull() - }) - it.each([ [true, PROVIDER_SUPERVISOR_MAX_STOP_MS + 500], [false, 1_500] diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index 697623498836..b90324a587a0 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -1,5 +1,6 @@ import type { CodexAppServerLaunch } from './codex-app-server-connection' import { waitForProcessExitUntil } from './codex-process-exit-deadline' +import { PROVIDER_SPAWN_FAILURE_MARKER } from './provider-spawn-failure-report' /** Time the provider gets to exit on its own after its stdin ends, before SIGTERM. */ export const PROVIDER_STDIN_END_GRACE_MS = 1_000 @@ -20,9 +21,6 @@ export const PROVIDER_OUTPUT_DRAIN_TIMEOUT_MS = 1_000 export const PROVIDER_SUPERVISOR_MAX_STOP_MS = PROVIDER_STDIN_END_GRACE_MS + PROVIDER_SIGTERM_GRACE_MS + PROVIDER_GROUP_REAP_TIMEOUT_MS -/** Starts the one stderr line that reports a provider the supervisor could not start. */ -export const PROVIDER_SPAWN_FAILURE_MARKER = '[orca-provider-supervisor] spawn failed: ' - /** Inline supervisor source kept dependency-free for the spawned Node child. */ export const POSIX_PROVIDER_SUPERVISOR_SCRIPT = ` const { spawn } = require('node:child_process') @@ -163,39 +161,6 @@ child.once('exit', (code, signal) => { */ export type ProviderSupervisorLifetime = 'session' | 'one-shot' -/** A provider the supervisor could not start: `thrown` when a direct spawn would have thrown. */ -export type SupervisedProviderSpawnFailure = { thrown: boolean; error: NodeJS.ErrnoException } - -/** The failure a supervisor reported on its last stderr line before exiting 127; null otherwise. */ -export function supervisedProviderSpawnFailure( - code: number | null, - stderr: string -): SupervisedProviderSpawnFailure | null { - const line = stderr.trimEnd().split('\n').at(-1) ?? '' - if (code !== 127 || !line.startsWith(PROVIDER_SPAWN_FAILURE_MARKER)) { - return null - } - let report: unknown - try { - report = JSON.parse(line.slice(PROVIDER_SPAWN_FAILURE_MARKER.length)) - } catch { - return null - } - if ( - !report || - typeof report !== 'object' || - !('thrown' in report && typeof report.thrown === 'boolean') || - !('code' in report && typeof report.code === 'string') || - !('message' in report && typeof report.message === 'string') - ) { - return null - } - return { - thrown: report.thrown, - error: Object.assign(new Error(report.message), { code: report.code }) - } -} - export type ProviderStopInput = { /** Asks the child to stop: a SIGTERM, or the stdin end that closes a session. */ request: () => void diff --git a/src/main/codex/provider-spawn-failure-report.test.ts b/src/main/codex/provider-spawn-failure-report.test.ts new file mode 100644 index 000000000000..2c02a9ba8d31 --- /dev/null +++ b/src/main/codex/provider-spawn-failure-report.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' +import { + PROVIDER_SPAWN_FAILURE_MARKER, + providerStderrForDisplay, + supervisedProviderSpawnFailure +} from './provider-spawn-failure-report' + +describe('provider spawn failure report', () => { + const report = (thrown: boolean, code: string, message: string): string => + `${PROVIDER_SPAWN_FAILURE_MARKER}${JSON.stringify({ thrown, code, message })}\n` + + it.each([ + ['an emitted ENOENT', report(false, 'ENOENT', 'spawn /opt/my tools/claude ENOENT'), false], + ['a thrown ENOTDIR', report(true, 'ENOTDIR', 'spawn ENOTDIR'), true], + [ + 'a report after a runtime warning', + `Warning: Ignoring extra certs from \`/missing.pem\`, load failed\n${report(false, 'EACCES', 'spawn /opt/claude EACCES')}`, + false + ] + ])('reads %s from the supervisor last stderr line', (_, stderr, thrown) => { + const failure = supervisedProviderSpawnFailure(127, stderr) + + expect(failure?.thrown).toBe(thrown) + expect(failure?.error.code).toMatch(/^E[A-Z]+$/) + expect(failure?.error.message).toMatch(/^spawn /) + }) + + it.each([ + ['another exit code', 1, report(false, 'ENOENT', 'spawn claude ENOENT')], + ['a provider line after the report', 127, `${report(true, 'ENOEXEC', 'spawn ENOEXEC')}more\n`], + ['an unmarked spawn line', 127, 'spawn claude ENOENT\n'], + ['a malformed report', 127, `${PROVIDER_SPAWN_FAILURE_MARKER}{"thrown":true}\n`] + ])('reads no spawn failure from %s', (_, code, stderr) => { + expect(supervisedProviderSpawnFailure(code, stderr)).toBeNull() + }) + + it('shows a report as the spawn error a direct spawn gives, and other stderr unchanged', () => { + expect( + providerStderrForDisplay( + `Warning: a runtime notice\n${report(false, 'ENOENT', 'spawn /opt/claude ENOENT')}` + ) + ).toBe('Warning: a runtime notice\nspawn /opt/claude ENOENT\n') + expect(providerStderrForDisplay('Error: auth failed\n')).toBe('Error: auth failed\n') + }) +}) diff --git a/src/main/codex/provider-spawn-failure-report.ts b/src/main/codex/provider-spawn-failure-report.ts new file mode 100644 index 000000000000..d959e8068364 --- /dev/null +++ b/src/main/codex/provider-spawn-failure-report.ts @@ -0,0 +1,52 @@ +/** Starts the one stderr line a provider supervisor writes when it could not start its provider. */ +export const PROVIDER_SPAWN_FAILURE_MARKER = '[orca-provider-supervisor] spawn failed: ' + +type SpawnFailureReport = { thrown: boolean; code: string; message: string } + +function parseSpawnFailureReport(line: string): SpawnFailureReport | null { + if (!line.startsWith(PROVIDER_SPAWN_FAILURE_MARKER)) { + return null + } + let report: unknown + try { + report = JSON.parse(line.slice(PROVIDER_SPAWN_FAILURE_MARKER.length)) + } catch { + return null + } + if ( + !report || + typeof report !== 'object' || + !('thrown' in report && typeof report.thrown === 'boolean') || + !('code' in report && typeof report.code === 'string') || + !('message' in report && typeof report.message === 'string') + ) { + return null + } + return { thrown: report.thrown, code: report.code, message: report.message } +} + +/** A provider the supervisor could not start: `thrown` when a direct spawn would have thrown. */ +export type SupervisedProviderSpawnFailure = { thrown: boolean; error: NodeJS.ErrnoException } + +/** The failure a supervisor reported on its last stderr line before exiting 127; null otherwise. */ +export function supervisedProviderSpawnFailure( + code: number | null, + stderr: string +): SupervisedProviderSpawnFailure | null { + const report = + code === 127 ? parseSpawnFailureReport(stderr.trimEnd().split('\n').at(-1) ?? '') : null + return report + ? { + thrown: report.thrown, + error: Object.assign(new Error(report.message), { code: report.code }) + } + : null +} + +/** Provider stderr as a direct spawn leaves it: a supervisor's report reads as Node's spawn error. */ +export function providerStderrForDisplay(stderr: string): string { + return stderr + .split('\n') + .map((line) => parseSpawnFailureReport(line)?.message ?? line) + .join('\n') +} diff --git a/src/main/text-generation/commit-message-model-discovery.ts b/src/main/text-generation/commit-message-model-discovery.ts index 7b7fa6f3f796..a672e84d530f 100644 --- a/src/main/text-generation/commit-message-model-discovery.ts +++ b/src/main/text-generation/commit-message-model-discovery.ts @@ -4,7 +4,7 @@ import type { CommitMessagePlan } from '../../shared/commit-message-plan' import { getAgentModelProbeSpec } from '../../shared/agent-model-probe-spec' import type { TuiAgent } from '../../shared/tui-agent' import { resolveCodexHomeProcessLockKeyForSpawnEnv } from '../codex-cli/codex-home-process-lock' -import { supervisedProviderSpawnFailure } from '../codex/codex-app-server-posix-supervisor' +import { supervisedProviderSpawnFailure } from '../codex/provider-spawn-failure-report' import { isSshRequestOutcomeUnverifiable } from '../ssh/ssh-channel-multiplexer' import { WINDOWS_BATCH_UNSAFE_ARGUMENTS_ERROR } from '../win32-utils' import { diff --git a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts index 8fece13fb86d..10d1d88451ad 100644 --- a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts +++ b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts @@ -9,7 +9,7 @@ import { discoverCommitMessageModelsLocal, discoverCommitMessageModelsRemote } from './commit-message-text-generation' -import { PROVIDER_SPAWN_FAILURE_MARKER } from '../codex/codex-app-server-posix-supervisor' +import { PROVIDER_SPAWN_FAILURE_MARKER } from '../codex/provider-spawn-failure-report' import { createChildTerminationExpectation, createMockDiscoveryChild, diff --git a/src/main/text-generation/source-control-local-process.ts b/src/main/text-generation/source-control-local-process.ts index 13c95d2b3c0d..516e84df253e 100644 --- a/src/main/text-generation/source-control-local-process.ts +++ b/src/main/text-generation/source-control-local-process.ts @@ -1,8 +1,6 @@ import type { CommitMessagePlan } from '../../shared/commit-message-plan' -import { - stopSupervisedProvider, - supervisedProviderSpawnFailure -} from '../codex/codex-app-server-posix-supervisor' +import { stopSupervisedProvider } from '../codex/codex-app-server-posix-supervisor' +import { supervisedProviderSpawnFailure } from '../codex/provider-spawn-failure-report' import { terminateCodexAppServerProcessTree } from '../codex/codex-app-server-process-teardown' import { UnsafeWindowsBatchArgumentsError } from '../win32-utils' import { terminateWindowsProcessTree } from '../windows-process-tree-kill' From 0c8ebc46d63d2d4c415a019774c0c9f8b710d960 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:58:07 -0700 Subject: [PATCH 15/38] fix(supervisor): report a spawn that failed without a pid instead of crashing on its missing pipes When the provider spawn fails outright (EMFILE, ENFILE), Node emits 'error' later and leaves the child with no pid and no stdio. Piping stdin into the missing pipe threw first, so the supervisor died with exit 1 and a stack trace and never wrote its spawn-failure report. The pipes are now wired only for a provider that started. --- ...erver-posix-supervisor.integration.test.ts | 40 ++++++++++++++++++- .../codex-app-server-posix-supervisor.ts | 15 ++++--- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts index be96ba07b1e7..108d0460b074 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts @@ -10,6 +10,7 @@ import { supervisedPosixLaunch, type ProviderSupervisorOptions } from './codex-app-server-posix-supervisor' +import { supervisedProviderSpawnFailure } from './provider-spawn-failure-report' // The provider leads its own group; its grandchild shares that group and ignores SIGTERM. const PROVIDER = String.raw` @@ -101,6 +102,19 @@ const SIGNAL_AFTER_SPAWN_PRELOAD = String.raw` } ` +// Preloaded into the supervisor: its spawn fails the way EMFILE does, with no pid and no pipes. +const SPAWN_WITHOUT_PID_PRELOAD = String.raw` + const childProcess = require('node:child_process') + const { EventEmitter } = require('node:events') + childProcess.spawn = (command) => { + const child = Object.assign(new EventEmitter(), { pid: undefined, stdin: null, stdout: null, stderr: null }) + process.nextTick(() => + child.emit('error', Object.assign(new Error('spawn ' + command + ' EMFILE'), { code: 'EMFILE' })) + ) + return child + } +` + const recordedPids = new Set() const tempDirs: string[] = [] @@ -166,12 +180,13 @@ function launchSupervisor( command: process.execPath, args: ['-e', PROVIDER] }, - nodeArgs: string[] = [] + nodeArgs: string[] = [], + stderr: 'pipe' | 'ignore' = 'ignore' ): { supervisor: ChildProcess; exit: Promise<{ code: number | null; signal: string | null }> } { const launch = supervisedPosixLaunch(provider, { ...process.env, ...env }, options) const supervisor = spawn(launch.command, [...nodeArgs, ...launch.args], { env: launch.env, - stdio: ['pipe', 'pipe', 'ignore'], + stdio: ['pipe', 'pipe', stderr], detached: true }) recordedPids.add(supervisor.pid!) @@ -447,6 +462,27 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe expect(stdout).toBe(nodeOptions) }) + it('reports a spawn that failed without a pid through the marked line', async () => { + const preload = join(tempDir(), 'spawn-without-pid.js') + writeFileSync(preload, SPAWN_WITHOUT_PID_PRELOAD) + const { supervisor, exit } = launchSupervisor( + { lifetime: 'one-shot' }, + {}, + { command: '/opt/agent', args: [] }, + ['--require', preload], + 'pipe' + ) + let stderr = '' + supervisor.stderr!.on('data', (chunk: Buffer) => (stderr += chunk.toString())) + supervisor.stdin!.end() + + await expect(exit).resolves.toEqual({ code: 127, signal: null }) + expect(supervisedProviderSpawnFailure(127, stderr)).toMatchObject({ + thrown: false, + error: { code: 'EMFILE', message: 'spawn /opt/agent EMFILE' } + }) + }) + it('hands the provider a near-cap argv prompt intact', async () => { const prompt = 'x'.repeat(110 * 1024) const { supervisor, exit } = launchSupervisor( diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index b90324a587a0..753be8b6561e 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -115,12 +115,15 @@ if (spec.lifetime !== 'one-shot') { process.stdin.once('end', scheduleOwnerShutdown) process.stdin.once('close', scheduleOwnerShutdown) } -process.stdin.pipe(child.stdin) -child.stdout.pipe(process.stdout) -child.stderr.pipe(process.stderr) -// A dead owner's stdout pipe raises EPIPE; unhandled, it would end this pid before the group. -for (const stream of [process.stdin, process.stdout, process.stderr, child.stdin, child.stdout, child.stderr]) { - stream.on('error', () => {}) +// A spawn that failed outright (EMFILE, ENFILE) has no pid and no pipes; its 'error' reports it. +if (child.pid) { + process.stdin.pipe(child.stdin) + child.stdout.pipe(process.stdout) + child.stderr.pipe(process.stderr) + // A dead owner's stdout pipe raises EPIPE; unhandled, it would end this pid before the group. + for (const stream of [process.stdin, process.stdout, process.stderr, child.stdin, child.stdout, child.stderr]) { + stream.on('error', () => {}) + } } // Exit can land before the provider's last output is relayed; a one-shot's answer is that output. const drainProviderOutput = () => { From ed5201b1c8433f900756eba1e6b77e7a7685160d Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:02:59 -0700 Subject: [PATCH 16/38] refactor(supervisor): share the stop of a supervised child process Agent one-shots stop their supervisor with SIGTERM through the shared stop ladder, watching the child's own exit. That adapter moves into one helper beside the ladder, so other supervised children can use it with their own stop request instead of a copy. The caller's breadcrumb site still reaches the forced teardown. Same request, wait and force as before. --- .../codex/supervised-child-process-stop.ts | 33 +++++++++++++++++++ .../source-control-local-process.ts | 25 ++------------ 2 files changed, 35 insertions(+), 23 deletions(-) create mode 100644 src/main/codex/supervised-child-process-stop.ts diff --git a/src/main/codex/supervised-child-process-stop.ts b/src/main/codex/supervised-child-process-stop.ts new file mode 100644 index 000000000000..b202e1a79069 --- /dev/null +++ b/src/main/codex/supervised-child-process-stop.ts @@ -0,0 +1,33 @@ +import type { ChildProcessHandle } from '../../shared/child-process/process-spec' +import { stopSupervisedProvider } from './codex-app-server-posix-supervisor' +import { terminateCodexAppServerProcessTree } from './codex-app-server-process-teardown' + +type SupervisedChild = Pick + +/** + * Stops a provider supervisor child: `request` asks it to stop (SIGTERM unless given), and its + * tree is forced, filed under `site`, only after the supervisor's full stop time. True when forced. + */ +export async function stopSupervisedChildProcess( + child: SupervisedChild, + request: () => void = () => child.kill('SIGTERM'), + site?: string +): Promise { + const exited = (): boolean => child.exitCode !== null || child.signalCode !== null + if (exited()) { + return false + } + return stopSupervisedProvider({ + request: () => { + try { + request() + } catch { + // The process may exit between the exit check and the request. + } + }, + exitPromise: new Promise((resolve) => child.once('exit', () => resolve())), + exited, + force: () => terminateCodexAppServerProcessTree(child, site === undefined ? {} : { site }), + supervised: true + }) +} diff --git a/src/main/text-generation/source-control-local-process.ts b/src/main/text-generation/source-control-local-process.ts index 516e84df253e..9dfa0ea88d43 100644 --- a/src/main/text-generation/source-control-local-process.ts +++ b/src/main/text-generation/source-control-local-process.ts @@ -1,7 +1,6 @@ import type { CommitMessagePlan } from '../../shared/commit-message-plan' -import { stopSupervisedProvider } from '../codex/codex-app-server-posix-supervisor' import { supervisedProviderSpawnFailure } from '../codex/provider-spawn-failure-report' -import { terminateCodexAppServerProcessTree } from '../codex/codex-app-server-process-teardown' +import { stopSupervisedChildProcess } from '../codex/supervised-child-process-stop' import { UnsafeWindowsBatchArgumentsError } from '../win32-utils' import { terminateWindowsProcessTree } from '../windows-process-tree-kill' import { @@ -35,7 +34,7 @@ export async function killSourceControlAgentProcess( return } if (child.supervised) { - await stopSupervisedAgent(child) + await stopSupervisedChildProcess(child, undefined, SOURCE_CONTROL_KILL_SITE) return } if (process.platform === 'win32') { @@ -52,26 +51,6 @@ export async function killSourceControlAgentProcess( } } -async function stopSupervisedAgent(child: SpawnedSourceControlAgentProcess): Promise { - const exited = (): boolean => child.exitCode !== null || child.signalCode !== null - if (exited()) { - return - } - await stopSupervisedProvider({ - request: () => { - try { - child.kill('SIGTERM') - } catch { - // The process may exit between the exit check and kill. - } - }, - exitPromise: new Promise((resolve) => child.once('exit', () => resolve())), - exited, - force: () => terminateCodexAppServerProcessTree(child, { site: SOURCE_CONTROL_KILL_SITE }), - supervised: true - }) -} - // Why: Windows caps the CreateProcess command line at 32,767 UTF-16 code units, // including the executable path, per-arg quoting, and separators. The budget // leaves headroom for cmd.exe `/d /c` shim wrappers. From 1cd119523513a705abfb7ad992e9d0ca95e76c2c Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:10:00 -0700 Subject: [PATCH 17/38] fix(codex,opencode): file forced backfill and preflight teardowns under their own breadcrumb sites A forced teardown of the Codex backfill app-server is filed under 'codex-state-db-backfill-recovery', and one of the OpenCode launch model preflight under 'opencode-launch-model-preflight', instead of the generic 'codex-app-server-teardown'. --- .../codex-state-db-backfill-recovery-process.test.ts | 4 +++- .../codex/codex-state-db-backfill-recovery-process.ts | 4 +++- .../opencode-launch-model-context-cleanup.test.ts | 4 +++- src/main/opencode/opencode-launch-model-context.ts | 8 +++++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/main/codex/codex-state-db-backfill-recovery-process.test.ts b/src/main/codex/codex-state-db-backfill-recovery-process.test.ts index ef25b1db1949..2a18deded9b8 100644 --- a/src/main/codex/codex-state-db-backfill-recovery-process.test.ts +++ b/src/main/codex/codex-state-db-backfill-recovery-process.test.ts @@ -67,7 +67,9 @@ describe('stopCodexBackfillRecoveryProcess for a supervised app-server', () => { await vi.advanceTimersByTimeAsync(1) await stopped - expect(terminateCodexAppServerProcessTree).toHaveBeenCalledWith(child) + expect(terminateCodexAppServerProcessTree).toHaveBeenCalledWith(child, { + site: 'codex-state-db-backfill-recovery' + }) // Its own SIGKILL would kill the supervisor mid-ladder and orphan the app-server's group. expect(child.kill).not.toHaveBeenCalledWith('SIGKILL') }) diff --git a/src/main/codex/codex-state-db-backfill-recovery-process.ts b/src/main/codex/codex-state-db-backfill-recovery-process.ts index ac9aa0899a09..e4afde3e2546 100644 --- a/src/main/codex/codex-state-db-backfill-recovery-process.ts +++ b/src/main/codex/codex-state-db-backfill-recovery-process.ts @@ -8,6 +8,8 @@ import { createProviderSpawnSpec } from './codex-app-server-posix-supervisor' import type { CodexAppServerSpawn } from './codex-app-server-process-tree-kill' import { stopSupervisedChildProcess } from './supervised-child-process-stop' +const BACKFILL_RECOVERY_KILL_SITE = 'codex-state-db-backfill-recovery' + export type CodexBackfillRecoveryProcess = { child: ChildProcessHandle; supervised: boolean } /** Starts the read-only app-server whose presence lets Codex finish its own backfill. */ @@ -57,7 +59,7 @@ export async function stopCodexBackfillRecoveryProcess( ): Promise { if (supervised) { // A session supervisor turns stdin end into its group stop, as a Codex connection close does. - await stopSupervisedChildProcess(child, () => child.stdin?.end()) + await stopSupervisedChildProcess(child, () => child.stdin?.end(), BACKFILL_RECOVERY_KILL_SITE) return } await terminateCodexProbeChild(child) diff --git a/src/main/opencode/opencode-launch-model-context-cleanup.test.ts b/src/main/opencode/opencode-launch-model-context-cleanup.test.ts index c1e12b0dc293..a86775ae5c61 100644 --- a/src/main/opencode/opencode-launch-model-context-cleanup.test.ts +++ b/src/main/opencode/opencode-launch-model-context-cleanup.test.ts @@ -176,7 +176,9 @@ describe('OpenCode model probe termination evidence', () => { await vi.advanceTimersByTimeAsync(1) expect(await probe).toMatchObject({ primaryAgent: 'build' }) - expect(terminateCodexAppServerProcessTree).toHaveBeenCalledWith(child) + expect(terminateCodexAppServerProcessTree).toHaveBeenCalledWith(child, { + site: 'opencode-launch-model-preflight' + }) }) it('never forces a POSIX supervisor whose root exited without closing its pipes', async () => { diff --git a/src/main/opencode/opencode-launch-model-context.ts b/src/main/opencode/opencode-launch-model-context.ts index e45eb32f9b5d..49e74a18ffd7 100644 --- a/src/main/opencode/opencode-launch-model-context.ts +++ b/src/main/opencode/opencode-launch-model-context.ts @@ -16,6 +16,8 @@ import { } from '../codex/codex-app-server-posix-supervisor' import { terminateCodexAppServerProcessTree } from '../codex/codex-app-server-process-teardown' +const OPENCODE_PREFLIGHT_KILL_SITE = 'opencode-launch-model-preflight' + const model = z.object({ id: z.string(), providerID: z.string() }) const availableModel = model.extend({ enabled: z.boolean() }) const agent = z.object({ @@ -219,7 +221,11 @@ export async function probeOpenCodeLaunchModelContext(options: { exited: () => childClosed, force: async () => { // A reaped root's PID can be reused; only a live supervisor's tree is ours to force. - forcedStop = !rootExited && (await terminateCodexAppServerProcessTree(child)) + forcedStop = + !rootExited && + (await terminateCodexAppServerProcessTree(child, { + site: OPENCODE_PREFLIGHT_KILL_SITE + })) }, supervised: true }) From 10c154dee98ad014af4b39d12f6053acaa5ee6b1 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:10:00 -0700 Subject: [PATCH 18/38] test(codex): write the stand-in pid report atomically A loaded host let the test read the pid file between its creation and its write (Unexpected end of JSON input); the stand-in now renames it into place. --- src/main/codex/supervised-probe-owner.test-fixture.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/codex/supervised-probe-owner.test-fixture.ts b/src/main/codex/supervised-probe-owner.test-fixture.ts index 5d3a75827732..a83a512755e0 100644 --- a/src/main/codex/supervised-probe-owner.test-fixture.ts +++ b/src/main/codex/supervised-probe-owner.test-fixture.ts @@ -22,10 +22,10 @@ const grandchild = spawn( { stdio: ['ignore', 'pipe', 'ignore'] } ) grandchild.stdout.once('data', () => { - fs.writeFileSync( - process.env.ORCA_TEST_PID_FILE, - JSON.stringify({ provider: process.pid, grandchild: grandchild.pid }) - ) + // Renamed into place, so a reader never sees a half-written file. + const partial = process.env.ORCA_TEST_PID_FILE + '.partial' + fs.writeFileSync(partial, JSON.stringify({ provider: process.pid, grandchild: grandchild.pid })) + fs.renameSync(partial, process.env.ORCA_TEST_PID_FILE) }) setInterval(() => {}, 60000) ${body} From e4071abcaaf69651e3030ce818b5a77b56ff98ce Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:14:08 -0700 Subject: [PATCH 19/38] fix(supervisor): give a session provider its stdin end and grace when its owner dies The owner-death watch went straight to the group SIGTERM and cancelled any stdin-end grace, so when Orca quit or crashed a session provider such as the Codex app-server never saw the EOF that lets it finish writing its state (auth.json, the state database). A session whose owner is gone now closes as an owner's stdin end does: the provider's stdin is ended, it gets the stdin-end grace, and only then the SIGTERM and SIGKILL ladder. A one-shot already had its EOF at the end of its request, so its owner's death still stops it at once. --- ...erver-posix-supervisor.integration.test.ts | 62 ++++++++++++++++++- .../codex-app-server-posix-supervisor.ts | 8 ++- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts index 108d0460b074..8c3b471e32ba 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts @@ -41,6 +41,20 @@ const EXITS_ON_STDIN_END_PROVIDER = String.raw` process.stdout.write(JSON.stringify({ provider: process.pid }) + '\n') ` +// Flushes for a moment once its stdin ends, then exits; records a SIGTERM if one arrives first. +const FLUSHES_ON_STDIN_END_PROVIDER = String.raw` + const { writeFileSync } = require('node:fs') + process.on('SIGTERM', () => { + writeFileSync(process.env.ORCA_TEST_PROVIDER_SIGNAL_FILE, 'SIGTERM') + process.exit(143) + }) + process.stdin.on('end', () => setTimeout(() => { + writeFileSync(process.env.ORCA_TEST_PROVIDER_FLUSH_FILE, 'flushed') + process.exit(0) + }, 200)).resume() + process.stdout.write(JSON.stringify({ provider: process.pid }) + '\n') +` + // Ignores stdin end and SIGTERM, recording when SIGTERM arrived, so only SIGKILL ends it. const RECORDS_SIGTERM_PROVIDER = String.raw` process.on('SIGTERM', () => { @@ -198,10 +212,14 @@ function launchSupervisor( async function launchUnderOwner( options: ProviderSupervisorOptions, - env: Record = {} + env: Record = {}, + provider: { script: string; pids: readonly string[] } = { + script: PROVIDER, + pids: ['provider', 'grandchild'] + } ): Promise<{ owner: ChildProcess; pids: Record }> { const launch = supervisedPosixLaunch( - { command: process.execPath, args: ['-e', PROVIDER] }, + { command: process.execPath, args: ['-e', provider.script] }, { ...process.env, ...env }, options ) @@ -210,7 +228,7 @@ async function launchUnderOwner( stdio: ['ignore', 'pipe', 'ignore'] }) recordedPids.add(owner.pid!) - const pids = await readPids(owner, ['supervisor', 'provider', 'grandchild']) + const pids = await readPids(owner, ['supervisor', ...provider.pids]) return { owner, pids } } @@ -383,6 +401,44 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe expect(existsSync(signalFile) && readFileSync(signalFile, 'utf8')).toBe('SIGTERM') }) + it('gives a session provider its stdin end and grace when its owner dies', async () => { + const dir = tempDir() + const signalFile = join(dir, 'provider-signal') + const flushFile = join(dir, 'provider-flushed') + const { owner, pids } = await launchUnderOwner( + {}, + { ORCA_TEST_PROVIDER_SIGNAL_FILE: signalFile, ORCA_TEST_PROVIDER_FLUSH_FILE: flushFile }, + { script: FLUSHES_ON_STDIN_END_PROVIDER, pids: ['provider'] } + ) + + owner.kill('SIGKILL') + + expect(await waitFor(() => !alive(pids.provider), PROVIDER_STDIN_END_GRACE_MS)).toBe(true) + expect(await waitFor(() => !alive(pids.supervisor), 3_000)).toBe(true) + expect(existsSync(flushFile)).toBe(true) + expect(existsSync(signalFile)).toBe(false) + }) + + it('stops a session provider that ignores its stdin end once the grace after owner death ends', async () => { + const signalFile = join(tempDir(), 'provider-sigterm-at') + const stdinEndGraceMs = 400 + const { owner, pids } = await launchUnderOwner( + { stdinEndGraceMs, sigtermGraceMs: 200 }, + { ORCA_TEST_PROVIDER_SIGNAL_FILE: signalFile }, + { script: RECORDS_SIGTERM_PROVIDER, pids: ['provider'] } + ) + + const killedAt = Date.now() + owner.kill('SIGKILL') + + expect(await waitFor(() => !alive(-pids.provider), 3_000)).toBe(true) + // Timers may fire a tick early against another process's clock. + expect(Number(readFileSync(signalFile, 'utf8')) - killedAt).toBeGreaterThanOrEqual( + stdinEndGraceMs - 20 + ) + expect(await waitFor(() => !alive(pids.supervisor), 3_000)).toBe(true) + }) + describe('one-shot lifetime', () => { it('lets a one-shot answer after its stdin ends and relays its exit code', async () => { const signalFile = join(tempDir(), 'provider-signal') diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index 753be8b6561e..d171b629a299 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -146,7 +146,13 @@ const reapProviderExit = async (code, signal) => { finishWithProviderOutcome(code, signal) } timer = setInterval(() => { - if (ownerGone()) stopProviderGroup(null) + if (!ownerGone()) return + if (spec.lifetime === 'one-shot') return stopProviderGroup(null) + // A session whose owner is gone closes as an owner's stdin end does: EOF, its grace, then the stop. + clearInterval(timer) + process.stdin.unpipe(child.stdin) + try { child.stdin.end() } catch {} + scheduleOwnerShutdown() }, 100) timer.unref() child.once('error', (error) => { From 80e4ceaf13909a6d1dc6a033f3290348f9c59e0f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:17:32 -0700 Subject: [PATCH 20/38] fix(opencode): stop the preflight server through the shared supervised stop, and trust only a proven stop The preflight's own stop wrapper waited on the supervisor's pipes and counted a forced teardown as proof, though the teardown reports success even when it found no descendants to check, and a supervisor that failed to reap its group exits 1 with its pipes closed. It now uses stopSupervisedChildProcess with its breadcrumb site, and counts the server stopped only when the supervisor ended on its stop signal or relayed the server's own exit. A forced stop, or an exit of 1, returns no context, as an unverified stop did before. --- ...ncode-launch-model-context-cleanup.test.ts | 39 +++++++++++-- .../opencode/opencode-launch-model-context.ts | 55 ++++++++++--------- 2 files changed, 63 insertions(+), 31 deletions(-) diff --git a/src/main/opencode/opencode-launch-model-context-cleanup.test.ts b/src/main/opencode/opencode-launch-model-context-cleanup.test.ts index a86775ae5c61..31e784d4cd10 100644 --- a/src/main/opencode/opencode-launch-model-context-cleanup.test.ts +++ b/src/main/opencode/opencode-launch-model-context-cleanup.test.ts @@ -29,6 +29,7 @@ class ProbeChild extends ChildProcess { override stderr = new PassThrough() override pid = 123456 override exitCode: number | null = null + override signalCode: NodeJS.Signals | null = null // A handle-less ChildProcess's own kill can signal this test's process group. override kill = vi.fn((_signal?: NodeJS.Signals | number): boolean => true) override stdio: [PassThrough, PassThrough, PassThrough, undefined, undefined] = [ @@ -59,6 +60,13 @@ function close() { child.emit('exit', 0, null) child.emit('close', 0, null) } +/** How a supervisor ends: its stop signal re-raised once the group is gone, or an exit code. */ +function supervisorExits(code: number | null, signal: NodeJS.Signals | null) { + child.exitCode = code + child.signalCode = signal + child.emit('exit', code, signal) + child.emit('close', code, signal) +} const options = { executable: '/private/opencode', cwd: directory, env: {} } function decodedSupervisorSpec(env: unknown): unknown { @@ -106,7 +114,7 @@ describe('OpenCode model probe termination evidence', () => { vi.mocked(forceTerminateProcessTree).mockResolvedValue(false) child.kill.mockImplementation((signal) => { if (signal === 'SIGTERM') { - close() + supervisorExits(null, 'SIGTERM') } return true }) @@ -164,10 +172,14 @@ describe('OpenCode model probe termination evidence', () => { expect(terminateCodexAppServerProcessTree).not.toHaveBeenCalled() }) - it('forces a POSIX supervisor tree only after its full stop time', async () => { + it('forces a POSIX supervisor tree only after its full stop time, and trusts no forced stop', async () => { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) child.kill.mockReturnValue(true) - vi.mocked(terminateCodexAppServerProcessTree).mockResolvedValue(true) + // The teardown reports true even when it found no descendants to prove gone. + vi.mocked(terminateCodexAppServerProcessTree).mockImplementation(async () => { + supervisorExits(null, 'SIGKILL') + return true + }) const probe = probeOpenCodeLaunchModelContext(options) await untilStopRequested() @@ -175,12 +187,31 @@ describe('OpenCode model probe termination evidence', () => { expect(terminateCodexAppServerProcessTree).not.toHaveBeenCalled() await vi.advanceTimersByTimeAsync(1) - expect(await probe).toMatchObject({ primaryAgent: 'build' }) + expect(await probe).toBeNull() expect(terminateCodexAppServerProcessTree).toHaveBeenCalledWith(child, { site: 'opencode-launch-model-preflight' }) }) + it('trusts no POSIX stop whose supervisor exited 1, which a failed group reap also exits', async () => { + child.kill.mockImplementation(() => { + supervisorExits(1, null) + return true + }) + + expect(await probeOpenCodeLaunchModelContext(options)).toBeNull() + expect(terminateCodexAppServerProcessTree).not.toHaveBeenCalled() + }) + + it('accepts a POSIX supervisor that relayed the server exiting on its own', async () => { + child.kill.mockImplementation(() => { + supervisorExits(0, null) + return true + }) + + expect(await probeOpenCodeLaunchModelContext(options)).toMatchObject({ primaryAgent: 'build' }) + }) + it('never forces a POSIX supervisor whose root exited without closing its pipes', async () => { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) child.kill.mockImplementation(() => { diff --git a/src/main/opencode/opencode-launch-model-context.ts b/src/main/opencode/opencode-launch-model-context.ts index 49e74a18ffd7..dcecd6b0c8e7 100644 --- a/src/main/opencode/opencode-launch-model-context.ts +++ b/src/main/opencode/opencode-launch-model-context.ts @@ -10,13 +10,13 @@ import { withTimeout } from '../../shared/promise-timeout-fallback' import { createOutputSink } from '../../shared/child-process/bounded-output-sink' import { readFetchResponseJsonWithinLimit } from '../../shared/fetch-response-body' import { cancelUnreadResponseBody } from '../lib/unread-response-body' -import { - createProviderSpawnSpec, - stopSupervisedProvider -} from '../codex/codex-app-server-posix-supervisor' -import { terminateCodexAppServerProcessTree } from '../codex/codex-app-server-process-teardown' +import type { ChildProcessHandle } from '../../shared/child-process/process-spec' +import { createProviderSpawnSpec } from '../codex/codex-app-server-posix-supervisor' +import { stopSupervisedChildProcess } from '../codex/supervised-child-process-stop' const OPENCODE_PREFLIGHT_KILL_SITE = 'opencode-launch-model-preflight' +// A supervisor's pipes are its own, so they close as it exits. +const SUPERVISOR_PIPE_CLOSE_MS = 1_000 const model = z.object({ id: z.string(), providerID: z.string() }) const availableModel = model.extend({ enabled: z.boolean() }) @@ -208,28 +208,7 @@ export async function probeOpenCodeLaunchModelContext(options: { context = null } finally { if (spawnSpec.supervised) { - let forcedStop = false - await stopSupervisedProvider({ - request: () => { - try { - child.kill('SIGTERM') - } catch { - // The supervisor may exit between the close check and kill. - } - }, - exitPromise: closed.then(() => undefined), - exited: () => childClosed, - force: async () => { - // A reaped root's PID can be reused; only a live supervisor's tree is ours to force. - forcedStop = - !rootExited && - (await terminateCodexAppServerProcessTree(child, { - site: OPENCODE_PREFLIGHT_KILL_SITE - })) - }, - supervised: true - }) - stopped = childClosed || forcedStop + stopped = await stopSupervisedPreflight(child, closed) } else { if (!childClosed && !rootExited) { await signalProcessTree(child, 'SIGTERM') @@ -243,3 +222,25 @@ export async function probeOpenCodeLaunchModelContext(options: { } return stopped ? context : null } + +/** + * True only when the server's group is proven gone: the supervisor ended on its stop signal or + * relayed the server's own exit. Exit 1 can be a reap that failed, and a forced teardown proves + * nothing about a group the supervisor left behind. + */ +async function stopSupervisedPreflight( + child: ChildProcessHandle, + closed: Promise +): Promise { + if (!child.pid) { + // Never started, so there is nothing to stop. + return true + } + if (await stopSupervisedChildProcess(child, undefined, OPENCODE_PREFLIGHT_KILL_SITE)) { + return false + } + if (child.signalCode === null && child.exitCode === 1) { + return false + } + return withTimeout(closed, SUPERVISOR_PIPE_CLOSE_MS, false) +} From ea5e3a6549dbc2ed7d2c3a0f24f5239ddb50836a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:17:32 -0700 Subject: [PATCH 21/38] chore(codex): state the supervised stop time in the probe and trust grant deadline comments --- src/main/codex/codex-hook-trust-grant.ts | 5 +++-- src/main/codex/codex-model-catalog-probe.ts | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/codex/codex-hook-trust-grant.ts b/src/main/codex/codex-hook-trust-grant.ts index d0a2ab748dc5..1ae0624f82c3 100644 --- a/src/main/codex/codex-hook-trust-grant.ts +++ b/src/main/codex/codex-hook-trust-grant.ts @@ -45,8 +45,9 @@ import { export { CODEX_TRUST_GRANT_TRANSIENT_RETRY_INTERVAL_MS } from './codex-trust-grant-cooldown' // Why: a cold `codex app-server` on a loaded Mac took over 10 s; a background -// grant blocks no launch, so it can wait for one. The session's own kill timer -// bounds the whole grant: everything before it is synchronous on native. +// grant blocks no launch, so it can wait for one. The session deadline bounds the whole grant, +// plus up to PROVIDER_SUPERVISOR_MAX_STOP_MS for a POSIX supervisor to stop the server; +// everything before it is synchronous on native. export const CODEX_BACKGROUND_TRUST_GRANT_TIMEOUT_MS = 30_000 /** Ops escape hatch (not a setting): forces the fallback lane for every trust grant. */ diff --git a/src/main/codex/codex-model-catalog-probe.ts b/src/main/codex/codex-model-catalog-probe.ts index 2cf354259d74..66def74598c2 100644 --- a/src/main/codex/codex-model-catalog-probe.ts +++ b/src/main/codex/codex-model-catalog-probe.ts @@ -10,8 +10,8 @@ import type { AgentModelCatalogSuccess } from '../native-chat/agent-model-catalog/agent-model-catalog-store' -// Why 15s: the whole probe session is SIGKILLed at the deadline, and a cold -// `model/list` may pay one network /models fetch behind provider auth. +// Why 15s: the whole probe session is stopped at the deadline (on POSIX its supervisor then gets +// up to PROVIDER_SUPERVISOR_MAX_STOP_MS), and a cold `model/list` may pay one /models fetch. const CODEX_MODEL_CATALOG_PROBE_TIMEOUT_MS = 15_000 export type CodexModelCatalogProbeDeps = Pick< From 76999bb8cf2c733faea548cf2c906aecddaeb5ae Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:17:32 -0700 Subject: [PATCH 22/38] test(codex): assert the signals a supervised stop sends itself, not a SIGKILL the mocked teardown never could --- src/main/codex/codex-app-server-session-stop.test.ts | 4 ++-- .../codex/codex-state-db-backfill-recovery-process.test.ts | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/codex/codex-app-server-session-stop.test.ts b/src/main/codex/codex-app-server-session-stop.test.ts index 8e9998225ace..d6e8159b4236 100644 --- a/src/main/codex/codex-app-server-session-stop.test.ts +++ b/src/main/codex/codex-app-server-session-stop.test.ts @@ -102,8 +102,8 @@ describe('runCodexAppServerSession stop', () => { }) await vi.advanceTimersByTimeAsync(1_000) expect(await outcome).toBeInstanceOf(CodexAppServerTimeoutError) - // Its own group SIGKILL would kill the supervisor mid-ladder and orphan the server's group. - expect(child.kill).not.toHaveBeenCalledWith('SIGKILL') + // The session's only own signal is the deadline SIGTERM; only the teardown forces. + expect(child.kill.mock.calls).toEqual([['SIGTERM']]) expect(killCodexAppServerProcessTree).not.toHaveBeenCalled() }) diff --git a/src/main/codex/codex-state-db-backfill-recovery-process.test.ts b/src/main/codex/codex-state-db-backfill-recovery-process.test.ts index 2a18deded9b8..4a0a30bf67e8 100644 --- a/src/main/codex/codex-state-db-backfill-recovery-process.test.ts +++ b/src/main/codex/codex-state-db-backfill-recovery-process.test.ts @@ -70,7 +70,8 @@ describe('stopCodexBackfillRecoveryProcess for a supervised app-server', () => { expect(terminateCodexAppServerProcessTree).toHaveBeenCalledWith(child, { site: 'codex-state-db-backfill-recovery' }) - // Its own SIGKILL would kill the supervisor mid-ladder and orphan the app-server's group. - expect(child.kill).not.toHaveBeenCalledWith('SIGKILL') + // The stop signals nothing itself: the stdin end asks, and only the teardown forces. + expect(child.stdin.end).toHaveBeenCalledOnce() + expect(child.kill).not.toHaveBeenCalled() }) }) From c01233e7eb075d6107d7d176029d6b0ffa03cce8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:17:49 -0700 Subject: [PATCH 23/38] fix(supervisor): kill the rest of the provider group once the provider exits on a stop A requested stop waited out the whole SIGTERM grace for the provider's group even after the provider itself had exited, so a SIGTERM-ignoring helper it left behind held every stop for up to 3 s. Under a stop, the rest of the group is now SIGKILLed as soon as the provider has exited, the same rule its own exit already follows. --- ...server-posix-supervisor.integration.test.ts | 18 ++++++++++++++++-- .../codex/codex-app-server-posix-supervisor.ts | 9 ++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts index 8c3b471e32ba..fae5f9d6f18c 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts @@ -261,6 +261,20 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe expect(alive(grandchild)).toBe(false) }) + it('kills the rest of the group at once when the provider exits on a stop', async () => { + const { supervisor, exit } = launchSupervisor({}) + const { provider, grandchild } = await readPids(supervisor, ['provider', 'grandchild']) + + const signalledAt = Date.now() + supervisor.kill('SIGTERM') + + await expect(exit).resolves.toEqual({ code: null, signal: 'SIGTERM' }) + // The grandchild ignores SIGTERM; waiting out the 3 s grace for it would be the old cost. + expect(Date.now() - signalledAt).toBeLessThan(PROVIDER_SIGTERM_GRACE_MS / 3) + expect(alive(provider)).toBe(false) + expect(alive(grandchild)).toBe(false) + }) + it('escalates a SIGTERM-ignoring provider to SIGKILL after the grace from the spec', async () => { const graceMs = 200 const { supervisor, exit } = launchSupervisor( @@ -323,8 +337,8 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe owner.kill('SIGKILL') expect(await waitFor(() => !alive(-pids.provider), 3_000)).toBe(true) - // The grandchild ignores SIGTERM, so the group lasts until the grace ends in SIGKILL. - expect(Date.now() - killedAt).toBeGreaterThanOrEqual(graceMs) + // The provider ignores its stdin end, so its group lasts through the stdin-end grace. + expect(Date.now() - killedAt).toBeGreaterThanOrEqual(PROVIDER_STDIN_END_GRACE_MS - 20) expect(await waitFor(() => !alive(pids.supervisor), 3_000)).toBe(true) expect(alive(pids.grandchild)).toBe(false) }) diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index d171b629a299..3af323dc2925 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -58,6 +58,7 @@ try { let timer let ownerShutdownTimer let settling = false +let providerExited = false const providerGroupExists = () => { if (!child.pid) return false try { @@ -67,10 +68,10 @@ const providerGroupExists = () => { return Boolean(error && error.code !== 'ESRCH') } } -const waitForProviderGroupExit = async (timeoutMs) => { +const waitForProviderGroupExit = async (timeoutMs, untilProviderExits = false) => { const deadline = Date.now() + timeoutMs while (providerGroupExists()) { - if (Date.now() >= deadline) return false + if (Date.now() >= deadline || (untilProviderExits && providerExited)) return false await new Promise((resolve) => setTimeout(resolve, 25)) } return true @@ -96,7 +97,8 @@ const stopProviderGroup = (receivedSignal) => { clearInterval(timer) if (ownerShutdownTimer) clearTimeout(ownerShutdownTimer) try { process.kill(-child.pid, 'SIGTERM') } catch {} - void waitForProviderGroupExit(spec.sigtermGraceMs) + // Once the provider itself has exited, the rest of its group dies at once, as on its own exit. + void waitForProviderGroupExit(spec.sigtermGraceMs, true) .then((exited) => exited || reapOwnedProviderGroup()) .then((reaped) => { if (!reaped) return process.exit(1) @@ -160,6 +162,7 @@ child.once('error', (error) => { exitWithSpawnFailure(error, false) }) child.once('exit', (code, signal) => { + providerExited = true void reapProviderExit(code, signal) }) ` From 6800a8bfa032ff6b1ad1bebf2f62bf58107e29b5 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:21:34 -0700 Subject: [PATCH 24/38] test(text-generation): cover a supervised Codex discovery past its output limit Model discovery's supervised stop was covered only under timeout. A Codex discovery that floods past the output limit now runs through a real supervisor: it settles with the too-much-data error, its agent is stopped through the supervisor, and the next discovery on the same Codex home starts only after that agent is gone. The direct-child suites' headers now list exactly the supervised cases that are covered. --- ...ssage-text-generation-cancellation.test.ts | 4 +- ...e-text-generation-local-subprocess.test.ts | 4 +- ...ge-text-generation-model-discovery.test.ts | 4 +- .../source-control-local-process.test.ts | 57 +++++++++++++++++++ 4 files changed, 63 insertions(+), 6 deletions(-) diff --git a/src/main/text-generation/commit-message-text-generation-cancellation.test.ts b/src/main/text-generation/commit-message-text-generation-cancellation.test.ts index e0e83431337b..dc46c7c9e41a 100644 --- a/src/main/text-generation/commit-message-text-generation-cancellation.test.ts +++ b/src/main/text-generation/commit-message-text-generation-cancellation.test.ts @@ -35,8 +35,8 @@ const spawnMock = vi.mocked(spawn) const expectChildTerminated = createChildTerminationExpectation(terminateWindowsProcessTreeMock) // These suites drive fake children down the Windows direct-child path, taskkill included. The POSIX -// supervised stop under timeout, cancel, output limit and the Codex home lock, for generation and -// discovery, is in source-control-local-process.test.ts. +// supervised stop with the Codex home lock (generation: timeout, cancel, output limit; discovery: +// timeout, output limit) is in source-control-local-process.test.ts. const hostPlatform = process.platform afterEach(() => { diff --git a/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts b/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts index 804bcaed9420..8dabc0f2bdf7 100644 --- a/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts +++ b/src/main/text-generation/commit-message-text-generation-local-subprocess.test.ts @@ -28,8 +28,8 @@ const spawnMock = vi.mocked(spawn) const expectChildTerminated = createChildTerminationExpectation(terminateWindowsProcessTreeMock) // These suites drive fake children down the Windows direct-child path, taskkill included. The POSIX -// supervised stop under timeout, cancel, output limit and the Codex home lock, for generation and -// discovery, is in source-control-local-process.test.ts. +// supervised stop with the Codex home lock (generation: timeout, cancel, output limit; discovery: +// timeout, output limit) is in source-control-local-process.test.ts. const hostPlatform = process.platform afterEach(() => { diff --git a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts index 10d1d88451ad..959596f4e8b8 100644 --- a/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts +++ b/src/main/text-generation/commit-message-text-generation-model-discovery.test.ts @@ -42,8 +42,8 @@ function spawnError(errno: string): Error { const expectChildTerminated = createChildTerminationExpectation(terminateWindowsProcessTreeMock) // These suites drive fake children down the Windows direct-child path, taskkill included. The POSIX -// supervised stop under timeout, cancel, output limit and the Codex home lock, for generation and -// discovery, is in source-control-local-process.test.ts. +// supervised stop with the Codex home lock (generation: timeout, cancel, output limit; discovery: +// timeout, output limit) is in source-control-local-process.test.ts. const hostPlatform = process.platform afterEach(() => { diff --git a/src/main/text-generation/source-control-local-process.test.ts b/src/main/text-generation/source-control-local-process.test.ts index 013c502849b9..96eed5ba82ef 100644 --- a/src/main/text-generation/source-control-local-process.test.ts +++ b/src/main/text-generation/source-control-local-process.test.ts @@ -347,5 +347,62 @@ setInterval(() => {}, 60000) rmSync(folder, { recursive: true, force: true }) } }) + + it('stops a Codex discovery through its supervisor once its output passes the limit, holding the home', async () => { + const folder = mkdtempSync(join(tmpdir(), 'orca-supervised-discovery-limit-')) + const pids: number[] = [] + try { + const home = join(folder, 'codex-home') + const pidFile = join(folder, 'flood-pid') + const floods = join(folder, 'floods.cjs') + // Floods stdout past the limit, then takes a moment to stop on SIGTERM. + writeFileSync( + floods, + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)) +process.on('SIGTERM', () => setTimeout(() => process.exit(0), 300)) +process.stdout.write('x'.repeat(5 * 1024 * 1024)) +setInterval(() => {}, 60000) +` + ) + const lists = join(folder, 'lists.cjs') + writeFileSync( + lists, + "console.log(JSON.stringify({ models: [{ slug: 'gpt-5.5', display_name: 'GPT-5.5' }] }))\n" + ) + const floodAliveAtSecondSpawn: boolean[] = [] + const spawnAgent = vi.fn((input: Parameters[0]) => { + if (pids.length > 0) { + floodAliveAtSecondSpawn.push(isAlive(pids[0]!)) + } + return spawnSourceControlAgent(input) + }) + const discover = (script: string): ReturnType => + discoverModelsLocal({ + agentId: 'codex', + env: process.env, + agentCommandOverride: `CODEX_HOME="${home}" "${process.execPath}" "${script}"`, + options: { cwd: folder }, + backslash: 'escape', + spawnAgent + }) + + await expect(discover(floods)).resolves.toEqual({ + success: false, + error: 'Codex returned too much model data.' + }) + pids.push(Number(readFileSync(pidFile, 'utf8'))) + + await expect(discover(lists)).resolves.toMatchObject({ success: true }) + expect(floodAliveAtSecondSpawn).toEqual([false]) + expect(terminateTreeMock).not.toHaveBeenCalled() + } finally { + for (const pid of pids) { + if (isAlive(pid)) { + process.kill(pid, 'SIGKILL') + } + } + rmSync(folder, { recursive: true, force: true }) + } + }) } ) From 5f917691349e4c8c91ae37b332ccfe02fac9dade Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:36:02 -0700 Subject: [PATCH 25/38] fix(supervisor): close a provider whose owner is gone the way its owner closes it Owner death gave every session provider the stdin-end grace, so after an Orca crash a Claude session, whose close is a stdin end plus SIGTERM, could keep working on its turn for a second with nobody watching. The spawn spec now names the provider's close request: 'stdin-end' (the Codex app-server drains and exits on EOF, then gets its grace) or 'stdin-end-and-sigterm' (Claude; the default). A gone owner gets that same request. One constant per provider feeds both its spawn spec and its owner-side close, through one requestProviderClose, so the two cannot drift. One-shots still stop at once. --- .../claude-agent-sdk-process-spawn.test.ts | 1 + .../claude/claude-agent-sdk-process-spawn.ts | 5 ++- .../claude/claude-child-exit-proof-ladder.ts | 25 ++++++----- .../codex/codex-app-server-connection.test.ts | 26 ++++++++++++ src/main/codex/codex-app-server-connection.ts | 24 +++++++---- ...erver-posix-supervisor.integration.test.ts | 33 ++++++++++++--- .../codex-app-server-posix-supervisor.test.ts | 1 + .../codex-app-server-posix-supervisor.ts | 42 +++++++++++++++++-- 8 files changed, 124 insertions(+), 33 deletions(-) diff --git a/src/main/claude/claude-agent-sdk-process-spawn.test.ts b/src/main/claude/claude-agent-sdk-process-spawn.test.ts index ad1c90859acf..9a5e55e48f99 100644 --- a/src/main/claude/claude-agent-sdk-process-spawn.test.ts +++ b/src/main/claude/claude-agent-sdk-process-spawn.test.ts @@ -99,6 +99,7 @@ describe('claude agent SDK process spawn', () => { Buffer.from(String(spec.env?.ORCA_PROVIDER_SUPERVISOR_SPEC), 'base64').toString() ) expect(supervisorSpec).toMatchObject({ + closeRequest: 'stdin-end-and-sigterm', cwd: '/work/repo', ownerPid: globalThis.process.pid }) diff --git a/src/main/claude/claude-agent-sdk-process-spawn.ts b/src/main/claude/claude-agent-sdk-process-spawn.ts index bbb4b295aade..6d70597e7064 100644 --- a/src/main/claude/claude-agent-sdk-process-spawn.ts +++ b/src/main/claude/claude-agent-sdk-process-spawn.ts @@ -1,6 +1,7 @@ import type { SpawnOptions as ClaudeAgentSdkSpawnOptions } from '@anthropic-ai/claude-agent-sdk' import { spawnProcess } from '../../shared/child-process/run-process' import { createProviderSpawnSpec } from '../codex/codex-app-server-posix-supervisor' +import { CLAUDE_CODE_CLOSE_REQUEST } from './claude-child-exit-proof-ladder' /** Derived rather than imported: only src/shared/child-process may name node:child_process. */ type ClaudeCodeChild = ReturnType @@ -58,7 +59,9 @@ export function createClaudeCodeProcessSpawn( ...(options.cwd === undefined ? {} : { cwd: options.cwd }) }, definedEnv(options.env), - platform + platform, + // A gone Orca closes Claude the way Orca's own close does. + { closeRequest: CLAUDE_CODE_CLOSE_REQUEST } ) // Why `options.signal` is dropped: it would let the SDK kill the child outside // Orca's ladder, and close() may never report an exit it did not observe. diff --git a/src/main/claude/claude-child-exit-proof-ladder.ts b/src/main/claude/claude-child-exit-proof-ladder.ts index 841958b17b5b..1d42c06a6570 100644 --- a/src/main/claude/claude-child-exit-proof-ladder.ts +++ b/src/main/claude/claude-child-exit-proof-ladder.ts @@ -2,7 +2,9 @@ import type { SpawnedProcess } from '../../shared/child-process/run-process' import { waitForProcessExitUntil } from '../codex/codex-process-exit-deadline' import { PROVIDER_SUPERVISOR_MAX_STOP_MS, - stopSupervisedProvider + requestProviderClose, + stopSupervisedProvider, + type ProviderCloseRequest } from '../codex/codex-app-server-posix-supervisor' import type { ClaudeChildTreeReaper } from './claude-agent-sdk-exit-proof' @@ -12,6 +14,8 @@ const SUPERVISED_EXIT_SLACK_MS = 500 export const SUPERVISED_GRACEFUL_EXIT_MS = PROVIDER_SUPERVISOR_MAX_STOP_MS + SUPERVISED_EXIT_SLACK_MS const FORCED_EXIT_MS = 1_000 +// Stdin end alone lets Claude finish its turn, tools and edits included; a close is a stop. +export const CLAUDE_CODE_CLOSE_REQUEST: ProviderCloseRequest = 'stdin-end-and-sigterm' export type ClaudeChildExitProofInput = { child: Pick @@ -30,18 +34,13 @@ export async function proveClaudeChildExitWithReaper( // Arm before the stop: only a live root can identify its descendants. await tree.capture() const reaped = await stopSupervisedProvider({ - request: () => { - try { - input.child.stdin?.end() - } catch { - // The reap below still owns the process. - } - // Stdin end alone lets Claude finish its turn, tools and edits included; a close is a stop. - // Windows has no supervisor, and a direct SIGTERM there is TerminateProcess. - if (input.supervised && !input.exited()) { - input.child.kill('SIGTERM') - } - }, + request: () => + requestProviderClose({ + child: input.child, + closeRequest: CLAUDE_CODE_CLOSE_REQUEST, + supervised: input.supervised === true, + exited: input.exited + }), exitPromise: input.exitPromise, exited: input.exited, force: async () => { diff --git a/src/main/codex/codex-app-server-connection.test.ts b/src/main/codex/codex-app-server-connection.test.ts index 797dd838eafb..d37c4f025a61 100644 --- a/src/main/codex/codex-app-server-connection.test.ts +++ b/src/main/codex/codex-app-server-connection.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os' import { PassThrough } from 'node:stream' import { providerDiagnosticOf } from '../../shared/agent-session-failure' import { afterEach, describe, expect, it, vi } from 'vitest' +import type { ProcessSpec } from '../../shared/child-process/process-spec' import type { spawnProcess } from '../../shared/child-process/run-process' import { isCodexAppServerRequestError, @@ -171,6 +172,31 @@ function responseLine(targetBytes: number, id: number): string { } describe('openCodexAppServerConnection', () => { + it.runIf(process.platform !== 'win32')( + 'has its supervisor close Codex by its stdin end, the way its own close does', + async () => { + const { child, spawnImpl } = stubChild() + const specs: ProcessSpec[] = [] + answerInitialize(child) + + const connection = await openCodexAppServerConnection( + { command: 'codex', args: ['app-server'] }, + {}, + (spec: ProcessSpec) => { + specs.push(spec) + return spawnImpl(spec) + } + ) + + expect( + JSON.parse( + Buffer.from(String(specs[0]?.env?.ORCA_PROVIDER_SUPERVISOR_SPEC), 'base64').toString() + ) + ).toMatchObject({ closeRequest: 'stdin-end' }) + await connection.close() + } + ) + it('advertises the experimental API required for rollout-path resume', async () => { const { child, spawnImpl, written } = stubChild() answerInitialize(child) diff --git a/src/main/codex/codex-app-server-connection.ts b/src/main/codex/codex-app-server-connection.ts index 8552619b7e02..80e9be692c77 100644 --- a/src/main/codex/codex-app-server-connection.ts +++ b/src/main/codex/codex-app-server-connection.ts @@ -2,7 +2,9 @@ import { spawnProcess } from '../../shared/child-process/run-process' import { RetryableProcessExitProof } from '../../shared/child-process/retryable-process-exit-proof' import { createProviderSpawnSpec, - stopSupervisedProvider + requestProviderClose, + stopSupervisedProvider, + type ProviderCloseRequest } from './codex-app-server-posix-supervisor' import { buildCodexAppServerExitError } from './codex-app-server-exit-error' import { initializeCodexAppServerConnection } from './codex-app-server-handshake' @@ -47,6 +49,8 @@ export type CodexAppServerLaunch = { const DEFAULT_REQUEST_TIMEOUT_MS = 30_000 export const GRACEFUL_EXIT_MS = 1_500 +// Codex finishes its writes (auth.json, the state database) and exits on its stdin end. +const CODEX_APP_SERVER_CLOSE_REQUEST: ProviderCloseRequest = 'stdin-end' const FORCED_EXIT_MS = 1_000 const STDERR_TAIL_MAX_BYTES = 8192 @@ -64,7 +68,9 @@ export async function openCodexAppServerConnection( for (const key of launch.envToDelete ?? []) { delete childEnv[key] } - const spawnSpec = createProviderSpawnSpec(launch, childEnv, process.platform) + const spawnSpec = createProviderSpawnSpec(launch, childEnv, process.platform, { + closeRequest: CODEX_APP_SERVER_CLOSE_REQUEST + }) const child = spawnImpl(spawnSpec) function terminateProcessTree(): Promise { @@ -246,13 +252,13 @@ export async function openCodexAppServerConnection( closing = true return exitProof.run(async () => { await stopSupervisedProvider({ - request: () => { - try { - child.stdin.end() - } catch { - // Already destroyed; the reap below still runs. - } - }, + request: () => + requestProviderClose({ + child, + closeRequest: CODEX_APP_SERVER_CLOSE_REQUEST, + supervised: spawnSpec.supervised, + exited: () => exited + }), exitPromise, exited: () => exited, force: async () => { diff --git a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts index fae5f9d6f18c..70c7d01614b5 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts @@ -337,8 +337,8 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe owner.kill('SIGKILL') expect(await waitFor(() => !alive(-pids.provider), 3_000)).toBe(true) - // The provider ignores its stdin end, so its group lasts through the stdin-end grace. - expect(Date.now() - killedAt).toBeGreaterThanOrEqual(PROVIDER_STDIN_END_GRACE_MS - 20) + // Closed with SIGTERM at once; its SIGTERM-ignoring grandchild dies with the provider. + expect(Date.now() - killedAt).toBeLessThan(PROVIDER_STDIN_END_GRACE_MS) expect(await waitFor(() => !alive(pids.supervisor), 3_000)).toBe(true) expect(alive(pids.grandchild)).toBe(false) }) @@ -415,12 +415,33 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe expect(existsSync(signalFile) && readFileSync(signalFile, 'utf8')).toBe('SIGTERM') }) - it('gives a session provider its stdin end and grace when its owner dies', async () => { + it('stops a session closed with SIGTERM at once when its owner dies', async () => { + const signalFile = join(tempDir(), 'provider-sigterm-at') + const { owner, pids } = await launchUnderOwner( + { sigtermGraceMs: 200 }, + { ORCA_TEST_PROVIDER_SIGNAL_FILE: signalFile }, + { script: RECORDS_SIGTERM_PROVIDER, pids: ['provider'] } + ) + + const killedAt = Date.now() + owner.kill('SIGKILL') + + // Non-empty, not just present: a read between create and write would pass any bound. + const signalledAt = (): number => + existsSync(signalFile) ? Number(readFileSync(signalFile, 'utf8')) : 0 + expect(await waitFor(() => signalledAt() > 0, 3_000)).toBe(true) + // No unwatched stdin-end grace: the owner-death watch's 100 ms poll is the whole delay. + expect(signalledAt() - killedAt).toBeGreaterThanOrEqual(0) + expect(signalledAt() - killedAt).toBeLessThan(PROVIDER_STDIN_END_GRACE_MS / 2) + expect(await waitFor(() => !alive(-pids.provider), 3_000)).toBe(true) + }) + + it('gives a session closed by its stdin end that EOF and grace when its owner dies', async () => { const dir = tempDir() const signalFile = join(dir, 'provider-signal') const flushFile = join(dir, 'provider-flushed') const { owner, pids } = await launchUnderOwner( - {}, + { closeRequest: 'stdin-end' }, { ORCA_TEST_PROVIDER_SIGNAL_FILE: signalFile, ORCA_TEST_PROVIDER_FLUSH_FILE: flushFile }, { script: FLUSHES_ON_STDIN_END_PROVIDER, pids: ['provider'] } ) @@ -433,11 +454,11 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe expect(existsSync(signalFile)).toBe(false) }) - it('stops a session provider that ignores its stdin end once the grace after owner death ends', async () => { + it('stops a stdin-end session that ignores its EOF once the grace after owner death ends', async () => { const signalFile = join(tempDir(), 'provider-sigterm-at') const stdinEndGraceMs = 400 const { owner, pids } = await launchUnderOwner( - { stdinEndGraceMs, sigtermGraceMs: 200 }, + { closeRequest: 'stdin-end', stdinEndGraceMs, sigtermGraceMs: 200 }, { ORCA_TEST_PROVIDER_SIGNAL_FILE: signalFile }, { script: RECORDS_SIGTERM_PROVIDER, pids: ['provider'] } ) diff --git a/src/main/codex/codex-app-server-posix-supervisor.test.ts b/src/main/codex/codex-app-server-posix-supervisor.test.ts index 9987fa33bdb6..57ad26b0e3d0 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.test.ts @@ -39,6 +39,7 @@ describe('structured provider supervision', () => { cwd: '/work/repo', ownerPid: process.pid, lifetime: 'session', + closeRequest: 'stdin-end-and-sigterm', stdinEndGraceMs: PROVIDER_STDIN_END_GRACE_MS, sigtermGraceMs: PROVIDER_SIGTERM_GRACE_MS }) diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index 3af323dc2925..bea088cc0e01 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -1,3 +1,4 @@ +import type { ChildProcessHandle } from '../../shared/child-process/process-spec' import type { CodexAppServerLaunch } from './codex-app-server-connection' import { waitForProcessExitUntil } from './codex-process-exit-deadline' import { PROVIDER_SPAWN_FAILURE_MARKER } from './provider-spawn-failure-report' @@ -147,13 +148,14 @@ const reapProviderExit = async (code, signal) => { await drainProviderOutput() finishWithProviderOutcome(code, signal) } +// An owner that is gone gets the close its owner would have asked for, made here on its behalf. timer = setInterval(() => { if (!ownerGone()) return - if (spec.lifetime === 'one-shot') return stopProviderGroup(null) - // A session whose owner is gone closes as an owner's stdin end does: EOF, its grace, then the stop. clearInterval(timer) process.stdin.unpipe(child.stdin) try { child.stdin.end() } catch {} + // A one-shot's stdin end was its request, so only a stop is left to ask for. + if (spec.lifetime === 'one-shot' || spec.closeRequest !== 'stdin-end') return stopProviderGroup(null) scheduleOwnerShutdown() }, 100) timer.unref() @@ -208,9 +210,37 @@ export async function stopSupervisedProvider(input: ProviderStopInput): Promise< return true } +/** + * How a provider's owner closes it: by ending its stdin, after which a session gets its grace (a + * drain), or by ending stdin and sending SIGTERM at once. A gone owner gets the same request. + */ +export type ProviderCloseRequest = 'stdin-end' | 'stdin-end-and-sigterm' + +/** + * Makes a provider's close request. Only a supervisor turns SIGTERM into its ladder; a direct + * (Windows) child gets the stdin end alone, since a SIGTERM there is TerminateProcess. + */ +export function requestProviderClose(input: { + child: Pick + closeRequest: ProviderCloseRequest + supervised: boolean + exited: () => boolean +}): void { + try { + input.child.stdin?.end() + } catch { + // Already destroyed; the caller's wait and force still run. + } + if (input.closeRequest === 'stdin-end-and-sigterm' && input.supervised && !input.exited()) { + input.child.kill('SIGTERM') + } +} + export type ProviderSupervisorOptions = { cwd?: string lifetime?: ProviderSupervisorLifetime + /** Defaults to the immediate stop; a provider that drains on its stdin end opts into 'stdin-end'. */ + closeRequest?: ProviderCloseRequest /** The process the supervisor serves; it must be the supervisor's parent. */ ownerPid?: number stdinEndGraceMs?: number @@ -234,6 +264,7 @@ export function supervisedPosixLaunch( cwd = launch.cwd ?? process.cwd(), ownerPid = process.pid, lifetime = 'session', + closeRequest = 'stdin-end-and-sigterm', stdinEndGraceMs = PROVIDER_STDIN_END_GRACE_MS, sigtermGraceMs = PROVIDER_SIGTERM_GRACE_MS }: ProviderSupervisorOptions = {} @@ -256,6 +287,7 @@ export function supervisedPosixLaunch( cwd, ownerPid, lifetime, + closeRequest, stdinEndGraceMs, sigtermGraceMs, nodeEnv @@ -278,7 +310,7 @@ export function createProviderSpawnSpec( launch: CodexAppServerLaunch, childEnv: NodeJS.ProcessEnv, platform: NodeJS.Platform, - { lifetime }: Pick = {} + { lifetime, closeRequest }: Pick = {} ): { program: string args: string[] @@ -289,7 +321,9 @@ export function createProviderSpawnSpec( supervised: boolean } { const supervisor = - platform === 'win32' ? null : supervisedPosixLaunch(launch, childEnv, { lifetime }) + platform === 'win32' + ? null + : supervisedPosixLaunch(launch, childEnv, { lifetime, closeRequest }) return { program: supervisor?.command ?? launch.command, args: supervisor?.args ?? launch.args, From 5ecc87fbccf3a45ca110981300cf884b9f69749f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:45:33 -0700 Subject: [PATCH 26/38] fix(codex): close short-lived sessions and the backfill app-server the way a Codex connection closes Codex finishes its writes and exits on its stdin end, so the Codex connection's supervisor closes it by ending stdin, and an owner that is gone now gets that same close. The short-lived sessions and the backfill app-server now name the same close request, through one shared constant, in their spawn spec and in their own close: Orca quitting or crashing gives them the stdin end and its grace before SIGTERM, instead of an immediate SIGTERM. A session past its deadline still adds a SIGTERM, since it is wedged. --- config/tsconfig.cli.json | 1 + .../codex/codex-app-server-close-request.ts | 4 ++++ src/main/codex/codex-app-server-connection.ts | 6 ++---- src/main/codex/codex-app-server-session.ts | 15 +++++++++------ .../codex-state-db-backfill-recovery-process.ts | 17 ++++++++++++++--- 5 files changed, 30 insertions(+), 13 deletions(-) create mode 100644 src/main/codex/codex-app-server-close-request.ts diff --git a/config/tsconfig.cli.json b/config/tsconfig.cli.json index 39f91e94c3cb..22039a8db962 100644 --- a/config/tsconfig.cli.json +++ b/config/tsconfig.cli.json @@ -132,6 +132,7 @@ "../src/main/codex/codex-app-server-capability-cache.ts", "../src/main/codex/codex-app-server-capability-signal.ts", "../src/main/codex/codex-app-server-client.ts", + "../src/main/codex/codex-app-server-close-request.ts", "../src/main/codex/codex-app-server-posix-supervisor.ts", "../src/main/codex/codex-app-server-process-teardown.ts", "../src/main/codex/codex-app-server-process-tree-kill.ts", diff --git a/src/main/codex/codex-app-server-close-request.ts b/src/main/codex/codex-app-server-close-request.ts new file mode 100644 index 000000000000..e06f8d8032fd --- /dev/null +++ b/src/main/codex/codex-app-server-close-request.ts @@ -0,0 +1,4 @@ +import type { ProviderCloseRequest } from './codex-app-server-posix-supervisor' + +/** Codex finishes its writes (auth.json, the state database) and exits on its stdin end. */ +export const CODEX_APP_SERVER_CLOSE_REQUEST: ProviderCloseRequest = 'stdin-end' diff --git a/src/main/codex/codex-app-server-connection.ts b/src/main/codex/codex-app-server-connection.ts index 80e9be692c77..54a7d92662e8 100644 --- a/src/main/codex/codex-app-server-connection.ts +++ b/src/main/codex/codex-app-server-connection.ts @@ -1,10 +1,10 @@ import { spawnProcess } from '../../shared/child-process/run-process' import { RetryableProcessExitProof } from '../../shared/child-process/retryable-process-exit-proof' +import { CODEX_APP_SERVER_CLOSE_REQUEST } from './codex-app-server-close-request' import { createProviderSpawnSpec, requestProviderClose, - stopSupervisedProvider, - type ProviderCloseRequest + stopSupervisedProvider } from './codex-app-server-posix-supervisor' import { buildCodexAppServerExitError } from './codex-app-server-exit-error' import { initializeCodexAppServerConnection } from './codex-app-server-handshake' @@ -49,8 +49,6 @@ export type CodexAppServerLaunch = { const DEFAULT_REQUEST_TIMEOUT_MS = 30_000 export const GRACEFUL_EXIT_MS = 1_500 -// Codex finishes its writes (auth.json, the state database) and exits on its stdin end. -const CODEX_APP_SERVER_CLOSE_REQUEST: ProviderCloseRequest = 'stdin-end' const FORCED_EXIT_MS = 1_000 const STDERR_TAIL_MAX_BYTES = 8192 diff --git a/src/main/codex/codex-app-server-session.ts b/src/main/codex/codex-app-server-session.ts index 76b77838edcc..a8251138f43d 100644 --- a/src/main/codex/codex-app-server-session.ts +++ b/src/main/codex/codex-app-server-session.ts @@ -2,8 +2,10 @@ import type { ChildProcessWithoutNullStreams } from 'node:child_process' import { waitForProcessExitUntil } from './codex-process-exit-deadline' import { stderrIndicatesMissingAppServer } from './codex-app-server-capability-signal' import { withCliRuntimeOnPath } from '../../shared/node-cli-command-resolution' +import { CODEX_APP_SERVER_CLOSE_REQUEST } from './codex-app-server-close-request' import { createProviderSpawnSpec, + requestProviderClose, stopSupervisedProvider } from './codex-app-server-posix-supervisor' import { terminateCodexAppServerProcessTree } from './codex-app-server-process-teardown' @@ -125,7 +127,7 @@ export async function runCodexAppServerSession( { command: invocation.command, args: invocation.args }, pairedEnv, process.platform, - { lifetime: 'session' } + { lifetime: 'session', closeRequest: CODEX_APP_SERVER_CLOSE_REQUEST } ) // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: every stdio slot is 'pipe', so stdin, stdout and stderr exist. const child = spawnImpl(spawnSpec.program, spawnSpec.args, { @@ -336,11 +338,12 @@ export async function runCodexAppServerSession( } finally { await stopSupervisedProvider({ request: () => { - try { - child.stdin.end() - } catch { - // stdin may already be destroyed after a kill; reaping below still runs. - } + requestProviderClose({ + child, + closeRequest: CODEX_APP_SERVER_CLOSE_REQUEST, + supervised: spawnSpec.supervised, + exited: () => exited + }) // A session past its deadline is wedged; its stdin end would only add a grace. if (timedOut && spawnSpec.supervised && !exited) { child.kill('SIGTERM') diff --git a/src/main/codex/codex-state-db-backfill-recovery-process.ts b/src/main/codex/codex-state-db-backfill-recovery-process.ts index e4afde3e2546..9d7e141eb802 100644 --- a/src/main/codex/codex-state-db-backfill-recovery-process.ts +++ b/src/main/codex/codex-state-db-backfill-recovery-process.ts @@ -4,7 +4,8 @@ import { parseWslUncPath } from '../../shared/wsl-paths' import { buildWslCodexAppServerArgs } from '../codex-accounts/wsl-codex-command' import { CODEX_READ_ONLY_APP_SERVER_ARGS } from '../codex-cli/codex-read-only-app-server-args' import { terminateCodexProbeChild } from '../rate-limits/codex-probe-termination' -import { createProviderSpawnSpec } from './codex-app-server-posix-supervisor' +import { CODEX_APP_SERVER_CLOSE_REQUEST } from './codex-app-server-close-request' +import { createProviderSpawnSpec, requestProviderClose } from './codex-app-server-posix-supervisor' import type { CodexAppServerSpawn } from './codex-app-server-process-tree-kill' import { stopSupervisedChildProcess } from './supervised-child-process-stop' @@ -41,7 +42,7 @@ export function spawnCodexBackfillRecoveryProcess( { command, args: [...CODEX_READ_ONLY_APP_SERVER_ARGS], cwd: codexHomePath }, withCliRuntimeOnPath(command, { ...process.env, CODEX_HOME: codexHomePath }), process.platform, - { lifetime: 'session' } + { lifetime: 'session', closeRequest: CODEX_APP_SERVER_CLOSE_REQUEST } ) const child = spawnProcess(spawnSpec.program, spawnSpec.args, { cwd: codexHomePath, @@ -59,7 +60,17 @@ export async function stopCodexBackfillRecoveryProcess( ): Promise { if (supervised) { // A session supervisor turns stdin end into its group stop, as a Codex connection close does. - await stopSupervisedChildProcess(child, () => child.stdin?.end(), BACKFILL_RECOVERY_KILL_SITE) + await stopSupervisedChildProcess( + child, + () => + requestProviderClose({ + child, + closeRequest: CODEX_APP_SERVER_CLOSE_REQUEST, + supervised: true, + exited: () => child.exitCode !== null || child.signalCode !== null + }), + BACKFILL_RECOVERY_KILL_SITE + ) return } await terminateCodexProbeChild(child) From f97d201218ad047216caa85ba694f5109567d5ed Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:45:33 -0700 Subject: [PATCH 27/38] test(codex,opencode): an owner's death drains Codex before SIGTERM, and the preflight stop no longer waits out the grace The stand-in now records when its stdin ended and when SIGTERM arrived. A SIGKILLed owner leaves a Codex session or backfill app-server its stdin-end grace before SIGTERM, and the OpenCode preflight ends within 1 s of its server's SIGTERM even with a descendant that ignores SIGTERM. --- ...dex-app-server-session-supervision.test.ts | 13 +++++++-- ...e-db-backfill-recovery-supervision.test.ts | 13 +++++++-- .../supervised-probe-owner.test-fixture.ts | 27 ++++++++++++++++--- ...e-launch-model-context-supervision.test.ts | 4 +++ 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/src/main/codex/codex-app-server-session-supervision.test.ts b/src/main/codex/codex-app-server-session-supervision.test.ts index bad9a0944761..88a21cac0c9f 100644 --- a/src/main/codex/codex-app-server-session-supervision.test.ts +++ b/src/main/codex/codex-app-server-session-supervision.test.ts @@ -3,7 +3,10 @@ import { existsSync, readFileSync } from 'node:fs' import { PassThrough } from 'node:stream' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' -import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from './codex-app-server-posix-supervisor' +import { + PROVIDER_STDIN_END_GRACE_MS, + PROVIDER_SUPERVISOR_MAX_STOP_MS +} from './codex-app-server-posix-supervisor' import type { CodexAppServerSpawn } from './codex-app-server-process-tree-kill' import { CodexAppServerTimeoutError, runCodexAppServerSession } from './codex-app-server-session' import { classifyCodexTrustGrantError } from './codex-trust-grant-telemetry' @@ -167,7 +170,7 @@ describe.runIf(process.platform !== 'win32')('supervised Codex app-server sessio expect(alive(pids.grandchild)).toBe(false) }) - it('stops the session group when its owner is SIGKILLed mid-session', async () => { + it('drains, then stops the session group, when its owner is SIGKILLed mid-session', async () => { rig = createSupervisedProbeRig() const standIn = rig.writeStandIn('codex', WEDGED_APP_SERVER) const bundle = await rig.bundleOwner(OWNER, __dirname) @@ -182,6 +185,12 @@ describe.runIf(process.platform !== 'win32')('supervised Codex app-server sessio PROVIDER_SUPERVISOR_MAX_STOP_MS + 1_000 ) ).toBe(true) + // A gone owner gets the close its owner makes: the stdin end, its grace, and only then SIGTERM. + const events = rig.readEvents() + expect(events['stdin-end']).toBeDefined() + expect((events.SIGTERM ?? 0) - (events['stdin-end'] ?? 0)).toBeGreaterThanOrEqual( + PROVIDER_STDIN_END_GRACE_MS - 50 + ) }) it('reports a missing Codex binary as the spawn error, not an early exit', async () => { diff --git a/src/main/codex/codex-state-db-backfill-recovery-supervision.test.ts b/src/main/codex/codex-state-db-backfill-recovery-supervision.test.ts index af5f58c1f2e7..5c30ef281b15 100644 --- a/src/main/codex/codex-state-db-backfill-recovery-supervision.test.ts +++ b/src/main/codex/codex-state-db-backfill-recovery-supervision.test.ts @@ -2,7 +2,10 @@ import { ChildProcess } from 'node:child_process' import { existsSync, readFileSync } from 'node:fs' import { afterEach, describe, expect, it, vi } from 'vitest' import { CODEX_READ_ONLY_APP_SERVER_ARGS } from '../codex-cli/codex-read-only-app-server-args' -import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from './codex-app-server-posix-supervisor' +import { + PROVIDER_STDIN_END_GRACE_MS, + PROVIDER_SUPERVISOR_MAX_STOP_MS +} from './codex-app-server-posix-supervisor' import type { CodexAppServerSpawn } from './codex-app-server-process-tree-kill' import type { CodexStateDbBackfillStatus } from './codex-state-db' import { runCodexStateDbBackfillRecovery } from './codex-state-db-backfill-recovery' @@ -115,7 +118,7 @@ describe.runIf(process.platform !== 'win32')('supervised Codex backfill recovery expect(alive(pids.grandchild)).toBe(false) }) - it('stops the recovery group when its owner is SIGKILLed', async () => { + it('drains, then stops the recovery group, when its owner is SIGKILLed', async () => { rig = createSupervisedProbeRig() const standIn = rig.writeStandIn('codex') const bundle = await rig.bundleOwner(OWNER, __dirname) @@ -133,5 +136,11 @@ describe.runIf(process.platform !== 'win32')('supervised Codex backfill recovery PROVIDER_SUPERVISOR_MAX_STOP_MS + 1_000 ) ).toBe(true) + // A gone owner gets the close its owner makes: the stdin end, its grace, and only then SIGTERM. + const events = rig.readEvents() + expect(events['stdin-end']).toBeDefined() + expect((events.SIGTERM ?? 0) - (events['stdin-end'] ?? 0)).toBeGreaterThanOrEqual( + PROVIDER_STDIN_END_GRACE_MS - 50 + ) }) }) diff --git a/src/main/codex/supervised-probe-owner.test-fixture.ts b/src/main/codex/supervised-probe-owner.test-fixture.ts index a83a512755e0..a8b131e8008c 100644 --- a/src/main/codex/supervised-probe-owner.test-fixture.ts +++ b/src/main/codex/supervised-probe-owner.test-fixture.ts @@ -6,16 +6,19 @@ import { build } from 'esbuild' // A stand-in agent CLI that, like `opencode serve` or a wedged Codex, ignores its stdin end. It // leaves a SIGTERM-ignoring grandchild in its group, so only a group SIGKILL ends everything, and -// records a SIGTERM before exiting on it. `body` adds the caller's protocol. +// records its stdin end and a SIGTERM (exiting on the latter). `body` adds the caller's protocol. function standInSource(body: string): string { return String.raw` const { spawn } = require('node:child_process') const fs = require('node:fs') +const record = (event) => + fs.appendFileSync(process.env.ORCA_TEST_EVENTS_FILE, JSON.stringify({ event, at: Date.now() }) + '\n') process.on('SIGTERM', () => { + record('SIGTERM') fs.writeFileSync(process.env.ORCA_TEST_SIGNAL_FILE, 'SIGTERM') process.exit(0) }) -process.stdin.on('error', () => {}).resume() +process.stdin.on('error', () => {}).on('end', () => record('stdin-end')).resume() const grandchild = spawn( process.execPath, ['-e', "process.on('SIGTERM', () => {}); process.stdout.write('armed'); setInterval(() => {}, 60000)"], @@ -32,10 +35,14 @@ ${body} ` } +export type StandInEvent = { event: 'stdin-end' | 'SIGTERM'; at: number } + export type SupervisedProbeRig = { dir: string pidFile: string signalFile: string + /** When the stand-in saw its stdin end and its SIGTERM, by its own clock (this host's). */ + readEvents: () => Partial> env: Record /** An executable stand-in, run as ` ` like a real agent CLI. */ writeStandIn: (name: string, body?: string) => string @@ -70,12 +77,26 @@ export function createSupervisedProbeRig(): SupervisedProbeRig { const dir = mkdtempSync(join(tmpdir(), 'orca-supervised-probe-')) const pidFile = join(dir, 'pids.json') const signalFile = join(dir, 'signal') + const eventsFile = join(dir, 'events.jsonl') const recordedPids = new Set() - const env = { ORCA_TEST_PID_FILE: pidFile, ORCA_TEST_SIGNAL_FILE: signalFile } + const env = { + ORCA_TEST_PID_FILE: pidFile, + ORCA_TEST_SIGNAL_FILE: signalFile, + ORCA_TEST_EVENTS_FILE: eventsFile + } return { dir, pidFile, signalFile, + readEvents: () => { + const events: Partial> = {} + const lines = existsSync(eventsFile) ? readFileSync(eventsFile, 'utf8').split('\n') : [] + for (const line of lines.filter(Boolean)) { + const { event, at }: StandInEvent = JSON.parse(line) + events[event] ??= at + } + return events + }, env, writeStandIn: (name, body = '') => { const script = join(dir, `${name}.cjs`) diff --git a/src/main/opencode/opencode-launch-model-context-supervision.test.ts b/src/main/opencode/opencode-launch-model-context-supervision.test.ts index 224911545930..03dbd14b8144 100644 --- a/src/main/opencode/opencode-launch-model-context-supervision.test.ts +++ b/src/main/opencode/opencode-launch-model-context-supervision.test.ts @@ -59,6 +59,7 @@ describe.runIf(process.platform !== 'win32')('supervised OpenCode launch-context cwd: rig.dir, env: { ...process.env, ...rig.env } }) + const stoppedAt = Date.now() const pids = await rig.readPids() expect(context).toEqual({ @@ -71,6 +72,9 @@ describe.runIf(process.platform !== 'win32')('supervised OpenCode launch-context // A descendant that ignores SIGTERM holds none of the probe's pipes, so a closed pipe never // proved it gone; only the supervisor's group SIGKILL ends it. expect(alive(pids.grandchild)).toBe(false) + // Once the server exits on its SIGTERM, the rest of its group is killed at once, not after the + // SIGTERM grace; every launch waits on this preflight. + expect(stoppedAt - (rig.readEvents().SIGTERM ?? 0)).toBeLessThan(1_000) }) it('stops the server group when its owner is SIGKILLed mid-probe', async () => { From f7d024a655258826d7f3cfbaaab5c7611f22f07b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:45:33 -0700 Subject: [PATCH 28/38] test(codex): give the trust-grant deadline tests room for a supervised start on a loaded host At 500 ms a loaded full run hit the deadline before the supervisor had started the stub, which never wrote the pid the test reads. --- src/main/codex/codex-app-server-client.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/codex/codex-app-server-client.test.ts b/src/main/codex/codex-app-server-client.test.ts index ce98288bdf06..5d54689a89e5 100644 --- a/src/main/codex/codex-app-server-client.test.ts +++ b/src/main/codex/codex-app-server-client.test.ts @@ -393,7 +393,8 @@ describe('runCodexHookTrustGrantSession', () => { hooks: keys.map((key) => managedHook(key)), expectedTrustKeys: keys, managedCommand: MANAGED_COMMAND, - timeoutMs: 500 + // Long enough for the supervisor and stub to start on a loaded host and write the pid. + timeoutMs: 2_000 }) const startedAt = Date.now() @@ -413,7 +414,8 @@ describe('runCodexHookTrustGrantSession', () => { hooks: [], expectedTrustKeys: [], managedCommand: MANAGED_COMMAND, - timeoutMs: 500 + // Long enough for the supervisor and stub to start on a loaded host and write the pid. + timeoutMs: 2_000 }) const startedAt = Date.now() From 0a2de70208bae7d79e885da5f2842204f05b4478 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:53:23 -0700 Subject: [PATCH 29/38] fix(supervisor): keep the SIGTERM grace for a session's group after its provider exits Killing the rest of the group the moment the provider exited under a stop also reached native chat's closes, so an MCP server, a tool's child or a dev server still in Claude's or Codex's group was SIGKILLed mid-cleanup instead of getting the rest of the SIGTERM grace. The early group kill now applies only to one-shots, where the saved wait was the point; a session's stop is back to waiting out the grace for its group. --- ...erver-posix-supervisor.integration.test.ts | 37 +++++++++++++++++-- .../codex-app-server-posix-supervisor.ts | 4 +- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts index 70c7d01614b5..dcea90f3c987 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.integration.test.ts @@ -35,6 +35,20 @@ const PROVIDER = String.raw` setInterval(() => {}, 60000) ` +// Dies on SIGTERM at once; its grandchild, like an MCP server or a tool's child, needs 300 ms to clean up. +const CLEANS_UP_AFTER_SIGTERM_PROVIDER = String.raw` + const { spawn } = require('node:child_process') + const grandchild = spawn( + process.execPath, + ['-e', "process.on('SIGTERM', () => setTimeout(() => { require('node:fs').writeFileSync(process.env.ORCA_TEST_CLEANUP_FILE, 'done'); process.exit(0) }, 300)); process.stdout.write('armed'); setInterval(() => {}, 60000)"], + { stdio: ['ignore', 'pipe', 'ignore'] } + ) + grandchild.stdout.once('data', () => + process.stdout.write(JSON.stringify({ provider: process.pid, grandchild: grandchild.pid }) + '\n') + ) + setInterval(() => {}, 60000) +` + // Exits the moment its stdin ends, as Codex does on a normal close. const EXITS_ON_STDIN_END_PROVIDER = String.raw` process.stdin.on('end', () => process.exit(0)).resume() @@ -261,8 +275,8 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe expect(alive(grandchild)).toBe(false) }) - it('kills the rest of the group at once when the provider exits on a stop', async () => { - const { supervisor, exit } = launchSupervisor({}) + it('kills the rest of a one-shot group at once when the provider exits on a stop', async () => { + const { supervisor, exit } = launchSupervisor({ lifetime: 'one-shot' }) const { provider, grandchild } = await readPids(supervisor, ['provider', 'grandchild']) const signalledAt = Date.now() @@ -275,6 +289,21 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe expect(alive(grandchild)).toBe(false) }) + it('lets a session provider group finish its SIGTERM cleanup after the provider exits', async () => { + const cleanupFile = join(tempDir(), 'grandchild-cleaned-up') + const { supervisor, exit } = launchSupervisor( + {}, + { ORCA_TEST_CLEANUP_FILE: cleanupFile }, + { command: process.execPath, args: ['-e', CLEANS_UP_AFTER_SIGTERM_PROVIDER] } + ) + await readPids(supervisor, ['provider', 'grandchild']) + + supervisor.kill('SIGTERM') + + await expect(exit).resolves.toEqual({ code: null, signal: 'SIGTERM' }) + expect(existsSync(cleanupFile)).toBe(true) + }) + it('escalates a SIGTERM-ignoring provider to SIGKILL after the grace from the spec', async () => { const graceMs = 200 const { supervisor, exit } = launchSupervisor( @@ -337,8 +366,8 @@ describe.runIf(process.platform !== 'win32')('POSIX provider supervisor processe owner.kill('SIGKILL') expect(await waitFor(() => !alive(-pids.provider), 3_000)).toBe(true) - // Closed with SIGTERM at once; its SIGTERM-ignoring grandchild dies with the provider. - expect(Date.now() - killedAt).toBeLessThan(PROVIDER_STDIN_END_GRACE_MS) + // The grandchild ignores SIGTERM, so the group lasts until the grace ends in SIGKILL. + expect(Date.now() - killedAt).toBeGreaterThanOrEqual(graceMs) expect(await waitFor(() => !alive(pids.supervisor), 3_000)).toBe(true) expect(alive(pids.grandchild)).toBe(false) }) diff --git a/src/main/codex/codex-app-server-posix-supervisor.ts b/src/main/codex/codex-app-server-posix-supervisor.ts index bea088cc0e01..1861eeee01c8 100644 --- a/src/main/codex/codex-app-server-posix-supervisor.ts +++ b/src/main/codex/codex-app-server-posix-supervisor.ts @@ -98,8 +98,8 @@ const stopProviderGroup = (receivedSignal) => { clearInterval(timer) if (ownerShutdownTimer) clearTimeout(ownerShutdownTimer) try { process.kill(-child.pid, 'SIGTERM') } catch {} - // Once the provider itself has exited, the rest of its group dies at once, as on its own exit. - void waitForProviderGroupExit(spec.sigtermGraceMs, true) + // A one-shot's helpers die with it once it has exited; a session's keep the grace to clean up. + void waitForProviderGroupExit(spec.sigtermGraceMs, spec.lifetime === 'one-shot') .then((exited) => exited || reapOwnedProviderGroup()) .then((reaped) => { if (!reaped) return process.exit(1) From d8f83589f9afee06f185ccee7b3a1cfe1afef131 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:55:54 -0700 Subject: [PATCH 30/38] test(claude): pin that Claude's spawn passes its close request explicitly The spawn-spec assertion matched the default close request, so dropping Claude's explicit request still passed. The test now checks that the spec is built with the exit-proof ladder's own constant. --- .../claude/claude-agent-sdk-process-spawn.test.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/claude/claude-agent-sdk-process-spawn.test.ts b/src/main/claude/claude-agent-sdk-process-spawn.test.ts index 9a5e55e48f99..62fc44e1b900 100644 --- a/src/main/claude/claude-agent-sdk-process-spawn.test.ts +++ b/src/main/claude/claude-agent-sdk-process-spawn.test.ts @@ -7,7 +7,10 @@ import type { ProcessSpec } from '../../shared/child-process/process-spec' import type * as ProviderSupervisor from '../codex/codex-app-server-posix-supervisor' import { createProviderSpawnSpec } from '../codex/codex-app-server-posix-supervisor' import { createClaudeCodeProcessSpawn } from './claude-agent-sdk-process-spawn' -import { proveClaudeChildExitWithReaper } from './claude-child-exit-proof-ladder' +import { + CLAUDE_CODE_CLOSE_REQUEST, + proveClaudeChildExitWithReaper +} from './claude-child-exit-proof-ladder' vi.mock('../codex/codex-app-server-posix-supervisor', async (importOriginal) => { const actual = await importOriginal() @@ -98,8 +101,16 @@ describe('claude agent SDK process spawn', () => { const supervisorSpec = JSON.parse( Buffer.from(String(spec.env?.ORCA_PROVIDER_SUPERVISOR_SPEC), 'base64').toString() ) + // Passed explicitly from the constant the exit-proof ladder closes with, never left to the + // spec default, so the two cannot drift. + expect(vi.mocked(createProviderSpawnSpec)).toHaveBeenLastCalledWith( + expect.anything(), + expect.anything(), + platform, + { closeRequest: CLAUDE_CODE_CLOSE_REQUEST } + ) expect(supervisorSpec).toMatchObject({ - closeRequest: 'stdin-end-and-sigterm', + closeRequest: CLAUDE_CODE_CLOSE_REQUEST, cwd: '/work/repo', ownerPid: globalThis.process.pid }) From cc043509a724b1e3859dc022e1c6e8078e78a870 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:05:19 -0700 Subject: [PATCH 31/38] refactor(codex): give the Codex app-server close request its own module Other Codex app-server spawns will name the same close request as the connection does. Holding it in its own small module lets them import it without the connection itself. --- src/main/codex/codex-app-server-close-request.ts | 4 ++++ src/main/codex/codex-app-server-connection.ts | 6 ++---- 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 src/main/codex/codex-app-server-close-request.ts diff --git a/src/main/codex/codex-app-server-close-request.ts b/src/main/codex/codex-app-server-close-request.ts new file mode 100644 index 000000000000..e06f8d8032fd --- /dev/null +++ b/src/main/codex/codex-app-server-close-request.ts @@ -0,0 +1,4 @@ +import type { ProviderCloseRequest } from './codex-app-server-posix-supervisor' + +/** Codex finishes its writes (auth.json, the state database) and exits on its stdin end. */ +export const CODEX_APP_SERVER_CLOSE_REQUEST: ProviderCloseRequest = 'stdin-end' diff --git a/src/main/codex/codex-app-server-connection.ts b/src/main/codex/codex-app-server-connection.ts index 80e9be692c77..54a7d92662e8 100644 --- a/src/main/codex/codex-app-server-connection.ts +++ b/src/main/codex/codex-app-server-connection.ts @@ -1,10 +1,10 @@ import { spawnProcess } from '../../shared/child-process/run-process' import { RetryableProcessExitProof } from '../../shared/child-process/retryable-process-exit-proof' +import { CODEX_APP_SERVER_CLOSE_REQUEST } from './codex-app-server-close-request' import { createProviderSpawnSpec, requestProviderClose, - stopSupervisedProvider, - type ProviderCloseRequest + stopSupervisedProvider } from './codex-app-server-posix-supervisor' import { buildCodexAppServerExitError } from './codex-app-server-exit-error' import { initializeCodexAppServerConnection } from './codex-app-server-handshake' @@ -49,8 +49,6 @@ export type CodexAppServerLaunch = { const DEFAULT_REQUEST_TIMEOUT_MS = 30_000 export const GRACEFUL_EXIT_MS = 1_500 -// Codex finishes its writes (auth.json, the state database) and exits on its stdin end. -const CODEX_APP_SERVER_CLOSE_REQUEST: ProviderCloseRequest = 'stdin-end' const FORCED_EXIT_MS = 1_000 const STDERR_TAIL_MAX_BYTES = 8192 From 5dfeaff9956806dc0a37abcefbc65af388485f10 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:50:13 -0700 Subject: [PATCH 32/38] build(cli): list the Codex close request and the provider supervisor in the CLI project The command-line build runs the short-lived Codex app-server session, which will name the same close request as the Codex connection. Listing the close request, the provider supervisor it takes its type from, and the spawn-failure report the supervisor uses lets the CLI project typecheck that import without pulling the connection in. --- config/tsconfig.cli.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/tsconfig.cli.json b/config/tsconfig.cli.json index 866bd903d3fd..b0b89e50bbf7 100644 --- a/config/tsconfig.cli.json +++ b/config/tsconfig.cli.json @@ -118,6 +118,7 @@ "../src/main/codex/codex-app-server-capability-cache.ts", "../src/main/codex/codex-app-server-capability-signal.ts", "../src/main/codex/codex-app-server-client.ts", + "../src/main/codex/codex-app-server-close-request.ts", "../src/main/codex/codex-app-server-process-tree-kill.ts", "../src/main/provider-process/provider-record-reader.ts", "../src/main/codex/codex-app-server-session.ts", @@ -152,6 +153,8 @@ "../src/main/codex/codex-managed-trust-reconciliation.ts", "../src/main/provider-process/provider-process-exit-deadline.ts", "../src/main/provider-process/provider-process-launch.ts", + "../src/main/provider-process/provider-process-supervisor.ts", + "../src/main/provider-process/provider-spawn-failure-report.ts", "../src/main/codex/codex-real-home-background-grant.ts", "../src/main/codex/codex-real-home-entry-trust.ts", "../src/main/codex/codex-real-home-hook-entry-plan.ts", From 2e230198bbabbf1f07f80de687df1cd0f8e443b0 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:27:34 -0700 Subject: [PATCH 33/38] test(codex): give a stand-in 20 s to report its pids on a loaded host At a load average near 50 both owner-death tests timed out waiting for the bundled owner's stand-in at 10 s; they pass alone. --- src/main/codex/supervised-probe-owner.test-fixture.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/codex/supervised-probe-owner.test-fixture.ts b/src/main/codex/supervised-probe-owner.test-fixture.ts index a8b131e8008c..e5d9b647e00f 100644 --- a/src/main/codex/supervised-probe-owner.test-fixture.ts +++ b/src/main/codex/supervised-probe-owner.test-fixture.ts @@ -127,7 +127,7 @@ export function createSupervisedProbeRig(): SupervisedProbeRig { recordedPids.add(owner.pid!) return owner }, - readPids: async (timeoutMs = 10_000) => { + readPids: async (timeoutMs = 20_000) => { if (!(await waitFor(() => existsSync(pidFile), timeoutMs))) { throw new Error('the stand-in never reported its pids') } From 91fb9351c635e6bf71060c3059f74ddf1988b2a7 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:26:58 -0700 Subject: [PATCH 34/38] test(codex): start the deadline tests' clocks past the stand-in's start, and cover a server that ignores SIGTERM The session and trust-grant deadline tests ran a 2-4 s deadline from spawn, so a loaded host could stop the stand-in before it wrote its pid. The session test's deadline now outlasts its pid-read budget, and the trust-grant deadlines are 8 s. The trust-grant comment said a wedged server may ignore everything but SIGKILL, but that stub dies on its stdin end; a new POSIX case pins that a server ignoring both its stdin end and SIGTERM is SIGKILLed after the SIGTERM grace. --- .../codex/codex-app-server-client.test.ts | 55 +++++++++++++++---- ...dex-app-server-session-supervision.test.ts | 14 ++++- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/src/main/codex/codex-app-server-client.test.ts b/src/main/codex/codex-app-server-client.test.ts index 5d54689a89e5..65b8e2d8b203 100644 --- a/src/main/codex/codex-app-server-client.test.ts +++ b/src/main/codex/codex-app-server-client.test.ts @@ -12,6 +12,10 @@ import { type CodexHookTrustGrantRequest } from './codex-app-server-client' import { killCodexAppServerProcessTree } from './codex-app-server-process-tree-kill' +import { + PROVIDER_SIGTERM_GRACE_MS, + PROVIDER_SUPERVISOR_MAX_STOP_MS +} from '../provider-process/provider-process-supervisor' import { runCodexAppServerSession } from './codex-app-server-session' // Stub codex app-server speaking the same JSONL protocol: initialize → @@ -19,6 +23,11 @@ import { runCodexAppServerSession } from './codex-app-server-session' // via STUB_CONFIG so each test controls listings, errors, and hangs. const STUB_SERVER_SOURCE = ` const config = JSON.parse(process.env.STUB_CONFIG) +// A wedged server ignores SIGTERM and its stdin end, so only SIGKILL ends it. +if (config.scenario === 'wedged') { + process.on('SIGTERM', () => {}) + setInterval(() => {}, 60000) +} require('node:fs').writeFileSync(config.pidFile, String(process.pid)) const trusted = new Set(config.hooks.filter(h => h.trustStatus === 'trusted').map(h => h.key)) let buffer = '' @@ -68,7 +77,7 @@ process.stdin.on('data', (chunk) => { continue } if (message.method === 'initialized') continue - if (config.scenario === 'hang') continue + if (config.scenario === 'hang' || config.scenario === 'wedged') continue if (message.method === 'hooks/list') { if (config.scenario === 'unknown-method') { send({ id: message.id, error: { code: -32601, message: 'Method not found' } }) @@ -88,7 +97,7 @@ process.stdin.on('data', (chunk) => { } } }) -process.stdin.on('end', () => process.exit(0)) +process.stdin.on('end', () => { if (config.scenario !== 'wedged') process.exit(0) }) function writeFileSyncSafe(file, contents) { require('node:fs').writeFileSync(file, contents) } ` @@ -149,6 +158,9 @@ function createStubRequest(options: { } const MANAGED_COMMAND = "/bin/sh '/tmp/orca/codex-hook.sh'" +// The deadline starts at spawn and the stub writes its pid only once it runs; on a loaded host the +// supervisor and stub have taken over 2 s to start, so a shorter deadline can stop it first. +const STUB_START_DEADLINE_MS = 8_000 function managedHook(key: string, trustStatus = 'untrusted'): StubHook { return { key, command: MANAGED_COMMAND, currentHash: `sha256:hash-of-${key}`, trustStatus } @@ -393,36 +405,59 @@ describe('runCodexHookTrustGrantSession', () => { hooks: keys.map((key) => managedHook(key)), expectedTrustKeys: keys, managedCommand: MANAGED_COMMAND, - // Long enough for the supervisor and stub to start on a loaded host and write the pid. - timeoutMs: 2_000 + timeoutMs: STUB_START_DEADLINE_MS }) const startedAt = Date.now() await expect(runCodexHookTrustGrantSession(request)).rejects.toBeInstanceOf( CodexAppServerTimeoutError ) - // Why: the reap path must not stack the grace periods on top of the - // deadline — a wedged server may ignore everything but SIGKILL. - expect(Date.now() - startedAt).toBeLessThan(5_000) + // Why: this stub exits on its stdin end and on SIGTERM, so the reap adds no grace on top of + // the deadline; a server that ignores both is the case below. + expect(Date.now() - startedAt - STUB_START_DEADLINE_MS).toBeLessThan(2_000) const childPid = Number(readFileSync(pidFile, 'utf8')) expect(() => process.kill(childPid, 0)).toThrow() }) + it.runIf(process.platform !== 'win32')( + 'stops a server that ignores its stdin end and SIGTERM after the SIGTERM grace', + async () => { + const keys = ['/home/a/.codex/hooks.json:session_start:0:0'] + const { request, pidFile } = createStubRequest({ + scenario: 'wedged', + hooks: keys.map((key) => managedHook(key)), + expectedTrustKeys: keys, + managedCommand: MANAGED_COMMAND, + timeoutMs: STUB_START_DEADLINE_MS + }) + + const startedAt = Date.now() + await expect(runCodexHookTrustGrantSession(request)).rejects.toBeInstanceOf( + CodexAppServerTimeoutError + ) + // The supervisor SIGTERMs its group at the deadline and SIGKILLs it after the grace. + const pastDeadline = Date.now() - startedAt - STUB_START_DEADLINE_MS + expect(pastDeadline).toBeGreaterThanOrEqual(PROVIDER_SIGTERM_GRACE_MS - 100) + expect(pastDeadline).toBeLessThan(PROVIDER_SUPERVISOR_MAX_STOP_MS + 1_000) + const childPid = Number(readFileSync(pidFile, 'utf8')) + expect(() => process.kill(childPid, 0)).toThrow() + } + ) + it('bounds a callback that stalls between RPC requests', async () => { const { request, pidFile } = createStubRequest({ scenario: 'happy', hooks: [], expectedTrustKeys: [], managedCommand: MANAGED_COMMAND, - // Long enough for the supervisor and stub to start on a loaded host and write the pid. - timeoutMs: 2_000 + timeoutMs: STUB_START_DEADLINE_MS }) const startedAt = Date.now() await expect( runCodexAppServerSession(request.invocation, async () => new Promise(() => {})) ).rejects.toBeInstanceOf(CodexAppServerTimeoutError) - expect(Date.now() - startedAt).toBeLessThan(5_000) + expect(Date.now() - startedAt - STUB_START_DEADLINE_MS).toBeLessThan(2_000) const childPid = Number(readFileSync(pidFile, 'utf8')) expect(() => process.kill(childPid, 0)).toThrow() }) diff --git a/src/main/codex/codex-app-server-session-supervision.test.ts b/src/main/codex/codex-app-server-session-supervision.test.ts index 1ea14fdebff6..c7649e3d4f76 100644 --- a/src/main/codex/codex-app-server-session-supervision.test.ts +++ b/src/main/codex/codex-app-server-session-supervision.test.ts @@ -148,11 +148,19 @@ describe.runIf(process.platform !== 'win32')('supervised Codex app-server sessio const readPids = rig.readPids const reported: { pids?: { provider: number; grandchild: number } } = {} - // The deadline must find the stand-in armed, however slowly a loaded host starts it. + // The deadline starts at spawn, so it outlasts the stand-in's whole start budget: a slow host + // fails the pid read inside that budget instead of stopping a stand-in that never armed. + const startBudgetMs = 8_000 const session = runCodexAppServerSession( - { command: standIn, cliPath: null, args: ['app-server'], env: rig.env, timeoutMs: 4_000 }, + { + command: standIn, + cliPath: null, + args: ['app-server'], + env: rig.env, + timeoutMs: startBudgetMs + 2_000 + }, async () => { - reported.pids = await readPids() + reported.pids = await readPids(startBudgetMs) return new Promise(() => {}) } ) From f31774ac7e1a5b8b13cf1f32f075030449612a21 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:16:32 -0700 Subject: [PATCH 35/38] test(text-generation): check the ENOEXEC start failure only where Node reports one On Linux, glibc's execvp hands an executable that is not a program to /bin/sh, so both a direct and a supervised spawn run it and it exits 127; only macOS throws ENOEXEC. The not-a-program case now runs on macOS only; the path-through-a-file case (ENOTDIR) still covers a thrown start failure everywhere. --- .../text-generation/source-control-agent-launch.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/text-generation/source-control-agent-launch.test.ts b/src/main/text-generation/source-control-agent-launch.test.ts index 189db8497a41..7c1670361c7a 100644 --- a/src/main/text-generation/source-control-agent-launch.test.ts +++ b/src/main/text-generation/source-control-agent-launch.test.ts @@ -118,8 +118,11 @@ describe.runIf(process.platform !== 'win32')('supervised agent processes', () => }) it.each([ - ['an executable that is not a program', 'garbage', (path: string) => path], - ['a path through a file', 'file', (path: string) => join(path, 'agent')] + // Only macOS throws ENOEXEC; glibc's execvp hands such a file to /bin/sh, direct or supervised. + ...(process.platform === 'darwin' + ? [['an executable that is not a program', 'garbage', (path: string) => path] as const] + : []), + ['a path through a file', 'file', (path: string) => join(path, 'agent')] as const ])('reports %s as not startable, as a direct spawn does', async (_, name, commandFor) => { const file = join(folder, name) writeFileSync(file, Buffer.from([0, 1, 2, 3]), { mode: 0o755 }) From b86c5b2579383649aa6fdb2b14c4aada5d14df42 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:37:24 -0700 Subject: [PATCH 36/38] test(wsl): follow the backfill's wsl.exe spawn into its new process module The WSL invocation boundary lists files that spawn wsl.exe directly. The backfill recovery's spawn moved into codex-state-db-backfill-recovery-process.ts, so the entry moves with it; the count is unchanged. --- src/main/wsl/__fixtures__/wsl-invocation-allowlist.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/wsl/__fixtures__/wsl-invocation-allowlist.txt b/src/main/wsl/__fixtures__/wsl-invocation-allowlist.txt index 45f99ffa2467..68509ee18d1d 100644 --- a/src/main/wsl/__fixtures__/wsl-invocation-allowlist.txt +++ b/src/main/wsl/__fixtures__/wsl-invocation-allowlist.txt @@ -18,7 +18,7 @@ main/agent-hooks/wsl-hook-relay-launch.ts main/browser/wsl-browser-network-relay-launch.ts main/claude-accounts/claude-command-process.ts main/codex-accounts/service.ts -main/codex/codex-state-db-backfill-recovery.ts +main/codex/codex-state-db-backfill-recovery-process.ts main/codex/codex-trust-grant-host.ts main/codex/codex-wsl-hook-install-plan.ts main/git/command-runner/wsl-command-resolution.ts From f289d1e20d249bf8e54c5496429cd38602030dd5 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:54:28 -0700 Subject: [PATCH 37/38] test(codex): keep factory child_process mocks loadable now that Codex stops reach the process-table reader The backfill recovery and Codex sessions now stop through the shared supervised teardown, whose process-table reader binds execFile when it loads. Five rate-limit fetcher tests mocked node:child_process with only spawn and failed at load: they now mock the backfill recovery, as their sibling fetcher tests already do. The account add-login tests' child_process mocks gain an execFile stub. --- .../codex-accounts/service-account-add-login.test.ts | 12 ++++++++++++ .../rate-limits/codex-fetcher-auth-errors.test.ts | 5 +++++ src/main/rate-limits/codex-fetcher-backend.test.ts | 5 +++++ .../rate-limits/codex-fetcher-credit-balance.test.ts | 5 +++++ .../rate-limits/codex-fetcher-probe-shutdown.test.ts | 5 +++++ .../codex-fetcher-session-supplement.test.ts | 5 +++++ 6 files changed, 37 insertions(+) diff --git a/src/main/codex-accounts/service-account-add-login.test.ts b/src/main/codex-accounts/service-account-add-login.test.ts index 4bde86486cc2..d9b963d8bc92 100644 --- a/src/main/codex-accounts/service-account-add-login.test.ts +++ b/src/main/codex-accounts/service-account-add-login.test.ts @@ -82,6 +82,8 @@ describe('CodexAccountService config sync', () => { ) vi.doMock('node:child_process', () => ({ + // The supervised Codex teardown's process-table reader binds execFile when it loads. + execFile: vi.fn(), execFileSync: vi.fn(), spawn: spawnMock })) @@ -146,6 +148,8 @@ describe('CodexAccountService config sync', () => { ) vi.doMock('node:child_process', () => ({ + // The supervised Codex teardown's process-table reader binds execFile when it loads. + execFile: vi.fn(), execFileSync: vi.fn(), spawn: spawnMock })) @@ -254,6 +258,8 @@ describe('CodexAccountService config sync', () => { ) vi.doMock('node:child_process', () => ({ + // The supervised Codex teardown's process-table reader binds execFile when it loads. + execFile: vi.fn(), execFileSync: vi.fn(), spawn: spawnMock })) @@ -411,6 +417,8 @@ describe('CodexAccountService config sync', () => { } ) vi.doMock('node:child_process', () => ({ + // The supervised Codex teardown's process-table reader binds execFile when it loads. + execFile: vi.fn(), execFileSync: vi.fn(), spawn: spawnMock })) @@ -486,6 +494,8 @@ describe('CodexAccountService config sync', () => { const spawnMock = vi.fn() vi.doMock('node:child_process', () => ({ + // The supervised Codex teardown's process-table reader binds execFile when it loads. + execFile: vi.fn(), execFileSync: vi.fn(), spawn: spawnMock })) @@ -532,6 +542,8 @@ describe('CodexAccountService config sync', () => { const spawnMock = vi.fn() vi.doMock('node:child_process', () => ({ + // The supervised Codex teardown's process-table reader binds execFile when it loads. + execFile: vi.fn(), execFileSync: vi.fn(), spawn: spawnMock })) diff --git a/src/main/rate-limits/codex-fetcher-auth-errors.test.ts b/src/main/rate-limits/codex-fetcher-auth-errors.test.ts index 5ce9df8675eb..20405233c9df 100644 --- a/src/main/rate-limits/codex-fetcher-auth-errors.test.ts +++ b/src/main/rate-limits/codex-fetcher-auth-errors.test.ts @@ -11,6 +11,11 @@ vi.mock('node:child_process', () => ({ spawn: childSpawnMock })) +// The recovery's supervised stop imports the process-table reader, which needs the real execFile. +vi.mock('../codex/codex-state-db-backfill-recovery', () => ({ + startCodexStateDbBackfillRecoveryInBackground: vi.fn() +})) + vi.mock('../codex-cli/command', () => ({ resolveCodexCommand: resolveCodexCommandMock })) diff --git a/src/main/rate-limits/codex-fetcher-backend.test.ts b/src/main/rate-limits/codex-fetcher-backend.test.ts index 3fefe5e3a2fd..b1c01a2a6e8c 100644 --- a/src/main/rate-limits/codex-fetcher-backend.test.ts +++ b/src/main/rate-limits/codex-fetcher-backend.test.ts @@ -15,6 +15,11 @@ vi.mock('./codex-auth-presence', () => ({ probeCodexAuthPresence: vi.fn(async () => 'present') })) +// The recovery's supervised stop imports the process-table reader, which needs the real execFile. +vi.mock('../codex/codex-state-db-backfill-recovery', () => ({ + startCodexStateDbBackfillRecoveryInBackground: vi.fn() +})) + import { consumeCodexRateLimitResetCredit, fetchCodexRateLimits } from './codex-fetcher' describe('Codex backend rate-limit requests', () => { diff --git a/src/main/rate-limits/codex-fetcher-credit-balance.test.ts b/src/main/rate-limits/codex-fetcher-credit-balance.test.ts index 780687c9a09d..016357735e94 100644 --- a/src/main/rate-limits/codex-fetcher-credit-balance.test.ts +++ b/src/main/rate-limits/codex-fetcher-credit-balance.test.ts @@ -14,6 +14,11 @@ vi.mock('./codex-auth-presence', () => ({ probeCodexAuthPresence: vi.fn(async () => 'present') })) +// The recovery's supervised stop imports the process-table reader, which needs the real execFile. +vi.mock('../codex/codex-state-db-backfill-recovery', () => ({ + startCodexStateDbBackfillRecoveryInBackground: vi.fn() +})) + import { fetchCodexRateLimits } from './codex-fetcher' function parseRpcRequest(line: string): { id?: number; method?: string } { diff --git a/src/main/rate-limits/codex-fetcher-probe-shutdown.test.ts b/src/main/rate-limits/codex-fetcher-probe-shutdown.test.ts index d5b2d56b6817..791bcf4c7bbf 100644 --- a/src/main/rate-limits/codex-fetcher-probe-shutdown.test.ts +++ b/src/main/rate-limits/codex-fetcher-probe-shutdown.test.ts @@ -12,6 +12,11 @@ vi.mock('node:child_process', () => ({ spawn: childSpawnMock })) +// The recovery's supervised stop imports the process-table reader, which needs the real execFile. +vi.mock('../codex/codex-state-db-backfill-recovery', () => ({ + startCodexStateDbBackfillRecoveryInBackground: vi.fn() +})) + vi.mock('node:fs/promises', () => ({ readFile: readFileMock })) diff --git a/src/main/rate-limits/codex-fetcher-session-supplement.test.ts b/src/main/rate-limits/codex-fetcher-session-supplement.test.ts index 232bfc9a83dc..44c086137215 100644 --- a/src/main/rate-limits/codex-fetcher-session-supplement.test.ts +++ b/src/main/rate-limits/codex-fetcher-session-supplement.test.ts @@ -14,6 +14,11 @@ vi.mock('./codex-auth-presence', () => ({ probeCodexAuthPresence: vi.fn(async () => 'present') })) +// The recovery's supervised stop imports the process-table reader, which needs the real execFile. +vi.mock('../codex/codex-state-db-backfill-recovery', () => ({ + startCodexStateDbBackfillRecoveryInBackground: vi.fn() +})) + import { fetchCodexRateLimits } from './codex-fetcher' function makeRpcChild(rateLimitResetCredits?: unknown) { From fba19a910539cff5f45defc3603bd8b7dcdac19a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 6 Oct 2026 02:27:06 -0700 Subject: [PATCH 38/38] fix(opencode): count no forced preflight stop as proof the server is gone A forced teardown walks and group-kills the supervisor's tree, but the server leads its own detached group, so a teardown verdict of 'exited' does not cover members left in the server's group. Only a supervisor that reaped the group itself, by its own stop or relaying the server's exit, now counts as a proven stop. The Windows session-stop test now says why it sees no direct kill. --- src/main/codex/codex-app-server-session-stop.test.ts | 2 ++ .../opencode-launch-model-context-cleanup.test.ts | 5 +++-- src/main/opencode/opencode-launch-model-context.ts | 10 +++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/codex/codex-app-server-session-stop.test.ts b/src/main/codex/codex-app-server-session-stop.test.ts index 32802fb1673f..3972613394f2 100644 --- a/src/main/codex/codex-app-server-session-stop.test.ts +++ b/src/main/codex/codex-app-server-session-stop.test.ts @@ -126,6 +126,8 @@ describe('runCodexAppServerSession stop', () => { await vi.advanceTimersByTimeAsync(1_000) expect(await outcome).toBeInstanceOf(CodexAppServerTimeoutError) + // The session sends no signal of its own on Windows (a SIGTERM there is TerminateProcess); + // the teardowns, mocked here, own the kill. expect(child.kill).not.toHaveBeenCalled() expect(killCodexAppServerProcessTree).toHaveBeenCalledTimes(1) }) diff --git a/src/main/opencode/opencode-launch-model-context-cleanup.test.ts b/src/main/opencode/opencode-launch-model-context-cleanup.test.ts index e655c68fc5f7..cf5df3a7d7f1 100644 --- a/src/main/opencode/opencode-launch-model-context-cleanup.test.ts +++ b/src/main/opencode/opencode-launch-model-context-cleanup.test.ts @@ -193,9 +193,10 @@ describe('OpenCode model probe termination evidence', () => { }) }) - it('accepts a forced POSIX stop whose teardown proved the tree gone', async () => { + it('trusts no forced POSIX stop, even one whose teardown reports the tree gone', async () => { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) child.kill.mockReturnValue(true) + // 'exited' covers the supervisor's tree only; the server leads its own detached group. vi.mocked(terminateProviderProcessTree).mockImplementation(async () => { supervisorExits(null, 'SIGKILL') return 'exited' @@ -205,7 +206,7 @@ describe('OpenCode model probe termination evidence', () => { await untilStopRequested() await vi.advanceTimersByTimeAsync(PROVIDER_SUPERVISOR_MAX_STOP_MS) - expect(await probe).toMatchObject({ primaryAgent: 'build' }) + expect(await probe).toBeNull() }) it('trusts no POSIX stop whose supervisor exited 1, which a failed group reap also exits', async () => { diff --git a/src/main/opencode/opencode-launch-model-context.ts b/src/main/opencode/opencode-launch-model-context.ts index 036f092b2261..134b7b4ab92e 100644 --- a/src/main/opencode/opencode-launch-model-context.ts +++ b/src/main/opencode/opencode-launch-model-context.ts @@ -225,8 +225,8 @@ export async function probeOpenCodeLaunchModelContext(options: { /** * True only when the server's group is proven gone: the supervisor exited on its own stop or - * relayed the server's exit, or a forced teardown proved the tree gone. A forced supervisor dies - * by SIGKILL and an unverified teardown proves nothing; exit 1 can be a group reap that failed. + * relayed the server's exit. A forced stop proves nothing: its teardown reaches the supervisor's + * group, not the server's own detached group. Exit 1 can be a group reap that failed. */ async function stopSupervisedPreflight( child: ChildProcessHandle, @@ -239,9 +239,9 @@ async function stopSupervisedPreflight( const { root, tree } = await stopSupervisedChildProcess(child, { site: OPENCODE_PREFLIGHT_KILL_SITE }) - const supervisorReapedGroup = - child.signalCode !== 'SIGKILL' && !(child.signalCode === null && child.exitCode === 1) - if (root !== 'exited' || !(tree === 'exited' || (tree === null && supervisorReapedGroup))) { + const forced = tree !== null || child.signalCode === 'SIGKILL' + const failedReap = child.signalCode === null && child.exitCode === 1 + if (root !== 'exited' || forced || failedReap) { return false } return withTimeout(closed, SUPERVISOR_PIPE_CLOSE_MS, false)