Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ import type { WorkerStartInput } from './worker-start-schema'
import {
persistGatedSetupSpawnFailure,
persistWorkerReadinessStage,
persistWorkerSetupWaitOutcome
persistWorkerSetupWaitOutcome,
setupErrorAtReadinessTimeout,
setupStillRunningError
} from './worker-setup-gate'
import { failWorkerStartWithReceipt } from './worker-start-receipt'
import { parseTaskDeps } from './task-deps-argument'
Expand Down Expand Up @@ -202,24 +204,54 @@ export async function startLocalWorker(args: {
// A structured session is ready the moment its attach returns ok: there is no boot-to-idle
// gap and no terminal title to read an idle edge from. Only the repo's wait-for-setup policy
// still holds it back, and that gate has to be waited on explicitly here.
const wait = structuredSession
? await awaitStructuredWorkerSetupGate({
runtime,
setup: setupReceipt,
effects,
timeoutMs: params.timeoutMs ?? 60_000
})
: await waitForWorkerAgentReady(runtime, terminalHandle, {
agent,
reusesTerminal: Boolean(params.terminal),
timeoutMs: params.timeoutMs ?? 60_000
})
const readinessTimeoutMs = params.timeoutMs ?? 60_000
// Under wait-for-setup a readiness timeout may be setup's, not the agent's. The structured
// gate's timeout is its own setup-completion race; a terminal one asks the setup terminal.
const failIfSetupBlockedAgent = async (): Promise<void> => {
const setupError = structuredSession
? setupStillRunningError(readinessTimeoutMs)
: await setupErrorAtReadinessTimeout({
...setupStage,
runtime,
timeoutMs: readinessTimeoutMs
})
if (setupError) {
failedStage = 'setup_wait'
throw setupError
}
}
let wait:
| Awaited<ReturnType<typeof awaitStructuredWorkerSetupGate>>
| Awaited<ReturnType<typeof waitForWorkerAgentReady>>
try {
wait = structuredSession
? await awaitStructuredWorkerSetupGate({
runtime,
setup: setupReceipt,
effects,
timeoutMs: readinessTimeoutMs
})
: await waitForWorkerAgentReady(runtime, terminalHandle, {
agent,
reusesTerminal: Boolean(params.terminal),
timeoutMs: readinessTimeoutMs
})
} catch (error) {
// A terminal readiness wait rejects with 'timeout' rather than returning a status.
if (error instanceof Error && error.message === 'timeout') {
await failIfSetupBlockedAgent()
}
throw error
}
if (wait) {
persistWorkerSetupWaitOutcome({ ...setupStage, wait })
if (!wait.satisfied) {
if (setupReceipt.state === 'failed') {
failedStage = 'setup_wait'
}
if (wait.status === 'timeout') {
await failIfSetupBlockedAgent()
}
throw new Error(
wait.blockedReason
? `Agent startup blocked: ${describeTerminalWaitBlockedReason(wait.blockedReason)}`
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import type { OrchestrationDb } from '../../../../orchestration/db'
import {
applyWaitForSetupOutcome,
Expand Down Expand Up @@ -67,3 +68,67 @@ export function persistWorkerSetupWaitOutcome(
residualResources: residualWorkerEffects(args.effects)
})
}

/** How long a readiness timeout asks the setup terminal whether setup exited. An exited setup
* normally answers at once from its replayed completion marker; a running one uses the whole probe. */
const SETUP_EXIT_PROBE_MS = 1_000

export function setupStillRunningError(timeoutMs: number): Error {
return new Error(
`Setup was still running after ${timeoutMs} ms (startup policy wait-for-setup), so the agent could not start. Retry with a larger --timeout-ms.`
)
}

/**
* Under wait-for-setup the agent cannot start before setup exits, so a terminal readiness timeout
* asks the setup terminal what setup did. The receipt cannot answer: it still says `running`,
* because only a readiness wait that returns updates it, and a failed setup leaves the pane's shell
* alive, so that wait times out too.
*
* Returns the setup error to report, or null to keep the agent-readiness timeout: setup exited 0,
* or there is no setup terminal to ask. An exit it observes is persisted as the setup outcome.
*/
export async function setupErrorAtReadinessTimeout(
args: WorkerSetupStageArgs & {
runtime: Pick<OrcaRuntimeService, 'waitForSetupTerminalCompletion'>
timeoutMs: number
}
): Promise<Error | null> {
if (args.setup.startupPolicy !== 'wait-for-setup' || args.setup.state !== 'running') {
return null
}
const setupTerminal = args.effects.find((effect) => effect.kind === 'setup')?.terminalId
if (!setupTerminal) {
return null
}
const abort = new AbortController()
let timer: ReturnType<typeof setTimeout> | undefined
let completion: { exitCode: number | null } | 'running'
try {
completion = await Promise.race([
args.runtime.waitForSetupTerminalCompletion(setupTerminal, abort.signal),
new Promise<'running'>((resolve) => {
timer = setTimeout(() => resolve('running'), SETUP_EXIT_PROBE_MS)
})
])
} catch {
// A setup terminal that is already gone gives no evidence either way.
return null
} finally {
clearTimeout(timer)
abort.abort()
}
if (completion === 'running') {
return setupStillRunningError(args.timeoutMs)
}
persistWorkerSetupWaitOutcome({
...args,
wait: { satisfied: completion.exitCode === 0, status: 'exited' }
})
if (completion.exitCode === 0) {
return null
}
return new Error(
`Setup failed (exit ${completion.exitCode ?? 'unknown'}), so the agent could not start (startup policy wait-for-setup).`
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@ import { OrchestrationDb } from '../../../../orchestration/db'
import { RpcDispatcher } from '../../../dispatcher'
import type { RpcRequest } from '../../../core'
import { ORCHESTRATION_METHODS } from '../../orchestration'
import { placeWorkerAgent } from './worker-start-agent-placement'

vi.mock(import('./worker-start-agent-placement'), async (importOriginal) => {
const actual = await importOriginal()
return { ...actual, placeWorkerAgent: vi.fn(actual.placeWorkerAgent) }
})

describe('orchestration new-worktree workers', () => {
type CreateWorktreeResult = Awaited<ReturnType<OrcaRuntimeService['createManagedWorktree']>>
Expand Down Expand Up @@ -132,6 +138,15 @@ describe('orchestration new-worktree workers', () => {
}
}

function mockGatedSetupReadinessTimeout(state: 'running' | 'skipped') {
mockCreatedWorktree({
startupPolicy: 'wait-for-setup',
state,
setupTerminalHandle: 'term_setup'
})
vi.mocked(runtime.waitForTerminal).mockRejectedValue(new Error('timeout'))
}

it('creates an independent top-level worktree and reuses its agent terminal', async () => {
mockCreatedWorktree()

Expand Down Expand Up @@ -455,6 +470,130 @@ describe('orchestration new-worktree workers', () => {
expect(runtime.sendTerminalAgentPrompt).not.toHaveBeenCalled()
})

it('reports a readiness timeout while wait-for-setup is still running as a setup wait', async () => {
mockGatedSetupReadinessTimeout('running')

const { result } = await startWorker({ timeoutMs: 1_000 })

expect(result).toMatchObject({
state: 'failed',
failedStage: 'setup_wait',
setup: { startupPolicy: 'wait-for-setup', state: 'running' }
})
const lastError = (result as { lastError: string }).lastError
expect(lastError).toContain('Setup was still running after 1000 ms')
expect(lastError).toContain('--timeout-ms')
expect(runtime.waitForSetupTerminalCompletion).toHaveBeenCalledWith(
'term_setup',
expect.any(AbortSignal)
)
expect(runtime.sendTerminalAgentPrompt).not.toHaveBeenCalled()
})

it('reports a wait-for-setup readiness timeout after setup failed as a setup failure', async () => {
// The gate skips the agent and the pane's shell stays alive, so readiness can only time out.
mockGatedSetupReadinessTimeout('running')
vi.mocked(runtime.waitForSetupTerminalCompletion).mockResolvedValue({ exitCode: 3 })

const { result } = await startWorker({ timeoutMs: 1_000 })

expect(result).toMatchObject({
state: 'failed',
failedStage: 'setup_wait',
setup: { startupPolicy: 'wait-for-setup', state: 'failed' },
effects: expect.arrayContaining([expect.objectContaining({ kind: 'setup', state: 'failed' })])
})
const lastError = (result as { lastError: string }).lastError
expect(lastError).toContain('Setup failed (exit 3)')
expect(lastError).not.toContain('still running')
expect(runtime.sendTerminalAgentPrompt).not.toHaveBeenCalled()
})

it('keeps a wait-for-setup readiness timeout after setup succeeded at agent readiness', async () => {
mockGatedSetupReadinessTimeout('running')
vi.mocked(runtime.waitForSetupTerminalCompletion).mockResolvedValue({ exitCode: 0 })

const { result } = await startWorker({ timeoutMs: 1_000 })

expect(result).toMatchObject({
state: 'failed',
failedStage: 'agent_readiness',
lastError: 'timeout',
setup: { startupPolicy: 'wait-for-setup', state: 'succeeded' }
})
})

it('keeps a wait-for-setup readiness timeout with no running setup at agent readiness', async () => {
mockGatedSetupReadinessTimeout('skipped')

const { result } = await startWorker({ timeoutMs: 1_000 })

expect(result).toMatchObject({
state: 'failed',
failedStage: 'agent_readiness',
lastError: 'timeout',
setup: { state: 'skipped' }
})
expect(runtime.waitForSetupTerminalCompletion).not.toHaveBeenCalled()
})

it('reports a structured worker whose setup gate timed out as a setup wait', async () => {
vi.mocked(placeWorkerAgent).mockImplementationOnce(async (args) => {
args.effects.push({
kind: 'setup',
action: 'run',
startupPolicy: 'wait-for-setup',
state: 'running',
terminalId: 'term_setup'
})
return {
mode: args.mode,
worktree: { id: 'repo::created', repoId: 'repo' },
terminalHandle: 'structured_worker_1',
structuredSession: {
identity: { sessionId: 'session_1', handle: 'structured_worker_1' }
} as never,
setupReceipt: {
requested: 'run',
effective: 'run',
source: 'orchestration_default',
hookFound: true,
startupPolicy: 'wait-for-setup',
state: 'running'
}
}
})

const { result } = await startWorker({ timeoutMs: 50 })

expect(result).toMatchObject({
state: 'failed',
failedStage: 'setup_wait',
setup: { startupPolicy: 'wait-for-setup', state: 'running' }
})
expect((result as { lastError: string }).lastError).toContain(
'Setup was still running after 50 ms'
)
expect(runtime.waitForTerminal).not.toHaveBeenCalled()
})

it('keeps a start-immediately readiness timeout at agent readiness', async () => {
mockCreatedWorktree({
startupPolicy: 'start-immediately',
state: 'running',
setupTerminalHandle: 'term_setup'
})
vi.mocked(runtime.waitForTerminal).mockRejectedValue(new Error('timeout'))

const { result } = await startWorker({ timeoutMs: 1_000 })

expect(result).toMatchObject({
state: 'failed',
failedStage: 'agent_readiness',
lastError: 'timeout'
})
})

it('distinguishes no-effect failure, unknown acceptance, and durable residual effects', async () => {
vi.spyOn(runtime, 'createManagedWorktree').mockRejectedValueOnce(
new Error('repository validation failed before creation')
Expand Down