Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,8 @@ import {
createRequestCanceledError,
isRequestCanceledError,
} from '@agent-device/kernel/errors';
import {
abortAllIosRunnerSessions,
DEFAULT_RUNNER_START_BUDGET_MS,
readRunnerSessionLiveness,
} from '../runner-session.ts';
import { abortAllIosRunnerSessions, readRunnerSessionLiveness } from '../runner-session.ts';
import { DEFAULT_RUNNER_START_BUDGET_MS } from '../runner-start-budget.ts';
import { RUNNER_STARTUP_TIMEOUT_MS } from '../runner-startup-transport.ts';
import type { RunnerLease } from '../runner-lease.ts';
import { executeRunnerCommand, prepareLocalIosRunner } from '../runner-lifecycle.ts';
Expand Down
122 changes: 4 additions & 118 deletions packages/platform-apple/src/runner/runner-session.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
import {
AppError,
createRequestCanceledError,
isRequestCanceledError,
} from '@agent-device/kernel/errors';
import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors';
import {
withKeyedLock,
Deadline,
Expand Down Expand Up @@ -39,7 +35,6 @@ import {
decodeRunnerResponseBody,
isRunnerResponseOk,
readRunnerResponseData,
resolveRunnerStartupSignal,
withRunnerCommandId,
type RunnerCommand,
} from './runner-contract.ts';
Expand Down Expand Up @@ -96,6 +91,7 @@ import {
} from './runner-session-types.ts';
import { launchRunnerProcess, type LaunchedRunnerProcess } from './runner-process-launch.ts';
import { isSameRunnerSimulator } from './runner-device-set.ts';
import type { RunnerStartBudget } from './runner-start-budget.ts';

export type { RunnerSession } from './runner-session-types.ts';

Expand Down Expand Up @@ -134,33 +130,6 @@ function withRunnerSessionLock<T>(deviceId: string, task: () => Promise<T>): Pro
return withKeyedLock(runnerSessionLocks, deviceId, task);
}

/**
* What a runner start may spend when its caller sets no `startupTimeoutMs`: the reuse probe,
* adoption, boot, device readiness, the xctestrun build and the launch, on one clock the start owns
* (#2894). Sized for a cold build: `prepare` defaults to 240 s for the same work plus its health
* check (`PREPARE_STARTUP_BUDGET_MS`), the iOS CI lane gives a cold `prepare` on a shared macOS
* runner 420 s (`AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS`), and a daemon queued on another process's
* build of the same artifact already gives up after 10 minutes
* (`RUNNER_XCTESTRUN_CACHE_LOCK_TIMEOUT_MS`). The two ceilings agree, so a start never waits on a
* peer's build longer than it would spend on its own.
*/
export const DEFAULT_RUNNER_START_BUDGET_MS = 10 * 60_000;

/**
* The budget one runner start spends, with the clock that enforces it. The abort fires when the
* deadline is spent, so every step that takes the signal (the xctestrun build and the launch kill
* their process tree through exec, the probes stop retrying) ends on the same clock the deadline
* reads. `close` retires the timer once the start has settled: a registered session is bounded by
* its {@link RunnerSession.launchDeadline} from then on, never by this abort.
*/
type RunnerStartBudget = Readonly<{
phase: RunnerPhaseBudget;
timeoutMs: number;
explicit: boolean;
exhausted: AbortSignal;
close: () => void;
}>;

export async function ensureRunnerSession(
device: DeviceInfo,
options: RunnerSessionOptions,
Expand All @@ -169,6 +138,7 @@ export async function ensureRunnerSession(
// from a retained-after-close runner no longer applies.
cancelIosRunnerIdleStop(device.id);
const start = withRunnerSessionLock(device.id, async () => {
const { openRunnerStartBudget } = await import('./runner-start-budget.ts');
// One budget for the whole start, opened once the lock is held so a start queued behind
// another does not spend its clock waiting: the reuse check's toolchain probes, adoption and
// the startup itself all read it. The request's cancellation rides with it, so a client
Expand All @@ -195,94 +165,10 @@ export async function ensureRunnerSession(
budget.close();
}
});
const { raceRunnerStartAgainstCaller } = await import('./runner-start-budget.ts');
return await raceRunnerStartAgainstCaller(start, options.signal);
}

/**
* Opens the start's budget from `startupTimeoutMs`, or {@link DEFAULT_RUNNER_START_BUDGET_MS}
* when the caller sets none. The startup signal (a cancelled request, never a caller's deadline)
* and the budget's own expiry abort the same signal.
*/
function openRunnerStartBudget(options: RunnerSessionOptions): RunnerStartBudget {
const explicitTimeoutMs = normalizeRunnerStartupTimeoutMs(options.startupTimeoutMs);
const timeoutMs = explicitTimeoutMs ?? DEFAULT_RUNNER_START_BUDGET_MS;
const exhausted = new AbortController();
const timer = setTimeout(() => {
exhausted.abort(runnerStartBudgetExhaustedError(timeoutMs, explicitTimeoutMs !== undefined));
}, timeoutMs);
timer.unref?.();
const startupSignal = resolveRunnerStartupSignal(options);
const signal = startupSignal
? AbortSignal.any([startupSignal, exhausted.signal])
: exhausted.signal;
return {
phase: createRunnerPhaseBudget(timeoutMs, signal),
timeoutMs,
explicit: explicitTimeoutMs !== undefined,
exhausted: exhausted.signal,
close: () => clearTimeout(timer),
};
}

/** Says the start's own budget ran out, whichever step it was in; the next request starts over. */
function runnerStartBudgetExhaustedError(timeoutMs: number, explicit: boolean): AppError {
return new AppError(
'COMMAND_FAILED',
`Apple runner start exceeded its ${explicit ? 'startup timeout' : 'default start budget'} of ${timeoutMs}ms`,
{
reason: 'runner_start_budget_exhausted',
timeoutMs,
retriable: true,
hint: explicit
? 'Raise the startup timeout, or run `prepare ios-runner` first so the runner is built before it is needed.'
: 'Run `prepare ios-runner --timeout <ms>` so the cold build has a budget of your choosing; the session runner.log names the step that stalled.',
},
);
}

/**
* The start runs detached under the session lock; the caller only waits for it as long as its own
* signal allows. A caller whose deadline lands during a cold xctestrun build leaves on time, the
* build keeps going under the lock, and the next request for the device queues behind it and joins
* the session it registers (#2894). Whatever the abort reason, the caller sees the same cancelled
* request it would have seen from any later step; a cancelled request's abort also reaches the
* start through its own startup signal, so nothing here decides whether the start survives. A
* start that fails after its caller left has nobody to report to, so its failure is logged here.
*/
async function raceRunnerStartAgainstCaller(
start: Promise<RunnerSession>,
signal: AbortSignal | undefined,
): Promise<RunnerSession> {
if (!signal) return await start;
return await new Promise<RunnerSession>((resolve, reject) => {
const abort = () => {
reject(createRequestCanceledError(undefined, signal.reason));
start.catch(emitDetachedRunnerStartFailed);
};
if (signal.aborted) {
abort();
} else {
signal.addEventListener('abort', abort, { once: true });
}
start.then(resolve, reject).finally(() => signal.removeEventListener('abort', abort));
});
}

/** A cancelled request killed its start on purpose; any other failure of a start nobody awaits is news. */
function emitDetachedRunnerStartFailed(error: unknown): void {
if (isRequestCanceledError(error)) return;
const appErr = error instanceof AppError ? error : undefined;
emitDiagnostic({
level: 'warn',
phase: 'ios_runner_detached_start_failed',
data: {
code: appErr?.code,
reason: appErr?.details?.reason,
error: error instanceof Error ? error.message : String(error),
},
});
}

/** How long the device-readiness probe may take, bounded by the startup budget it runs inside. */
const RUNNER_DEVICE_READINESS_BUDGET_MS = 10_000;

Expand Down
124 changes: 124 additions & 0 deletions packages/platform-apple/src/runner/runner-start-budget.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
import {
AppError,
createRequestCanceledError,
isRequestCanceledError,
} from '@agent-device/kernel/errors';
import { emitDiagnostic } from './host.ts';
import { resolveRunnerStartupSignal } from './runner-contract.ts';
import { createRunnerPhaseBudget, type RunnerPhaseBudget } from './runner-xctestrun.ts';
import { normalizeRunnerStartupTimeoutMs, type RunnerSession } from './runner-session-types.ts';
import type { AppleRunnerLifecycleOptions } from './runner-provider.ts';

type RunnerSessionOptions = AppleRunnerLifecycleOptions;

/**
* What a runner start may spend when its caller sets no `startupTimeoutMs`: the reuse probe,
* adoption, boot, device readiness, the xctestrun build and the launch, on one clock the start owns
* (#2894). Sized for a cold build: `prepare` defaults to 240 s for the same work plus its health
* check (`PREPARE_STARTUP_BUDGET_MS`), the iOS CI lane gives a cold `prepare` on a shared macOS
* runner 420 s (`AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS`), and a daemon queued on another process's
* build of the same artifact already gives up after 10 minutes
* (`RUNNER_XCTESTRUN_CACHE_LOCK_TIMEOUT_MS`). The two ceilings agree, so a start never waits on a
* peer's build longer than it would spend on its own.
*/
export const DEFAULT_RUNNER_START_BUDGET_MS = 10 * 60_000;

/**
* The budget one runner start spends, with the clock that enforces it. The abort fires when the
* deadline is spent, so every step that takes the signal (the xctestrun build and the launch kill
* their process tree through exec, the probes stop retrying) ends on the same clock the deadline
* reads. `close` retires the timer once the start has settled: a registered session is bounded by
* its {@link RunnerSession.launchDeadline} from then on, never by this abort.
*/
export type RunnerStartBudget = Readonly<{
phase: RunnerPhaseBudget;
timeoutMs: number;
explicit: boolean;
exhausted: AbortSignal;
close: () => void;
}>;

/**
* Opens the start's budget from `startupTimeoutMs`, or {@link DEFAULT_RUNNER_START_BUDGET_MS}
* when the caller sets none. The startup signal (a cancelled request, never a caller's deadline)
* and the budget's own expiry abort the same signal.
*/
export function openRunnerStartBudget(options: RunnerSessionOptions): RunnerStartBudget {
const explicitTimeoutMs = normalizeRunnerStartupTimeoutMs(options.startupTimeoutMs);
const timeoutMs = explicitTimeoutMs ?? DEFAULT_RUNNER_START_BUDGET_MS;
const exhausted = new AbortController();
const timer = setTimeout(() => {
exhausted.abort(runnerStartBudgetExhaustedError(timeoutMs, explicitTimeoutMs !== undefined));
}, timeoutMs);
timer.unref?.();
const startupSignal = resolveRunnerStartupSignal(options);

@cubic-dev-ai cubic-dev-ai Bot Sep 28, 2026 •

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: openRunnerStartBudget leaves the caller-signal listener installed after the start settles, because close only clears the timer. Repeated runner starts on one long-lived signal accumulate listeners and retain each filtered controller; return cleanup from the signal resolver and invoke it from close.

Prompt for AI agents
Check if this issue is valid — if so, understand the root cause and fix it. At packages/platform-apple/src/runner/runner-start-budget.ts, line 54:

<comment>`openRunnerStartBudget` leaves the caller-signal listener installed after the start settles, because `close` only clears the timer. Repeated runner starts on one long-lived signal accumulate listeners and retain each filtered controller; return cleanup from the signal resolver and invoke it from `close`.</comment>

<file context>
@@ -0,0 +1,124 @@
+    exhausted.abort(runnerStartBudgetExhaustedError(timeoutMs, explicitTimeoutMs !== undefined));
+  }, timeoutMs);
+  timer.unref?.();
+  const startupSignal = resolveRunnerStartupSignal(options);
+  const signal = startupSignal
+    ? AbortSignal.any([startupSignal, exhausted.signal])
</file context>
Fix with cubic

const signal = startupSignal
? AbortSignal.any([startupSignal, exhausted.signal])
: exhausted.signal;
return {
phase: createRunnerPhaseBudget(timeoutMs, signal),
timeoutMs,
explicit: explicitTimeoutMs !== undefined,
exhausted: exhausted.signal,
close: () => clearTimeout(timer),
};
}

/** Says the start's own budget ran out, whichever step it was in; the next request starts over. */
function runnerStartBudgetExhaustedError(timeoutMs: number, explicit: boolean): AppError {
return new AppError(
'COMMAND_FAILED',
`Apple runner start exceeded its ${explicit ? 'startup timeout' : 'default start budget'} of ${timeoutMs}ms`,
{
reason: 'runner_start_budget_exhausted',
timeoutMs,
retriable: true,
hint: explicit
? 'Raise the startup timeout, or run `prepare ios-runner` first so the runner is built before it is needed.'
: 'Run `prepare ios-runner --timeout <ms>` so the cold build has a budget of your choosing; the session runner.log names the step that stalled.',
},
);
}

/**
* The start runs detached under the session lock; the caller only waits for it as long as its own
* signal allows. A caller whose deadline lands during a cold xctestrun build leaves on time, the
* build keeps going under the lock, and the next request for the device queues behind it and joins
* the session it registers (#2894). Whatever the abort reason, the caller sees the same cancelled
* request it would have seen from any later step; a cancelled request's abort also reaches the
* start through its own startup signal, so nothing here decides whether the start survives. A
* start that fails after its caller left has nobody to report to, so its failure is logged here.
*/
export async function raceRunnerStartAgainstCaller(
start: Promise<RunnerSession>,
signal: AbortSignal | undefined,
): Promise<RunnerSession> {
if (!signal) return await start;
return await new Promise<RunnerSession>((resolve, reject) => {
const abort = () => {
reject(createRequestCanceledError(undefined, signal.reason));
start.catch(emitDetachedRunnerStartFailed);
};
if (signal.aborted) {
abort();
} else {
signal.addEventListener('abort', abort, { once: true });
}
start.then(resolve, reject).finally(() => signal.removeEventListener('abort', abort));
});
}

/** A cancelled request killed its start on purpose; any other failure of a start nobody awaits is news. */
function emitDetachedRunnerStartFailed(error: unknown): void {
if (isRequestCanceledError(error)) return;
const appErr = error instanceof AppError ? error : undefined;
emitDiagnostic({
level: 'warn',
phase: 'ios_runner_detached_start_failed',
data: {
code: appErr?.code,
reason: appErr?.details?.reason,
error: error instanceof Error ? error.message : String(error),
},
});
}
Loading