diff --git a/packages/platform-apple/src/core/__tests__/xcrun-shim-first-launch.test.ts b/packages/platform-apple/src/core/__tests__/xcrun-shim-first-launch.test.ts new file mode 100644 index 0000000000..da54cb76f5 --- /dev/null +++ b/packages/platform-apple/src/core/__tests__/xcrun-shim-first-launch.test.ts @@ -0,0 +1,263 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { afterEach, test, vi } from 'vitest'; +import type { ExecResult, ExecOptions } from '@agent-device/host-kit/command'; +import { COLD_TOOLCHAIN_PROBE_TIMEOUT_MS } from '../../runner/apple-runner-platform.ts'; +import { createLocalAppleToolProvider, withAppleToolProvider } from '../tool-provider.ts'; +import type { DeadlineClock } from '@agent-device/host-kit/retry'; +import { + probeXcrunShimFirstLaunchHooks, + XCRUN_SHIM_TOOL_NAMES, + type XcrunShimProbeOptions, + type XctestDeviceSetCleanupArming, +} from '../xcrun-shim-first-launch.ts'; +import { mkdtempForTest } from '../../__tests__/tmp-dir.ts'; +import { + fakeFrameworkInfoPlistPath, + hookedShimText, + withFakeXcrunHost, + writeFakeXcrunShims, +} from './xcrun-shim-fixtures.ts'; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +async function tempRoot(): Promise { + return await mkdtempForTest('xcrun-shim-first-launch-'); +} + +async function probeShims(options?: XcrunShimProbeOptions): Promise { + const probe = await probeXcrunShimFirstLaunchHooks(options); + if (probe.outcome !== 'read') assert.fail(`the probe read no shim: ${probe.outcome}`); + return probe.xcrunShims; +} + +function phaseClock(remainingMs: number): DeadlineClock { + return { remainingMs: () => remainingMs, elapsedMs: () => 0, isExpired: () => remainingMs <= 0 }; +} + +test('only the tools Xcode ships as first-launch shims are probed', async () => { + const host = writeFakeXcrunShims(await tempRoot(), {}); + + const shims = await withFakeXcrunHost(host, () => probeShims()); + + assert.deepEqual(XCRUN_SHIM_TOOL_NAMES, ['simctl', 'devicectl']); + assert.deepEqual(host.finds, XCRUN_SHIM_TOOL_NAMES); + for (const shim of shims) { + assert.deepEqual( + { hook: shim.hook, shimPath: shim.shimPath }, + { hook: 'armed', shimPath: null }, + `${shim.tool} was not found, so it cannot be called safe`, + ); + assert.equal(shim.hook === 'armed' && shim.armedBy, 'shim_not_located'); + } +}); + +test('a hooked shim found by xcrun --find is read against the plist its own text names', async () => { + const root = await tempRoot(); + const otherPlist = path.join(root, 'Other.framework', 'Info.plist'); + const machO = Buffer.concat([ + Buffer.from([0xcf, 0xfa, 0xed, 0xfe]), + Buffer.from(' -runFirstLaunch'), + ]); + const host = writeFakeXcrunShims(root, { + simctl: { text: hookedShimText('1051.17.7', otherPlist) }, + devicectl: { text: machO }, + }); + host.installedVersions.set(otherPlist, '1155.4'); + + const [simctl, devicectl] = await withFakeXcrunHost(host, () => probeShims()); + + assert.deepEqual(host.plistReads, [otherPlist]); + assert.deepEqual(simctl, { + tool: 'simctl', + shimPath: host.xcrunShimPaths.simctl, + hook: 'armed', + armedBy: 'version_mismatch', + expectedVersion: '1051.17.7', + frameworkInfoPlistPath: otherPlist, + installedVersion: '1155.4', + }); + assert.deepEqual(devicectl, { + tool: 'devicectl', + shimPath: host.xcrunShimPaths.devicectl, + hook: 'none', + }); +}); + +test('equal versions disarm a hooked shim', async () => { + const host = writeFakeXcrunShims(await tempRoot(), { + simctl: { expectedVersion: '1155.4', installedVersion: '1155.4' }, + devicectl: { expectedVersion: '629.3', installedVersion: '629.3' }, + }); + + const shims = await withFakeXcrunHost(host, () => probeShims()); + + assert.deepEqual( + shims.map((shim) => [shim.tool, shim.hook]), + [ + ['simctl', 'disarmed'], + ['devicectl', 'disarmed'], + ], + ); +}); + +test('a shim xcrun found but that cannot be read counts as armed', async () => { + const root = await tempRoot(); + const host = writeFakeXcrunShims(root, { + devicectl: { expectedVersion: '629.3', installedVersion: '629.3' }, + }); + const missing = path.join(root, 'xcrun-shims', 'simctl-removed'); + host.xcrunShimPaths.simctl = missing; + + const [simctl] = await withFakeXcrunHost(host, () => probeShims()); + + assert.deepEqual(simctl, { + tool: 'simctl', + shimPath: missing, + hook: 'armed', + armedBy: 'shim_unreadable', + expectedVersion: null, + frameworkInfoPlistPath: null, + installedVersion: null, + }); +}); + +// Each shape breaks one value and keeps the rest readable and equal, so the verdict is that value's. +const UNREADABLE_VERSION_SHAPES: Record string> = { + 'no EXPECTED_VERSION': (plistPath) => + hookedShimText('1', plistPath).replace('EXPECTED_VERSION="1"', 'EXPECTED_VERSION='), + 'no Info.plist path on the CURRENT_VERSION line': (plistPath) => + hookedShimText('1', plistPath).replace(`"${plistPath}"`, '"$PLIST"'), + 'an unreadable framework Info.plist': (plistPath) => hookedShimText('1', `${plistPath}.missing`), +}; + +for (const [shape, text] of Object.entries(UNREADABLE_VERSION_SHAPES)) { + test(`a hooked shim with ${shape} fails closed`, async () => { + const root = await tempRoot(); + const plistPath = fakeFrameworkInfoPlistPath(root, 'devicectl'); + const host = writeFakeXcrunShims(root, { devicectl: { text: text(plistPath) } }); + host.installedVersions.set(plistPath, '1'); + + const [, devicectl] = await withFakeXcrunHost(host, () => probeShims()); + + assert.equal(devicectl?.hook === 'armed' && devicectl.armedBy, 'version_unreadable'); + }); +} + +function hangingXcrun(onStart: (options: ExecOptions | undefined) => void) { + return createLocalAppleToolProvider({ + runCommand: async (_cmd, _args, options): Promise => { + onStart(options); + return await new Promise(() => {}); + }, + }); +} + +test('the probe spends the cold-toolchain budget on each xcrun --find and reads a timeout as armed', async () => { + const budget = new AbortController(); + const timeout = vi.spyOn(AbortSignal, 'timeout').mockReturnValue(budget.signal); + const findTimeoutsMs: Array = []; + const provider = hangingXcrun((options) => { + findTimeoutsMs.push(options?.timeoutMs); + if (findTimeoutsMs.length === XCRUN_SHIM_TOOL_NAMES.length) budget.abort(); + }); + + const shims = await withAppleToolProvider(provider, () => probeShims()); + + assert.deepEqual(timeout.mock.calls, [[COLD_TOOLCHAIN_PROBE_TIMEOUT_MS]]); + assert.deepEqual( + findTimeoutsMs, + XCRUN_SHIM_TOOL_NAMES.map(() => COLD_TOOLCHAIN_PROBE_TIMEOUT_MS), + ); + assert.deepEqual( + shims.map((shim) => [shim.tool, shim.hook, shim.hook === 'armed' && shim.armedBy]), + XCRUN_SHIM_TOOL_NAMES.map((tool) => [tool, 'armed', 'probe_out_of_budget']), + ); +}); + +test('a phase with less left than the cold budget caps the probe and owns its stop', async () => { + const budget = new AbortController(); + const timeout = vi.spyOn(AbortSignal, 'timeout').mockReturnValue(budget.signal); + let started = 0; + const provider = hangingXcrun(() => { + started += 1; + if (started === XCRUN_SHIM_TOOL_NAMES.length) budget.abort(); + }); + + const probe = await withAppleToolProvider(provider, () => + probeXcrunShimFirstLaunchHooks({ deadline: phaseClock(5_000) }), + ); + + assert.deepEqual(timeout.mock.calls, [[5_000]]); + assert.equal(started, XCRUN_SHIM_TOOL_NAMES.length); + assert.deepEqual(probe, { outcome: 'phase_budget_exhausted' }); +}); + +test('a phase with more left than the cold budget reads a cold-budget stop as armed', async () => { + const budget = new AbortController(); + const timeout = vi.spyOn(AbortSignal, 'timeout').mockReturnValue(budget.signal); + let started = 0; + const provider = hangingXcrun(() => { + started += 1; + if (started === XCRUN_SHIM_TOOL_NAMES.length) budget.abort(); + }); + + const shims = await withAppleToolProvider(provider, () => + probeShims({ deadline: phaseClock(120_000) }), + ); + + assert.deepEqual(timeout.mock.calls, [[COLD_TOOLCHAIN_PROBE_TIMEOUT_MS]]); + assert.deepEqual( + shims.map((shim) => shim.hook === 'armed' && shim.armedBy), + XCRUN_SHIM_TOOL_NAMES.map(() => 'probe_out_of_budget'), + ); +}); + +test('a spent phase spawns no xcrun and reads as the phase running out', async () => { + const host = writeFakeXcrunShims(await tempRoot(), { + simctl: { expectedVersion: '1155.4', installedVersion: '1155.4' }, + devicectl: { expectedVersion: '629.3', installedVersion: '629.3' }, + }); + const timeout = vi.spyOn(AbortSignal, 'timeout'); + + const probe = await withFakeXcrunHost(host, () => + probeXcrunShimFirstLaunchHooks({ deadline: phaseClock(0) }), + ); + + assert.deepEqual(probe, { outcome: 'phase_budget_exhausted' }); + assert.deepEqual(host.finds, []); + assert.equal(timeout.mock.calls.length, 0); +}); + +test('a request canceled mid-probe reads as canceled, never as an armed shim', async () => { + const request = new AbortController(); + let started = 0; + const provider = hangingXcrun((options) => { + started += 1; + assert.equal(options?.signal?.aborted, false); + if (started === XCRUN_SHIM_TOOL_NAMES.length) request.abort(); + }); + + const probe = await withAppleToolProvider(provider, () => + probeXcrunShimFirstLaunchHooks({ signal: request.signal }), + ); + + assert.equal(started, XCRUN_SHIM_TOOL_NAMES.length); + assert.deepEqual(probe, { outcome: 'request_canceled' }); +}); + +test('an already-canceled request spawns no xcrun at all', async () => { + const host = writeFakeXcrunShims(await tempRoot(), { + simctl: { expectedVersion: '1155.4', installedVersion: '1155.4' }, + devicectl: { expectedVersion: '629.3', installedVersion: '629.3' }, + }); + + const probe = await withFakeXcrunHost(host, () => + probeXcrunShimFirstLaunchHooks({ signal: AbortSignal.abort() }), + ); + + assert.deepEqual(host.finds, []); + assert.deepEqual(probe, { outcome: 'request_canceled' }); +}); diff --git a/packages/platform-apple/src/core/__tests__/xcrun-shim-fixtures.ts b/packages/platform-apple/src/core/__tests__/xcrun-shim-fixtures.ts new file mode 100644 index 0000000000..4f97301503 --- /dev/null +++ b/packages/platform-apple/src/core/__tests__/xcrun-shim-fixtures.ts @@ -0,0 +1,145 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import type { ExecResult } from '@agent-device/host-kit/command'; +import { createLocalAppleToolProvider, withAppleToolProvider } from '../tool-provider.ts'; +import { XCRUN_SHIM_TOOL_NAMES, type XcrunShimToolName } from '../xcrun-shim-first-launch.ts'; + +/** + * The fake Xcode `xcrun` shims every first-launch probe test reads (#2935). Hooked shims are the + * captured Xcode 26.2 `simctl` shim with its version and Info.plist path swapped, and installed + * versions are answered by a fake plist reader, so no test runs `xcrun`, `plutil`, or reads the + * host's Xcode. + */ +export const XCODE_26_2_SIMCTL_SHIM = { + command: 'cat "$(xcrun --find simctl)"', + xcodeVersion: 'Xcode 26.2 (17C52)', + expectedVersion: '1051.17.7', + infoPlistPath: + '/Library/Developer/PrivateFrameworks/CoreSimulator.framework/Versions/A/Resources/Info.plist', + text: [ + '#!/bin/bash', + 'DEVELOPER_USR_BIN_DIR=${0%/*}', + 'DEVELOPER_USR_BIN_DIR=${DEVELOPER_USR_BIN_DIR%/local/bin}', + 'DEVELOPER_USR_BIN_DIR=${DEVELOPER_USR_BIN_DIR%/bin}', + 'DEVELOPER_USR_BIN_DIR=${DEVELOPER_USR_BIN_DIR}/bin', + 'DEVELOPER_USR_DIR=${DEVELOPER_USR_BIN_DIR%/*}', + 'export DEVELOPER_DIR=${DEVELOPER_USR_DIR%/*}', + '', + 'EXPECTED_VERSION="1051.17.7"', + 'CURRENT_VERSION="$(/usr/libexec/PlistBuddy -c "Print :CFBundleVersion" "/Library/Developer/PrivateFrameworks/CoreSimulator.framework/Versions/A/Resources/Info.plist" 2>&1)"', + '', + 'if [[ "${EXPECTED_VERSION}" != "${CURRENT_VERSION}" ]]; then', + ' "${DEVELOPER_DIR}/usr/bin/xcodebuild" -runFirstLaunch >&2', + 'fi', + 'exec "/Library/Developer/PrivateFrameworks/CoreSimulator.framework/Versions/A/Resources/bin/simctl" "${@}"', + '', + ].join('\n'), +} as const; + +export function hookedShimText(expectedVersion: string, infoPlistPath: string): string { + return XCODE_26_2_SIMCTL_SHIM.text + .replace( + `EXPECTED_VERSION="${XCODE_26_2_SIMCTL_SHIM.expectedVersion}"`, + `EXPECTED_VERSION="${expectedVersion}"`, + ) + .replace(`"${XCODE_26_2_SIMCTL_SHIM.infoPlistPath}"`, `"${infoPlistPath}"`); +} + +export type FakeXcrunShim = + | { expectedVersion: string; installedVersion: string } + | { hook: 'none' } + | { text: string | Buffer }; + +export type FakeXcrunHost = { + xcrunShimPaths: Partial>; + /** `CFBundleVersion` by Info.plist path; a path left out reads as an unreadable plist. */ + installedVersions: Map; + /** Every tool `xcrun --find` was asked for, in call order. */ + finds: string[]; + /** Every Info.plist path the probe asked for, in call order. */ + plistReads: string[]; + /** Runs as each Info.plist read starts, before it is answered. */ + onPlistRead?: () => void; + /** When set, `xcrun --find` answers only once its signal aborts, as a cold toolchain stalls. */ + findStalls?: boolean; +}; + +const HOOKLESS_SHIM_TEXT = '#!/bin/bash\nexec "${DEVELOPER_DIR}/usr/bin/tool" "${@}"\n'; + +const FAKE_FRAMEWORK_NAMES: Record = { + simctl: 'CoreSimulator', + devicectl: 'CoreDevice', +}; + +/** Where a fake tool's framework Info.plist lives under `root`. */ +export function fakeFrameworkInfoPlistPath(root: string, tool: XcrunShimToolName): string { + return path.join(root, `${FAKE_FRAMEWORK_NAMES[tool]}.framework`, 'Info.plist'); +} + +/** Writes one shim file per listed tool; a tool left out is absent from `xcrunShimPaths`. */ +export function writeFakeXcrunShims( + root: string, + shims: Readonly>>, +): FakeXcrunHost { + const shimDir = path.join(root, 'xcrun-shims'); + fs.mkdirSync(shimDir, { recursive: true }); + const host: FakeXcrunHost = { + xcrunShimPaths: {}, + installedVersions: new Map(), + finds: [], + plistReads: [], + }; + for (const tool of XCRUN_SHIM_TOOL_NAMES) { + const shim = shims[tool]; + if (!shim) continue; + const plistPath = fakeFrameworkInfoPlistPath(root, tool); + let text: string | Buffer = HOOKLESS_SHIM_TEXT; + if ('expectedVersion' in shim) { + text = hookedShimText(shim.expectedVersion, plistPath); + host.installedVersions.set(plistPath, shim.installedVersion); + } else if ('text' in shim) { + text = shim.text; + } + const shimPath = path.join(shimDir, tool); + fs.writeFileSync(shimPath, text); + host.xcrunShimPaths[tool] = shimPath; + } + return host; +} + +/** + * Runs `task` with `xcrun --find` answered from `host.xcrunShimPaths` and every Info.plist read + * answered from the fake shims' installed versions, both recorded on `host`. + */ +export async function withFakeXcrunHost( + host: FakeXcrunHost, + task: () => Promise, +): Promise { + const provider = createLocalAppleToolProvider({ + runCommand: async (cmd, args, options): Promise => { + const tool = cmd === 'xcrun' && args[0] === '--find' ? args[1] : undefined; + if (tool !== undefined) host.finds.push(tool); + if (tool !== undefined && host.findStalls) await abortOf(options?.signal); + const found = tool === undefined ? undefined : host.xcrunShimPaths[tool as XcrunShimToolName]; + return found + ? { exitCode: 0, stdout: `${found}\n`, stderr: '' } + : { exitCode: 1, stdout: '', stderr: `xcrun: error: unable to find utility "${tool}"` }; + }, + plist: { + readJson: async (plistPath) => { + host.plistReads.push(plistPath); + host.onPlistRead?.(); + const version = host.installedVersions.get(plistPath); + return version === undefined ? null : { CFBundleVersion: version }; + }, + }, + }); + return await withAppleToolProvider(provider, task); +} + +async function abortOf(signal: AbortSignal | undefined): Promise { + if (!signal || signal.aborted) return; + await new Promise((resolve) => { + signal.addEventListener('abort', () => resolve(), { once: true }); + }); +} diff --git a/packages/platform-apple/src/core/runner-host.ts b/packages/platform-apple/src/core/runner-host.ts index c6bdb0828e..1c11a1f1dc 100644 --- a/packages/platform-apple/src/core/runner-host.ts +++ b/packages/platform-apple/src/core/runner-host.ts @@ -95,6 +95,8 @@ export const appleRunnerHost: AppleRunnerHost = { runAppleToolCommand, runXcrun, readApplePlistJson, + probeXcrunShimFirstLaunchHooks: async (options) => + await (await import('./xcrun-shim-first-launch.ts')).probeXcrunShimFirstLaunchHooks(options), buildSimctlArgsForDevice, resolveIosPhysicalDeviceControl, visitXmlPlistEntries, diff --git a/packages/platform-apple/src/core/tool-provider.ts b/packages/platform-apple/src/core/tool-provider.ts index 8e0667e32b..025dd198ca 100644 --- a/packages/platform-apple/src/core/tool-provider.ts +++ b/packages/platform-apple/src/core/tool-provider.ts @@ -137,8 +137,21 @@ export async function runAppleToolCommand( return await resolveAppleToolProvider().runCommand(cmd, args, options); } +/** + * Every Xcode tool agent-device runs through `xcrun`. `firstLaunchShim` marks a tool Xcode ships as a + * script shim that can run `xcodebuild -runFirstLaunch` before the tool; the others are Mach-O binaries. + */ +export const XCRUN_TOOLS = { + simctl: { firstLaunchShim: true }, + devicectl: { firstLaunchShim: true }, + xcdevice: { firstLaunchShim: false }, + xctrace: { firstLaunchShim: false }, +} as const satisfies Record; + +export type XcrunToolName = keyof typeof XCRUN_TOOLS; + /** An xcrun argv for a tool other than simctl; a simctl argv is a ScopedSimctlCommand. */ -type XcrunToolArgs = readonly ['devicectl' | 'xcdevice' | 'xctrace', ...string[]]; +type XcrunToolArgs = readonly [Exclude, ...string[]]; export async function runXcrun( args: ScopedSimctlCommand | XcrunToolArgs, diff --git a/packages/platform-apple/src/core/xcrun-shim-first-launch.ts b/packages/platform-apple/src/core/xcrun-shim-first-launch.ts new file mode 100644 index 0000000000..219c27080b --- /dev/null +++ b/packages/platform-apple/src/core/xcrun-shim-first-launch.ts @@ -0,0 +1,228 @@ +import { readHostTextFile } from '@agent-device/host-kit/host-file'; +import type { DeadlineClock } from '@agent-device/host-kit/retry'; +import { COLD_TOOLCHAIN_PROBE_TIMEOUT_MS } from '../runner/apple-runner-platform.ts'; +import { + readApplePlistJson, + runAppleToolCommand, + XCRUN_TOOLS, + type XcrunToolName, +} from './tool-provider.ts'; + +const FIRST_LAUNCH_FLAG = '-runFirstLaunch'; + +export type XcrunShimToolName = { + [Tool in XcrunToolName]: (typeof XCRUN_TOOLS)[Tool]['firstLaunchShim'] extends true + ? Tool + : never; +}[XcrunToolName]; + +/** The {@link XCRUN_TOOLS} entries Xcode ships as a shim that can carry a first-launch hook. */ +export const XCRUN_SHIM_TOOL_NAMES = (Object.keys(XCRUN_TOOLS) as XcrunToolName[]).filter( + (tool): tool is XcrunShimToolName => XCRUN_TOOLS[tool].firstLaunchShim, +); + +/** + * Whether one Xcode `xcrun` shim runs `xcodebuild -runFirstLaunch` before the tool it wraps. That + * cleanup deletes every device in `~/Library/Developer/XCTestDevices`. The shim compares its own + * `EXPECTED_VERSION` with the installed framework's `CFBundleVersion`; a shim whose versions could + * not be read is `armed`. + */ +export type XcrunShimFirstLaunchHook = + | { tool: XcrunShimToolName; shimPath: string; hook: 'none' } + | { + tool: XcrunShimToolName; + shimPath: string; + hook: 'disarmed'; + expectedVersion: string; + frameworkInfoPlistPath: string; + installedVersion: string; + } + | ArmedXcrunShimFirstLaunchHook; + +/** + * Why a shim reads as armed; every value but `version_mismatch` is a probe that could not decide. + * `probe_out_of_budget` is a stop by the cold-toolchain budget, never by the owning phase's clock. + */ +export type XcrunShimArmedBy = + | 'version_mismatch' + | 'version_unreadable' + | 'shim_unreadable' + | 'shim_not_located' + | 'probe_out_of_budget'; + +type XcrunShimEvidence = { + tool: XcrunShimToolName; + /** Null when `xcrun --find` failed or the probe stopped first. */ + shimPath: string | null; + expectedVersion: string | null; + frameworkInfoPlistPath: string | null; + installedVersion: string | null; +}; + +export type ArmedXcrunShimFirstLaunchHook = XcrunShimEvidence & { + hook: 'armed'; + armedBy: XcrunShimArmedBy; +}; + +/** One entry per {@link XCRUN_SHIM_TOOL_NAMES} tool. */ +export type XctestDeviceSetCleanupArming = readonly XcrunShimFirstLaunchHook[]; + +/** + * What one probe came back with. A probe its request canceled, or one the owning phase's clock + * stopped, reads no shim: the stop belongs to the request or the phase, never to an arming. + */ +export type XcrunShimProbe = + | { outcome: 'request_canceled' } + | { outcome: 'phase_budget_exhausted' } + | { outcome: 'read'; xcrunShims: XctestDeviceSetCleanupArming }; + +export type XcrunShimProbeOptions = { + /** The owning request's cancellation. */ + signal?: AbortSignal; + /** The owning phase's clock; the probe spends no more than it has left. */ + deadline?: DeadlineClock; +}; + +/** + * Reads every shim's first-launch hook within one shared budget: the cold-toolchain budget, or what + * the owning phase has left when that is no more. Only a stop by the cold-toolchain budget reads a + * shim as armed; a stop by the phase's clock is the phase running out. + */ +export async function probeXcrunShimFirstLaunchHooks( + options: XcrunShimProbeOptions = {}, +): Promise { + if (options.signal?.aborted) return { outcome: 'request_canceled' }; + const phaseRemainingMs = options.deadline + ? Math.floor(options.deadline.remainingMs()) + : Number.POSITIVE_INFINITY; + if (phaseRemainingMs <= 0) return { outcome: 'phase_budget_exhausted' }; + const phaseOwnsBudget = phaseRemainingMs <= COLD_TOOLCHAIN_PROBE_TIMEOUT_MS; + const budget = AbortSignal.timeout(Math.min(COLD_TOOLCHAIN_PROBE_TIMEOUT_MS, phaseRemainingMs)); + const signal = options.signal ? AbortSignal.any([budget, options.signal]) : budget; + const xcrunShims = await Promise.all( + XCRUN_SHIM_TOOL_NAMES.map(async (tool) => await probeWithinBudget(tool, signal)), + ); + if (options.signal?.aborted) return { outcome: 'request_canceled' }; + if (phaseOwnsBudget && xcrunShims.some(isStoppedByBudget)) { + return { outcome: 'phase_budget_exhausted' }; + } + return { outcome: 'read', xcrunShims }; +} + +function isStoppedByBudget(shim: XcrunShimFirstLaunchHook): boolean { + return shim.hook === 'armed' && shim.armedBy === 'probe_out_of_budget'; +} + +async function probeWithinBudget( + tool: XcrunShimToolName, + signal: AbortSignal, +): Promise { + const evidence: XcrunShimEvidence = { + tool, + shimPath: null, + expectedVersion: null, + frameworkInfoPlistPath: null, + installedVersion: null, + }; + if (signal.aborted) return armed(evidence, 'probe_out_of_budget'); + let onAbort = (): void => {}; + const stopped = new Promise((resolve) => { + onAbort = () => resolve(armed(evidence, 'probe_out_of_budget')); + }); + signal.addEventListener('abort', onAbort, { once: true }); + try { + return await Promise.race([readShimHook(evidence, signal), stopped]); + } finally { + signal.removeEventListener('abort', onAbort); + } +} + +function armed( + evidence: XcrunShimEvidence, + armedBy: XcrunShimArmedBy, +): ArmedXcrunShimFirstLaunchHook { + return { ...evidence, hook: 'armed', armedBy }; +} + +async function readShimHook( + evidence: XcrunShimEvidence, + signal: AbortSignal, +): Promise { + const { tool } = evidence; + const shimPath = await locateShim(tool, signal); + if (shimPath === null) return armed(evidence, 'shim_not_located'); + evidence.shimPath = shimPath; + const text = await readShimText(shimPath, signal); + if (text === null) return armed(evidence, 'shim_unreadable'); + if (!text.startsWith('#!') || !text.includes(FIRST_LAUNCH_FLAG)) { + return { tool, shimPath, hook: 'none' }; + } + await readShimVersions(evidence, text, signal); + return settleShimHook(evidence, shimPath); +} + +async function readShimVersions( + evidence: XcrunShimEvidence, + text: string, + signal: AbortSignal, +): Promise { + evidence.expectedVersion = /^\s*EXPECTED_VERSION="([^"]+)"/m.exec(text)?.[1] ?? null; + evidence.frameworkInfoPlistPath = parseFrameworkInfoPlistPath(text); + if (evidence.frameworkInfoPlistPath !== null) { + evidence.installedVersion = await readBundleVersion(evidence.frameworkInfoPlistPath, signal); + } +} + +function settleShimHook(evidence: XcrunShimEvidence, shimPath: string): XcrunShimFirstLaunchHook { + const { tool, expectedVersion, frameworkInfoPlistPath, installedVersion } = evidence; + if (expectedVersion === null || frameworkInfoPlistPath === null || installedVersion === null) { + return armed(evidence, 'version_unreadable'); + } + if (expectedVersion !== installedVersion) return armed(evidence, 'version_mismatch'); + return { + tool, + shimPath, + hook: 'disarmed', + expectedVersion, + frameworkInfoPlistPath, + installedVersion, + }; +} + +async function locateShim(tool: XcrunShimToolName, signal: AbortSignal): Promise { + try { + const result = await runAppleToolCommand('xcrun', ['--find', tool], { + allowFailure: true, + timeoutMs: COLD_TOOLCHAIN_PROBE_TIMEOUT_MS, + signal, + }); + const found = result.stdout.trim(); + return result.exitCode === 0 && found ? found : null; + } catch { + return null; + } +} + +async function readShimText(shimPath: string, signal: AbortSignal): Promise { + try { + return await readHostTextFile(shimPath, { signal }); + } catch { + return null; + } +} + +function parseFrameworkInfoPlistPath(text: string): string | null { + const currentVersionLine = /^\s*CURRENT_VERSION=.*$/m.exec(text)?.[0]; + if (currentVersionLine === undefined) return null; + return /"([^"]*Info\.plist)"/.exec(currentVersionLine)?.[1] ?? null; +} + +async function readBundleVersion(plistPath: string, signal: AbortSignal): Promise { + try { + const plist = await readApplePlistJson(plistPath, signal); + const version = plist?.CFBundleVersion; + return typeof version === 'string' && version.trim() ? version.trim() : null; + } catch { + return null; + } +} diff --git a/packages/platform-apple/src/runner/__tests__/runner-artifact.test.ts b/packages/platform-apple/src/runner/__tests__/runner-artifact.test.ts new file mode 100644 index 0000000000..27f08f10f2 --- /dev/null +++ b/packages/platform-apple/src/runner/__tests__/runner-artifact.test.ts @@ -0,0 +1,144 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { afterEach, beforeEach, test, vi } from 'vitest'; +import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; +import { resetAllProcessMemosForTests } from '@agent-device/kernel/ttl-memo'; +import type { ExecResult } from '@agent-device/host-kit/command'; +import { createLocalAppleToolProvider, withAppleToolProvider } from '../../core/tool-provider.ts'; +import { appleRunnerTestHost } from '../test-host.ts'; +import { COLD_TOOLCHAIN_PROBE_TIMEOUT_MS } from '../apple-runner-platform.ts'; +import { createRunnerPhaseBudget, ensureXctestrunArtifact } from '../runner-xctestrun.ts'; +import { resolveXcodebuildSimulatorDeviceSetPath } from '../runner-device-set.ts'; +import { appleToolchainProbeResult } from './apple-toolchain-fixtures.ts'; +import { IOS_SIMULATOR } from './device-fixtures.ts'; +import { mkdtempForTestSync } from './tmp-dir.ts'; +import { + withFakeXcrunHost, + writeFakeXcrunShims, +} from '../../core/__tests__/xcrun-shim-fixtures.ts'; + +const runCmdSync = vi.fn(); +const runCmdStreaming = vi.fn(); +const originalHome = process.env.HOME; +let root: string; + +beforeEach(() => { + resetAllProcessMemosForTests(); + root = mkdtempForTestSync('agent-device-runner-artifact-'); + const projectRoot = path.join(root, 'project'); + fs.mkdirSync( + path.join(projectRoot, 'apple', 'runner', 'AgentDeviceRunner', 'AgentDeviceRunner.xcodeproj'), + { recursive: true }, + ); + process.env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH = path.join(root, 'derived'); + // The build path redirects the host's own `~/Library/Developer/XCTestDevices`. + process.env.HOME = path.join(root, 'home'); + runCmdSync.mockReset().mockImplementation(appleToolchainProbeResult); + runCmdStreaming + .mockReset() + .mockImplementation(async (): Promise => ({ exitCode: 0, stdout: '', stderr: '' })); + appleRunnerTestHost.update({ + runCmdSync, + runCmdStreaming, + findProjectRoot: () => projectRoot, + readVersion: () => '0.0.0-test', + }); +}); + +afterEach(() => { + vi.restoreAllMocks(); + delete process.env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH; + process.env.HOME = originalHome; +}); + +test('a scoped-set simulator on a cache miss is refused before build-for-testing', async () => { + const requestedSetPath = path.join(root, 'user-set'); + fs.mkdirSync(requestedSetPath, { recursive: true }); + const xctestDeviceSetPath = resolveXcodebuildSimulatorDeviceSetPath(); + assert.equal(xctestDeviceSetPath.startsWith(root), true); + fs.mkdirSync(xctestDeviceSetPath, { recursive: true }); + const host = writeFakeXcrunShims(root, { + simctl: { expectedVersion: '1051.17.7', installedVersion: '1155.4' }, + devicectl: { expectedVersion: '506.6', installedVersion: '629.3' }, + }); + + await assert.rejects( + withFakeXcrunHost(host, () => + ensureXctestrunArtifact( + { ...IOS_SIMULATOR, simulatorSetPath: requestedSetPath }, + { budget: createRunnerPhaseBudget(120_000, undefined) }, + ), + ), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'xctest_device_set_cleanup_armed', + ); + + assert.equal(runCmdStreaming.mock.calls.length, 0, 'no xcodebuild build-for-testing ran'); + assert.equal(fs.lstatSync(xctestDeviceSetPath).isSymbolicLink(), false); +}); + +test('a build phase with less left than the cold budget caps the shim probe at what it has left', async () => { + const requestedSetPath = path.join(root, 'user-set'); + fs.mkdirSync(requestedSetPath, { recursive: true }); + fs.mkdirSync(resolveXcodebuildSimulatorDeviceSetPath(), { recursive: true }); + const host = writeFakeXcrunShims(root, { + simctl: { expectedVersion: '1051.17.7', installedVersion: '1155.4' }, + devicectl: { hook: 'none' }, + }); + const phaseMs = 5_000; + const timeout = vi.spyOn(AbortSignal, 'timeout'); + + await assert.rejects( + withFakeXcrunHost(host, () => + ensureXctestrunArtifact( + { ...IOS_SIMULATOR, simulatorSetPath: requestedSetPath }, + { budget: createRunnerPhaseBudget(phaseMs, undefined) }, + ), + ), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'xctest_device_set_cleanup_armed', + ); + + assert.equal(phaseMs < COLD_TOOLCHAIN_PROBE_TIMEOUT_MS, true); + assert.equal(timeout.mock.calls.length, 1); + const probeBudgetMs = timeout.mock.calls[0]?.[0] ?? Number.NaN; + assert.equal(probeBudgetMs > 0 && probeBudgetMs <= phaseMs, true, `${probeBudgetMs} ms`); +}); + +test('a build canceled while the shims are probed releases the device set without building', async () => { + const requestedSetPath = path.join(root, 'user-set'); + fs.mkdirSync(requestedSetPath, { recursive: true }); + fs.mkdirSync(resolveXcodebuildSimulatorDeviceSetPath(), { recursive: true }); + const request = new AbortController(); + let finds = 0; + const xcrun = createLocalAppleToolProvider({ + runCommand: async (_cmd, _args, options): Promise => { + finds += 1; + request.abort(); + const stopped: ExecResult = { exitCode: 1, stdout: '', stderr: '' }; + const signal = options?.signal; + if (!signal || signal.aborted) return stopped; + return await new Promise((resolve) => { + signal.addEventListener('abort', () => resolve(stopped), { once: true }); + }); + }, + }); + + await assert.rejects( + withAppleToolProvider(xcrun, () => + ensureXctestrunArtifact( + { ...IOS_SIMULATOR, simulatorSetPath: requestedSetPath }, + { budget: createRunnerPhaseBudget(120_000, request.signal) }, + ), + ), + (error: unknown) => isRequestCanceledError(error), + ); + + assert.notEqual(finds, 0, 'the probe had started'); + assert.equal(runCmdStreaming.mock.calls.length, 0, 'no xcodebuild build-for-testing ran'); + assert.equal( + fs.existsSync(path.join(root, 'home', '.agent-device', 'xctest-device-set.lock')), + false, + ); +}); diff --git a/packages/platform-apple/src/runner/__tests__/runner-device-set-cleanup-arming.test.ts b/packages/platform-apple/src/runner/__tests__/runner-device-set-cleanup-arming.test.ts new file mode 100644 index 0000000000..92c0afaf87 --- /dev/null +++ b/packages/platform-apple/src/runner/__tests__/runner-device-set-cleanup-arming.test.ts @@ -0,0 +1,372 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { test, vi } from 'vitest'; +import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { DeadlineClock } from '@agent-device/host-kit/retry'; +import { + XCRUN_SHIM_TOOL_NAMES, + type ArmedXcrunShimFirstLaunchHook, + type XcrunShimProbeOptions, + type XcrunShimToolName, +} from '../../core/xcrun-shim-first-launch.ts'; +import { appleRunnerTestHost } from '../test-host.ts'; +import { acquireXcodebuildSimulatorSetRedirect } from '../runner-device-set.ts'; +import { RUNNER_ERROR_RULES } from '../runner-error-classification.ts'; +import { mkdtempForTestSync } from './tmp-dir.ts'; +import { RUNNER_STARTUP_FAILURE_FIXTURES } from './runner-startup-failure-fixtures.ts'; +import { + fakeFrameworkInfoPlistPath, + withFakeXcrunHost, + writeFakeXcrunShims, + type FakeXcrunHost, +} from '../../core/__tests__/xcrun-shim-fixtures.ts'; + +// #2935: pointing `XCTestDevices` at a user's simulator set while an Xcode shim would run +// `xcodebuild -runFirstLaunch` lets that cleanup delete every device in the user's set. These cases +// drive the real probe over fake shims and a fake plist reader. + +const REASON = 'xctest_device_set_cleanup_armed'; + +const ROW_HINT = RUNNER_ERROR_RULES.find((rule) => rule.buildFailure?.reason === REASON) + ?.buildFailure?.hint; + +type Layout = { + root: string; + requestedSetPath: string; + xctestDeviceSetPath: string; + backupPath: string; + lockDirPath: string; +}; + +function makeLayout(): Layout { + const root = mkdtempForTestSync('device-set-cleanup-arming-'); + const layout = { + root, + requestedSetPath: path.join(root, 'requested'), + xctestDeviceSetPath: path.join(root, 'Library', 'Developer', 'XCTestDevices'), + backupPath: path.join(root, 'Library', 'Developer', 'XCTestDevices.set-aside'), + lockDirPath: path.join(root, '.agent-device', 'xctest-device-set.lock'), + }; + fs.mkdirSync(layout.requestedSetPath, { recursive: true }); + fs.mkdirSync(layout.xctestDeviceSetPath, { recursive: true }); + fs.writeFileSync(path.join(layout.xctestDeviceSetPath, 'host-device.txt'), 'the host owns this'); + return layout; +} + +function scopedSimulator(setPath: string): DeviceInfo { + return { + platform: 'apple', + id: 'sim-scoped', + name: 'iPhone Simulator', + kind: 'simulator', + appleOs: 'ios', + booted: true, + simulatorSetPath: setPath, + }; +} + +async function acquire(layout: Layout, host: FakeXcrunHost, budget: XcrunShimProbeOptions = {}) { + return await withFakeXcrunHost(host, () => + acquireXcodebuildSimulatorSetRedirect(scopedSimulator(layout.requestedSetPath), { + xctestDeviceSetPath: layout.xctestDeviceSetPath, + backupPath: layout.backupPath, + lockDirPath: layout.lockDirPath, + ...budget, + }), + ); +} + +async function assertRefused( + layout: Layout, + host: FakeXcrunHost, + budget: XcrunShimProbeOptions = {}, + expected: 'armed' | 'request_canceled' | 'phase_budget_exhausted' = 'armed', +): Promise { + let refusal: AppError | undefined; + await assert.rejects(acquire(layout, host, budget), (error: unknown) => { + assert.ok(error instanceof AppError); + refusal = error; + return true; + }); + assert.ok(refusal); + assert.equal(refusal.code, 'COMMAND_FAILED'); + if (expected === 'request_canceled') { + assert.equal(isRequestCanceledError(refusal), true); + } else if (expected === 'phase_budget_exhausted') { + assert.equal(refusal.details?.reason, 'runner_phase_budget_exhausted'); + assert.equal(refusal.details?.xcrunShims, undefined); + } else { + assert.equal(refusal.details?.reason, REASON); + assert.equal(refusal.details?.hint, ROW_HINT); + } + assert.equal(fs.lstatSync(layout.xctestDeviceSetPath).isSymbolicLink(), false); + assert.equal( + fs.readFileSync(path.join(layout.xctestDeviceSetPath, 'host-device.txt'), 'utf8'), + 'the host owns this', + ); + assert.equal(fs.existsSync(layout.backupPath), false, 'nothing was renamed aside'); + assert.equal(fs.existsSync(layout.lockDirPath), false, 'the lock was given back'); + return refusal; +} + +async function assertRedirected(layout: Layout, host: FakeXcrunHost): Promise { + const handle = await acquire(layout, host); + try { + assert.notEqual(handle, null); + assert.equal(fs.lstatSync(layout.xctestDeviceSetPath).isSymbolicLink(), true); + assert.equal( + fs.realpathSync.native(layout.xctestDeviceSetPath), + fs.realpathSync.native(layout.requestedSetPath), + ); + } finally { + await handle?.release(); + } +} + +function shimsOf(refusal: AppError): Array> { + const shims = refusal.details?.xcrunShims; + assert.ok(Array.isArray(shims)); + return shims as Array>; +} + +function shimOf(refusal: AppError, tool: XcrunShimToolName): Record | undefined { + return shimsOf(refusal).find((shim) => shim.tool === tool); +} + +const SIMCTL_EQUAL = { expectedVersion: '1155.4', installedVersion: '1155.4' }; +const DEVICECTL_EQUAL = { expectedVersion: '629.3', installedVersion: '629.3' }; + +test('an armed simctl shim refuses the redirect', async () => { + const layout = makeLayout(); + const host = writeFakeXcrunShims(layout.root, { + simctl: { expectedVersion: '1051.17.7', installedVersion: '1155.4' }, + devicectl: { hook: 'none' }, + }); + + const refusal = await assertRefused(layout, host); + + assert.deepEqual(shimOf(refusal, 'simctl'), { + tool: 'simctl', + shimPath: host.xcrunShimPaths.simctl, + hook: 'armed', + armedBy: 'version_mismatch', + expectedVersion: '1051.17.7', + frameworkInfoPlistPath: fakeFrameworkInfoPlistPath(layout.root, 'simctl'), + installedVersion: '1155.4', + }); +}); + +test('an armed devicectl shim refuses the redirect even when simctl matches', async () => { + const layout = makeLayout(); + const host = writeFakeXcrunShims(layout.root, { + simctl: SIMCTL_EQUAL, + devicectl: { expectedVersion: '506.6', installedVersion: '629.3' }, + }); + + const refusal = await assertRefused(layout, host); + + assert.deepEqual( + shimsOf(refusal).map((shim) => [shim.tool, shim.hook]), + [ + ['simctl', 'disarmed'], + ['devicectl', 'armed'], + ], + ); + assert.match(refusal.message, /Xcode's devicectl expects CoreDevice 506\.6; installed 629\.3/); +}); + +test('an unreadable framework Info.plist fails closed at the gate', async () => { + const layout = makeLayout(); + const host = writeFakeXcrunShims(layout.root, { + simctl: SIMCTL_EQUAL, + devicectl: DEVICECTL_EQUAL, + }); + host.installedVersions.delete(fakeFrameworkInfoPlistPath(layout.root, 'devicectl')); + + const refusal = await assertRefused(layout, host); + + assert.equal(shimOf(refusal, 'devicectl')?.armedBy, 'version_unreadable'); + assert.match( + refusal.message, + /Xcode's devicectl expects CoreDevice 629\.3; installed \(unreadable\)/, + ); +}); + +test('matching or hookless shims let the redirect through', async () => { + for (const shims of [ + { simctl: SIMCTL_EQUAL, devicectl: DEVICECTL_EQUAL }, + { simctl: { hook: 'none' }, devicectl: { hook: 'none' } }, + ] as const) { + const layout = makeLayout(); + await assertRedirected(layout, writeFakeXcrunShims(layout.root, shims)); + } +}); + +test('the captured Xcode 26.2 simctl shim reads as armed against CoreSimulator 1155.4', async () => { + const captured = RUNNER_STARTUP_FAILURE_FIXTURES.find( + (fixture) => fixture.id === 'xcode-26-2-simctl-shim-first-launch', + ); + assert.ok(captured); + const layout = makeLayout(); + const host = writeFakeXcrunShims(layout.root, { + simctl: { text: captured.output }, + devicectl: DEVICECTL_EQUAL, + }); + const coreSimulatorPlist = + '/Library/Developer/PrivateFrameworks/CoreSimulator.framework/Versions/A/Resources/Info.plist'; + host.installedVersions.set(coreSimulatorPlist, '1155.4'); + + const refusal = await assertRefused(layout, host); + + assert.equal(captured.reason, refusal.details?.reason); + assert.match( + refusal.message, + /Xcode's simctl expects CoreSimulator 1051\.17\.7; installed 1155\.4/, + ); + assert.ok(host.plistReads.includes(coreSimulatorPlist)); +}); + +test('a simulator that needs no redirect never probes the shims', async () => { + const probe = vi.fn(); + appleRunnerTestHost.update({ probeXcrunShimFirstLaunchHooks: probe }); + const layout = makeLayout(); + + const defaultSet = await acquireXcodebuildSimulatorSetRedirect( + { ...scopedSimulator(layout.requestedSetPath), simulatorSetPath: undefined }, + { lockDirPath: layout.lockDirPath, xctestDeviceSetPath: layout.xctestDeviceSetPath }, + ); + const xctestSet = await acquireXcodebuildSimulatorSetRedirect( + scopedSimulator(layout.xctestDeviceSetPath), + { lockDirPath: layout.lockDirPath, xctestDeviceSetPath: layout.xctestDeviceSetPath }, + ); + + assert.equal(defaultSet, null); + assert.equal(xctestSet, null); + assert.equal(probe.mock.calls.length, 0); +}); + +test('an already-canceled request gives the lock back as a cancellation without reading a shim', async () => { + const layout = makeLayout(); + const host = writeFakeXcrunShims(layout.root, { + simctl: SIMCTL_EQUAL, + devicectl: DEVICECTL_EQUAL, + }); + + await assertRefused(layout, host, { signal: AbortSignal.abort() }, 'request_canceled'); + + assert.deepEqual(host.finds, []); + assert.deepEqual(host.plistReads, []); +}); + +test('a request canceled while a shim is read gives the lock back as a cancellation, not a refusal', async () => { + const layout = makeLayout(); + const host = writeFakeXcrunShims(layout.root, { + simctl: SIMCTL_EQUAL, + devicectl: DEVICECTL_EQUAL, + }); + const request = new AbortController(); + host.onPlistRead = () => request.abort(); + + await assertRefused(layout, host, { signal: request.signal }, 'request_canceled'); + + assert.notDeepEqual(host.plistReads, [], 'the probe was reading a shim when the request ended'); +}); + +test('a phase that runs out while a stalled xcrun is probed is the phase budget, not an armed shim', async () => { + const layout = makeLayout(); + const host = writeFakeXcrunShims(layout.root, { + simctl: SIMCTL_EQUAL, + devicectl: DEVICECTL_EQUAL, + }); + host.findStalls = true; + const deadline: DeadlineClock = { + remainingMs: () => 3, + elapsedMs: () => 0, + isExpired: () => false, + }; + + await assertRefused(layout, host, { deadline }, 'phase_budget_exhausted'); + + assert.deepEqual(host.finds, XCRUN_SHIM_TOOL_NAMES, 'the probe was waiting on xcrun'); +}); + +test('the message tells a shim xcrun could not locate from one the probe ran out of budget on', async () => { + const messages: string[] = []; + for (const armedBy of ['shim_not_located', 'probe_out_of_budget'] as const) { + const layout = makeLayout(); + const host = writeFakeXcrunShims(layout.root, {}); + if (armedBy === 'probe_out_of_budget') { + appleRunnerTestHost.update({ + probeXcrunShimFirstLaunchHooks: async () => ({ + outcome: 'read', + xcrunShims: XCRUN_SHIM_TOOL_NAMES.map((tool): ArmedXcrunShimFirstLaunchHook => ({ + tool, + shimPath: null, + hook: 'armed', + armedBy, + expectedVersion: null, + frameworkInfoPlistPath: null, + installedVersion: null, + })), + }), + }); + } + const refusal = await assertRefused(layout, host); + assert.deepEqual( + shimsOf(refusal).map((shim) => [shim.tool, shim.armedBy]), + XCRUN_SHIM_TOOL_NAMES.map((tool) => [tool, armedBy]), + ); + messages.push(refusal.message); + } + + assert.match(messages[0] ?? '', /Xcode's simctl could not be located/); + assert.match(messages[1] ?? '', /Xcode's simctl shim was not read within the probe budget/); +}); + +test('a restore that could not give the host set back outranks the shim refusal', async () => { + const layout = makeLayout(); + const host = writeFakeXcrunShims(layout.root, { + simctl: { expectedVersion: '1051.17.7', installedVersion: '1155.4' }, + devicectl: { hook: 'none' }, + }); + + // Nothing is left to restore on the way in, so only the give-back's own reconcile can be made to + // fail: force `XCTestDevices` to keep reading as an orphaned symlink, and let only the give-back's + // unlink attempt refuse. + const realLstatSync = fs.lstatSync.bind(fs); + const lstatSpy = vi.spyOn(fs, 'lstatSync').mockImplementation((( + target: fs.PathLike, + options?: unknown, + ) => { + if (String(target) === layout.xctestDeviceSetPath) { + return { isSymbolicLink: () => true } as fs.Stats; + } + return (realLstatSync as (p: fs.PathLike, o?: unknown) => fs.Stats)(target, options); + }) as typeof fs.lstatSync); + const realUnlinkSync = fs.unlinkSync.bind(fs); + let unlinkAttempts = 0; + const unlinkSpy = vi.spyOn(fs, 'unlinkSync').mockImplementation(((target: fs.PathLike) => { + if (String(target) !== layout.xctestDeviceSetPath) { + realUnlinkSync(target); + return; + } + unlinkAttempts += 1; + if (unlinkAttempts > 1) { + throw Object.assign(new Error('EACCES: permission denied'), { code: 'EACCES' }); + } + }) as typeof fs.unlinkSync); + + try { + await assert.rejects( + acquire(layout, host), + (error: unknown) => (error as NodeJS.ErrnoException).code === 'EACCES', + ); + assert.equal(unlinkAttempts, 2, 'the redirect-in reconcile ran once, the give-back once more'); + assert.equal(fs.existsSync(layout.lockDirPath), false, 'the lock still went back'); + } finally { + lstatSpy.mockRestore(); + unlinkSpy.mockRestore(); + } +}); diff --git a/packages/platform-apple/src/runner/__tests__/runner-device-set.test.ts b/packages/platform-apple/src/runner/__tests__/runner-device-set.test.ts index f2fcba7f86..a53cab47f5 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-device-set.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-device-set.test.ts @@ -1,10 +1,11 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; -import { test, vi } from 'vitest'; +import { beforeEach, test, vi } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { mkdtempForTestSync } from './tmp-dir.ts'; +import { appleRunnerTestHost } from '../test-host.ts'; import { acquireXcodebuildSimulatorSetRedirect, resolveXcodebuildSimulatorDeviceSetPath, @@ -15,6 +16,12 @@ import { // the two failures it can report have an order: the build that failed outranks a redirect it could // not hand back, and a build that succeeded does not get to hide one. +beforeEach(() => { + appleRunnerTestHost.update({ + probeXcrunShimFirstLaunchHooks: async () => ({ outcome: 'read', xcrunShims: [] }), + }); +}); + const iosSimulator: DeviceInfo = { platform: 'apple', id: 'sim-1', @@ -96,11 +103,11 @@ test('a build that failed outranks the redirect it could not give back', async ( () => withXcodebuildSimulatorSetRedirect( makeScopedSimulator(paths), + redirectOptions(paths), async () => { makeReleaseUnverifiable(paths); throw buildFailure; }, - redirectOptions(paths), ), (error: unknown) => { assert.equal(error, buildFailure); @@ -120,11 +127,11 @@ test('a build that succeeded still reports the redirect it could not give back', () => withXcodebuildSimulatorSetRedirect( makeScopedSimulator(paths), + redirectOptions(paths), async () => { makeReleaseUnverifiable(paths); return 'built'; }, - redirectOptions(paths), ), (error: unknown) => { assert.ok(error instanceof AppError); diff --git a/packages/platform-apple/src/runner/__tests__/runner-request-cancellation.test.ts b/packages/platform-apple/src/runner/__tests__/runner-request-cancellation.test.ts index 20f0a770cd..df1f0111ca 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-request-cancellation.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-request-cancellation.test.ts @@ -180,6 +180,10 @@ test('direct command cancellation reaches runner launch without a registered req (error: unknown) => isRequestCanceledError(error), ); + assert.equal( + mockAcquireXcodebuildSimulatorSetRedirect.mock.calls.at(-1)?.[1]?.signal, + controller.signal, + ); assert.equal(readRunnerSessionLiveness(device.id), null); }); diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts index cbb75e7ccc..5a869800eb 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts @@ -3,6 +3,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { beforeEach, test, vi } from 'vitest'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; import { IOS_DEVICE, IOS_SIMULATOR, MACOS_DEVICE } from './device-fixtures.ts'; import { appleRunnerTestHost } from '../test-host.ts'; import { resolveRunnerLaunchLogPath } from '../runner-io.ts'; @@ -17,6 +18,11 @@ import { redirectRelease, } from './runner-session-fixtures.ts'; import { mkdtempForTestSync } from './tmp-dir.ts'; +import { + withFakeXcrunHost, + writeFakeXcrunShims, +} from '../../core/__tests__/xcrun-shim-fixtures.ts'; +import { acquireXcodebuildSimulatorSetRedirect as acquireRealSimulatorSetRedirect } from '../runner-device-set.ts'; const { mockAcquireXcodebuildSimulatorSetRedirect, @@ -387,6 +393,67 @@ test('a scoped simulator-set session stays on the kill path that restores the re assert.equal(redirectRelease.mock.calls.length, 0); }); +test('an armed xcrun shim refuses a scoped-set session before the runner launches', async () => { + const root = mkdtempForTestSync('runner-lifecycle-armed-shim-'); + const requestedSetPath = path.join(root, 'user-set'); + const xctestDeviceSetPath = path.join(root, 'XCTestDevices'); + fs.mkdirSync(requestedSetPath, { recursive: true }); + fs.mkdirSync(xctestDeviceSetPath, { recursive: true }); + const host = writeFakeXcrunShims(root, { + simctl: { expectedVersion: '1155.4', installedVersion: '1155.4' }, + devicectl: { expectedVersion: '506.6', installedVersion: '629.3' }, + }); + const redirectRemainingMs: Array = []; + mockAcquireXcodebuildSimulatorSetRedirect.mockImplementation( + async (device: DeviceInfo, options: Parameters[1]) => { + redirectRemainingMs.push(options?.deadline?.remainingMs()); + return await acquireRealSimulatorSetRedirect(device, { + ...options, + xctestDeviceSetPath, + lockDirPath: path.join(root, 'xctest-device-set.lock'), + }); + }, + ); + const buildMs = 50_000; + const realNow = Date.now.bind(Date); + mockEnsureXctestrunArtifact.mockImplementation(async () => { + vi.spyOn(Date, 'now').mockImplementation(() => realNow() + buildMs); + return { + xctestrunPath: '/tmp/base-runner.xctestrun', + derived: '/tmp/derived', + cache: 'exact', + artifact: 'rebuilt', + buildMs, + xctestrunPathSource: 'build', + }; + }); + const device = { + ...IOS_SIMULATOR, + id: 'runner-lifecycle-armed-shim', + simulatorSetPath: requestedSetPath, + }; + + try { + await assert.rejects( + withFakeXcrunHost(host, () => ensureRunnerSession(device, { startupTimeoutMs: 60_000 })), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'xctest_device_set_cleanup_armed', + ); + } finally { + vi.mocked(Date.now).mockRestore(); + } + + const remainingMs = redirectRemainingMs[0] ?? Number.NaN; + assert.equal( + remainingMs > 60_000 - buildMs && remainingMs <= 60_000, + true, + `the redirect spends the startup time read before the build, not what the build left: ${remainingMs} ms`, + ); + assert.equal(mockRunCmdBackground.mock.calls.length, 0, 'no test-without-building was spawned'); + assert.equal(fs.lstatSync(xctestDeviceSetPath).isSymbolicLink(), false); + assert.equal(readRunnerSessionLiveness(device.id), null); +}); + // #2681: the handoff lanes and every gate that keeps a runner on the kill path. async function serveOneCommand(device: DeviceInfo, session: RunnerSession): Promise { mockWaitForRunner.mockResolvedValueOnce(runnerResponse({ nodes: [], truncated: false })); diff --git a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts index c38b2b1dd7..1595877325 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts @@ -5,6 +5,7 @@ import type { } from '../runner-error-classification.ts'; import { RUNNER_DEVICE_READINESS_FAILURE_REASONS } from '../runner-error-classification.ts'; import type { IosPhysicalDeviceRunnerControl } from '../../core/physical-device-routing.ts'; +import { XCODE_26_2_SIMCTL_SHIM } from '../../core/__tests__/xcrun-shim-fixtures.ts'; /** * Recorded startup failures for {@link classifyRunnerStartupFailure} (#2680). @@ -43,7 +44,8 @@ import type { IosPhysicalDeviceRunnerControl } from '../../core/physical-device- export type RunnerStartupFailureSite = | 'build-for-testing' | 'host-dev-tools-security' - | 'device-readiness'; + | 'device-readiness' + | 'xctest-device-set-redirect'; /** * The two states a device reports about itself (#2683), in the shape `readIosDeviceReadiness` @@ -87,10 +89,19 @@ export type RunnerStartupFailureFixture = Readonly<{ hostTimeoutMs?: number; /** The invocation that produced {@link RunnerStartupFailureFixture.output}, once one is recorded. */ command?: string; - /** `xcodebuild -version` recorded from that run, or `unobserved`. */ + /** + * `xcodebuild -version` recorded from that run, or `unobserved`. The `xctest-device-set-redirect` + * site reads no `xcodebuild` output before refusing, so its entries record the shim's own + * `version.plist` reading instead, in that source's shape rather than `xcodebuild -version`'s; each + * one says so in its `note`. + */ xcodeVersion: string; provenance: 'captured' | 'shipped-sniff-trigger' | 'invented-shape'; - /** The tool's own stdout/stderr. */ + /** + * The tool's own stdout/stderr, except on the `xctest-device-set-redirect` site: there the redirect + * refuses on the shim script's own text, before any tool runs, and this field carries that text + * instead. + */ output: string; /** The argv the exec reported, which is never evidence of a cause (#2680). */ args?: readonly string[]; @@ -348,6 +359,16 @@ export const RUNNER_STARTUP_FAILURE_FIXTURES: readonly RunnerStartupFailureFixtu output: 'Developer mode is currently disabled for development tools.\n', note: "Host-side refusal. It says nothing about the device's Developer Mode toggle (#2683 reads that).", }, + { + id: 'xcode-26-2-simctl-shim-first-launch', + reason: 'xctest_device_set_cleanup_armed', + site: 'xctest-device-set-redirect', + command: XCODE_26_2_SIMCTL_SHIM.command, + xcodeVersion: XCODE_26_2_SIMCTL_SHIM.xcodeVersion, + provenance: 'captured', + output: XCODE_26_2_SIMCTL_SHIM.text, + note: 'The shim text, not tool output: the redirect refuses on what the shim would do, before any xcodebuild runs (#2935). Installed CoreSimulator on that host was 1155.4. The Xcode version was read from its version.plist, not from `xcodebuild -version`.', + }, ]; export function buildForTestingFixtures(): RunnerStartupFailureFixture[] { diff --git a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts index 94a26223e6..62530c1c64 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts @@ -59,6 +59,7 @@ const HINT_FOR_REASON: Record = { signing_provisioning_profile_missing: 'AGENT_DEVICE_IOS_PROVISIONING_PROFILE', signing_unspecified: 'Automatic Signing', devtools_security_developer_mode_disabled: 'DevToolsSecurity -enable', + xctest_device_set_cleanup_armed: 'xcode-select -s', // Both device remedies are owned by `core/devicectl.ts` and travel on the device report, so this // table quotes them instead of restating them; `runner-device-readiness.test.ts` is where the // preflight publishing them is asserted. @@ -289,6 +290,35 @@ test('an identical message without the typed host fact is not read as a DevTools assert.doesNotMatch(String(envelope.hint), /DevToolsSecurity/); }); +test('the XCTest device-set refusal is classified by its typed shim list, never by its wording', () => { + const message = + "Refusing to redirect XCTest device set: Xcode's simctl expects CoreSimulator 1051.17.7; installed 1155.4"; + + const typed = classifyRunnerStartupFailure( + new AppError('COMMAND_FAILED', message, { + // The redirect only ever builds this message once `armed.length > 0`, so the typed fact this + // reads always carries at least one shim; an empty array is not a shape production publishes. + xcrunShims: [ + { + tool: 'simctl', + shimPath: '/path/to/simctl', + hook: 'armed', + armedBy: 'version_mismatch', + expectedVersion: '1051.17.7', + frameworkInfoPlistPath: + '/Library/Developer/PrivateFrameworks/CoreSimulator.framework/Versions/A/Resources/Info.plist', + installedVersion: '1155.4', + }, + ], + }), + ); + const wordingOnly = classifyRunnerStartupFailure(new AppError('COMMAND_FAILED', message)); + + assert.equal(typed.reason, 'xctest_device_set_cleanup_armed'); + assert.ok(typed.hint.includes(HINT_FOR_REASON.xctest_device_set_cleanup_armed)); + assert.equal(wordingOnly.reason, RUNNER_STARTUP_FAILURE_UNCLASSIFIED_REASON); +}); + test('an app identifier named without the availability fact is not read as a taken bundle id', async () => { const nearMiss: RunnerStartupFailureFixture = { ...buildFixtureById('app-id-not-available'), diff --git a/packages/platform-apple/src/runner/apple-runner-platform.ts b/packages/platform-apple/src/runner/apple-runner-platform.ts index 5bf365141f..b1ce6e878d 100644 --- a/packages/platform-apple/src/runner/apple-runner-platform.ts +++ b/packages/platform-apple/src/runner/apple-runner-platform.ts @@ -8,15 +8,15 @@ import { } from '@agent-device/kernel/device'; /** - * Ceiling on one Apple toolchain identity probe attempt (`xcodebuild -version`, `xcrun - * --sdk --show-sdk-version`). On a fresh macOS host Apple's syspolicyd signature + * Ceiling on one Apple toolchain probe attempt (`xcodebuild -version`, `xcrun --sdk + * --show-sdk-version`, `xcrun --find `). On a fresh macOS host Apple's syspolicyd signature * scan blocks the first `xcodebuild`/`xcrun` exec after boot for roughly 18 to 19 seconds * at 0% CPU, and the next exec of the same tool is instant; a budget sized for a warm * toolchain (the old 10 s / 5 s split) trips on that stall and reports a toolchain * timeout that says nothing about the toolchain (#2422). * - * It sits beside the SDK names the probes run against so both Apple toolchain probers - * read one value without either owning it. + * It sits beside the SDK names the probes run against so every Apple toolchain prober + * reads one value without any of them owning it. */ export const COLD_TOOLCHAIN_PROBE_TIMEOUT_MS = 30_000; diff --git a/packages/platform-apple/src/runner/host.ts b/packages/platform-apple/src/runner/host.ts index 1c5c9bbb60..bd9e872c08 100644 --- a/packages/platform-apple/src/runner/host.ts +++ b/packages/platform-apple/src/runner/host.ts @@ -18,6 +18,7 @@ import type * as ApplePlistXml from '../core/plist-xml.ts'; import type * as AppleRunnerOwnerState from '../core/runner-owner-state.ts'; import type * as AppleSimctl from '../core/simctl.ts'; import type * as AppleToolProvider from '../core/tool-provider.ts'; +import type * as AppleXcrunShimFirstLaunch from '../core/xcrun-shim-first-launch.ts'; /** * The host-capability port for the Apple runner client. Every effectful or @@ -82,6 +83,7 @@ export type AppleRunnerHost = Pick< Pick & Pick & Pick & + Pick & Pick & Pick & { /** @@ -116,6 +118,12 @@ export type { IosDeviceReadiness, } from '../core/physical-device-coredevice.ts'; +export type { + ArmedXcrunShimFirstLaunchHook, + XcrunShimArmedBy, + XcrunShimProbeOptions, +} from '../core/xcrun-shim-first-launch.ts'; + let boundHost: AppleRunnerHost | undefined; /** @@ -184,6 +192,7 @@ export const bootFailureHint = delegate('bootFailureHint'); export const runAppleToolCommand = delegate('runAppleToolCommand'); export const runXcrun = delegate('runXcrun'); export const readApplePlistJson = delegate('readApplePlistJson'); +export const probeXcrunShimFirstLaunchHooks = delegate('probeXcrunShimFirstLaunchHooks'); export const buildSimctlArgsForDevice = delegate('buildSimctlArgsForDevice'); export const visitXmlPlistEntries = delegate('visitXmlPlistEntries'); export const resolveIosPhysicalDeviceControl = delegate('resolveIosPhysicalDeviceControl'); diff --git a/packages/platform-apple/src/runner/runner-artifact.ts b/packages/platform-apple/src/runner/runner-artifact.ts index ba7cff5734..bc49dcc59d 100644 --- a/packages/platform-apple/src/runner/runner-artifact.ts +++ b/packages/platform-apple/src/runner/runner-artifact.ts @@ -472,7 +472,7 @@ async function buildRunnerXctestrun( const provisioningArgs = device.kind === 'device' ? ['-allowProvisioningUpdates'] : []; const performanceBuildSettings = resolveRunnerPerformanceBuildSettings(); const sandboxBuildArgs = resolveRunnerSandboxBuildArgs(); - await withXcodebuildSimulatorSetRedirect(device, async () => { + await withXcodebuildSimulatorSetRedirect(device, options.budget ?? {}, async () => { try { await runCmdStreaming( 'xcodebuild', diff --git a/packages/platform-apple/src/runner/runner-cache-metadata.ts b/packages/platform-apple/src/runner/runner-cache-metadata.ts index cdf874e44a..b384f8f5a3 100644 --- a/packages/platform-apple/src/runner/runner-cache-metadata.ts +++ b/packages/platform-apple/src/runner/runner-cache-metadata.ts @@ -110,7 +110,7 @@ export function requireRunnerPhaseRemainingMs( } /** Says the phase budget ran out, not that the step it would have run is broken. */ -function runnerPhaseBudgetExhaustedError(phase: string): AppError { +export function runnerPhaseBudgetExhaustedError(phase: string): AppError { return new AppError('COMMAND_FAILED', 'The Apple runner budget ran out before this step began', { phase, reason: 'runner_phase_budget_exhausted', diff --git a/packages/platform-apple/src/runner/runner-device-set.ts b/packages/platform-apple/src/runner/runner-device-set.ts index bde300e25c..a46441b3db 100644 --- a/packages/platform-apple/src/runner/runner-device-set.ts +++ b/packages/platform-apple/src/runner/runner-device-set.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { AppError } from '@agent-device/kernel/errors'; +import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; import { resolveIosSimulatorDeviceSetPath, @@ -9,8 +9,14 @@ import { readProcessStartTime, acquireProcessLock, withProcessLock, + probeXcrunShimFirstLaunchHooks, + type ArmedXcrunShimFirstLaunchHook, + type XcrunShimArmedBy, + type XcrunShimProbeOptions, } from './host.ts'; import type { ProcessLockRelease } from '@agent-device/host-kit/file'; +import { classifyRunnerStartupFailure } from './runner-error-classification.ts'; +import { runnerPhaseBudgetExhaustedError } from './runner-cache-metadata.ts'; const XCTEST_DEVICE_SET_BASE_NAME = 'XCTestDevices'; const XCTEST_DEVICE_SET_BACKUP_SUFFIX = '.agent-device-backup'; @@ -18,6 +24,7 @@ const XCTEST_DEVICE_SET_LEGACY_BACKUP_PREFIX = '.agent-device-xctestdevices-back const XCTEST_DEVICE_SET_LOCK_TIMEOUT_MS = 30_000; const XCTEST_DEVICE_SET_LOCK_POLL_MS = 100; const XCTEST_DEVICE_SET_LOCK_OWNER_GRACE_MS = 5_000; +const XCRUN_SHIM_PROBE_PHASE = 'xctest_device_set_shim_probe'; export type XcodebuildSimulatorSetRedirectHandle = { /** @@ -34,7 +41,7 @@ export type XcodebuildSimulatorSetRedirectHandle = { releaseBestEffort: () => Promise; }; -type XcodebuildSimulatorSetRedirectOptions = { +type XcodebuildSimulatorSetRedirectOptions = XcrunShimProbeOptions & { xctestDeviceSetPath?: string; backupPath?: string; lockDirPath?: string; @@ -62,8 +69,8 @@ function resolveXcodebuildSimulatorDeviceSetBackupPath( */ export async function withXcodebuildSimulatorSetRedirect( device: DeviceInfo, + options: XcodebuildSimulatorSetRedirectOptions, task: () => Promise, - options: XcodebuildSimulatorSetRedirectOptions = {}, ): Promise { const redirect = await acquireXcodebuildSimulatorSetRedirect(device, options); if (!redirect) return await task(); @@ -106,6 +113,7 @@ export async function acquireXcodebuildSimulatorSetRedirect( const paths = { xctestDeviceSetPath, backupPath }; let needsRedirect = false; + let redirectRefusal: AppError | null = null; // One try, so the lock cannot be given back and then worked under: the restore of an interrupted // build's leftovers runs first because the same-set check follows symlinks, and `XCTestDevices` left @@ -115,6 +123,9 @@ export async function acquireXcodebuildSimulatorSetRedirect( reconcileXcodebuildSimulatorSetRedirect(paths); needsRedirect = !sameResolvedPath(requestedSetPath, xctestDeviceSetPath); if (needsRedirect) { + redirectRefusal = await xcrunShimProbeRefusal(options); + } + if (needsRedirect && redirectRefusal === null) { installDeviceSetRedirect(paths, requestedSetPath); } } catch (error) { @@ -124,13 +135,15 @@ export async function acquireXcodebuildSimulatorSetRedirect( throw redirectFailure(error, handBack, { requestedSetPath, ...paths }); } + if (redirectRefusal !== null) { + await handBackOrThrowRestoreFailure(paths, lockDirPath, releaseLock); + throw redirectRefusal; + } + if (!needsRedirect) { // Nothing is displaced and the caller gets no handle: a lock this simulator never needed must not // arrive as a redirect problem, and a host device set that could not be put back still must. - const handBack = await handBackDeviceSet(paths, lockDirPath, releaseLock); - if (handBack.restoreFailure !== null) { - throw handBack.restoreFailure; - } + await handBackOrThrowRestoreFailure(paths, lockDirPath, releaseLock); return null; } @@ -140,10 +153,7 @@ export async function acquireXcodebuildSimulatorSetRedirect( return; } givenBack = true; - const handBack = await handBackDeviceSet(paths, lockDirPath, releaseLock); - if (handBack.restoreFailure !== null) { - throw handBack.restoreFailure; - } + const handBack = await handBackOrThrowRestoreFailure(paths, lockDirPath, releaseLock); if (handBack.releaseFailure !== null && reportUnverifiedRelease) { throw handBack.releaseFailure; } @@ -154,6 +164,58 @@ export async function acquireXcodebuildSimulatorSetRedirect( }; } +/** + * Why the redirect may not go ahead, or null when every shim is safe. A host where an Xcode shim + * would run `xcodebuild -runFirstLaunch` — which deletes every device in `XCTestDevices` and so, + * through the redirect, every device in the requested set — is refused with the reason and hint + * {@link classifyRunnerStartupFailure} keys on `xcrunShims`. A probe the request canceled gets the + * canceled-request error, and one the owning phase's clock stopped gets the phase's budget error; + * neither is a host refusal. + */ +async function xcrunShimProbeRefusal(options: XcrunShimProbeOptions): Promise { + const probe = await probeXcrunShimFirstLaunchHooks({ + signal: options.signal, + deadline: options.deadline, + }); + if (probe.outcome === 'request_canceled') { + return createRequestCanceledError({ phase: XCRUN_SHIM_PROBE_PHASE }); + } + if (probe.outcome === 'phase_budget_exhausted') { + return runnerPhaseBudgetExhaustedError(XCRUN_SHIM_PROBE_PHASE); + } + const { xcrunShims } = probe; + const armed = xcrunShims.filter( + (shim): shim is ArmedXcrunShimFirstLaunchHook => shim.hook === 'armed', + ); + if (armed.length === 0) return null; + const described = armed.map((shim) => DESCRIBE_ARMED_SHIM[shim.armedBy](shim)); + const message = `Refusing to redirect XCTest device set: ${described.join('; ')}`; + const { reason, hint } = classifyRunnerStartupFailure( + new AppError('COMMAND_FAILED', message, { xcrunShims }), + ); + return new AppError('COMMAND_FAILED', message, { reason, hint, xcrunShims }); +} + +const DESCRIBE_ARMED_SHIM: Record< + XcrunShimArmedBy, + (shim: ArmedXcrunShimFirstLaunchHook) => string +> = { + version_mismatch: describeShimVersions, + version_unreadable: describeShimVersions, + shim_unreadable: (shim) => `Xcode's ${shim.tool} shim at ${shim.shimPath} could not be read`, + shim_not_located: (shim) => `Xcode's ${shim.tool} could not be located`, + probe_out_of_budget: (shim) => `Xcode's ${shim.tool} shim was not read within the probe budget`, +}; + +function describeShimVersions(shim: ArmedXcrunShimFirstLaunchHook): string { + const framework = + /([^/]+)\.framework\//.exec(shim.frameworkInfoPlistPath ?? '')?.[1] ?? 'its framework'; + return ( + `Xcode's ${shim.tool} expects ${framework} ${shim.expectedVersion ?? '(unreadable)'}; ` + + `installed ${shim.installedVersion ?? '(unreadable)'}` + ); +} + /** The two paths a redirect moves around: where the host keeps its set, and where this run put it. */ type DeviceSetPaths = { xctestDeviceSetPath: string; @@ -239,6 +301,23 @@ async function handBackDeviceSet( return { restoreFailure, renamedAsidePath, releaseFailure }; } +/** + * The hand-back every exit that does not already have a more specific error uses: a restore failure + * outranks whatever that exit was about to report, because it is a fact about this machine's device set + * that outlives the request. + */ +async function handBackOrThrowRestoreFailure( + paths: DeviceSetPaths, + lockDirPath: string, + releaseLock: ProcessLockRelease, +): Promise { + const handBack = await handBackDeviceSet(paths, lockDirPath, releaseLock); + if (handBack.restoreFailure !== null) { + throw handBack.restoreFailure; + } + return handBack; +} + /** * Where the host's own device set sits when it is not in place: the backup this run would have written, * or the older name an earlier version used, but only while it is really on disk. Once the host's set is diff --git a/packages/platform-apple/src/runner/runner-error-classification.ts b/packages/platform-apple/src/runner/runner-error-classification.ts index 97dd1b393f..96fb8d8d0d 100644 --- a/packages/platform-apple/src/runner/runner-error-classification.ts +++ b/packages/platform-apple/src/runner/runner-error-classification.ts @@ -80,6 +80,9 @@ const hasRunnerBusyCode: RunnerErrorDetailsMatch = (details) => */ const hasDevToolsSecurityStatus: RunnerErrorDetailsMatch = (details) => typeof details.devToolsSecurityStatus === 'string'; +/** The per-shim first-launch hooks the XCTest device-set redirect read before refusing. */ +const hasXcrunShimFirstLaunchHooks: RunnerErrorDetailsMatch = (details) => + Array.isArray(details.xcrunShims); const hasUsbmuxDeviceUnattached: RunnerErrorDetailsMatch = (details) => details.usbmuxDeviceAttached === false; const hasRunnerConnectFailureReason = @@ -157,6 +160,7 @@ export const RUNNER_STARTUP_FAILURE_REASONS = [ 'signing_provisioning_profile_missing', 'signing_unspecified', 'devtools_security_developer_mode_disabled', + 'xctest_device_set_cleanup_armed', ...RUNNER_DEVICE_READINESS_FAILURE_REASONS, 'build_failed_unclassified', ] as const; @@ -434,6 +438,15 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ hint: 'Run `sudo DevToolsSecurity -enable`, then retry the iOS runner. UI test runners start suspended until Xcode/testmanagerd can attach.', }, }, + { + reason: 'xctest_device_set_cleanup_armed', + match: { code: 'COMMAND_FAILED', details: hasXcrunShimFirstLaunchHooks }, + verdicts: {}, + buildFailure: { + reason: 'xctest_device_set_cleanup_armed', + hint: "The selected Xcode does not match the installed CoreSimulator or CoreDevice framework, or the shim could not be located or its version data could not be read in time. Until this is verified, its simctl or devicectl shim may run `xcodebuild -runFirstLaunch`, which deletes all devices in ~/Library/Developer/XCTestDevices. For a mismatch, select the Xcode that installed those frameworks (`xcode-select -s` or DEVELOPER_DIR); for an unreadable or timed-out probe, retry once the host is not under load. details.xcrunShims names each shim's state and its expected/installed versions when they could be read.", + }, + }, ]; function matchesRunnerErrorRule(error: AppError, match: RunnerErrorMatch): boolean { diff --git a/packages/platform-apple/src/runner/runner-session.ts b/packages/platform-apple/src/runner/runner-session.ts index c8f655733b..3fbd57d805 100644 --- a/packages/platform-apple/src/runner/runner-session.ts +++ b/packages/platform-apple/src/runner/runner-session.ts @@ -241,7 +241,8 @@ async function startRunnerSessionWithLease( phase: 'ios_runner_startup_cleanup_stale_bundles_skipped', }); } - // Read before the build, which is a phase of its own with its own budget (#2422). + // Read before the build, which is a phase of its own with its own budget (#2422); every startup step + // after the build, the device-set redirect included, spends from this snapshot. const startupTimeoutMs = requireRunnerPhaseRemainingMs(startupBudget, 'runner_session_startup'); let xctestrunArtifact: Awaited>; let port: number; @@ -286,7 +287,11 @@ async function startRunnerSessionWithLease( simulatorSetRedirect = await measureRunnerStartupStep( startupTimings, 'simulator_set_redirect', - async () => await acquireXcodebuildSimulatorSetRedirect(device), + async () => + await acquireXcodebuildSimulatorSetRedirect( + device, + createRunnerPhaseBudget(startupTimeoutMs, signal), + ), ); if (xctestrunArtifact.buildMs > 0) { emitRequestProgress({ diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 444a845d1d..b9ed8f51e6 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -690,7 +690,7 @@ iOS physical-device prerequisites: If Xcode cannot choose a profile, set AGENT_DEVICE_IOS_PROVISIONING_PROFILE to the profile name/specifier, not a file path. AGENT_DEVICE_IOS_SIGNING_IDENTITY is optional; omit it unless xcodebuild asks for a specific identity. The profile/team must allow AGENT_DEVICE_IOS_BUNDLE_ID and .uitests. - A runner startup failure names its class in error details.reason rather than only in prose: signing_no_development_team, signing_provisioning_profile_missing, bundle_identifier_already_registered, signing_unspecified, devtools_security_developer_mode_disabled (the Mac's DevToolsSecurity setting, which says nothing about the device's Developer Mode toggle), device_developer_mode_disabled (read from an iPhone's own report before the runner builds, and the only device state that stops a run up front), device_developer_disk_image_unavailable (also read from the device, and published on a startup failure that named no cause of its own, since iOS 17+ mounts the developer disk image on demand during build and launch rather than gating the build), or build_failed_unclassified when nothing proved a cause. The two device reasons are never inferred from tool output or from each other. Branch on details.reason and follow hint; the message is for humans. + A runner startup failure names its class in error details.reason rather than only in prose: signing_no_development_team, signing_provisioning_profile_missing, bundle_identifier_already_registered, signing_unspecified, devtools_security_developer_mode_disabled (the Mac's DevToolsSecurity setting, which says nothing about the device's Developer Mode toggle), xctest_device_set_cleanup_armed (a simulator in a --ios-simulator-device-set set, refused before the runner builds or launches; see Runner and daemon lifecycle), device_developer_mode_disabled (read from an iPhone's own report before the runner builds, and the only device state that stops a run up front), device_developer_disk_image_unavailable (also read from the device, and published on a startup failure that named no cause of its own, since iOS 17+ mounts the developer disk image on demand during build and launch rather than gating the build), or build_failed_unclassified when nothing proved a cause. The two device reasons are never inferred from tool output or from each other. Branch on details.reason and follow hint; the message is for humans. First-run XCTest setup/build can take longer than normal commands; keep the device connected and use --debug to inspect signing/build diagnostics if setup times out. Android physical-device prerequisites: @@ -701,6 +701,7 @@ Runner and daemon lifecycle (applies to simulators too): open without --relaunch is idempotent-foreground for an already-running app (it brings the process forward; it does not restart it). open --relaunch restarts the app; on iOS simulators this collapses to one simctl launch --terminate-running-process call instead of a separate terminate-then-launch. No runner read launches a session app that is not running: snapshot, wait, is, get, a reading find, and an interaction's leading reads (a gesture's viewport read, the capture that resolves a selector click/fill) answer the retriable APP_NOT_RUNNING instead of bare-launching over a launch SpringBoard still holds behind its deep-link confirmation. Only open, activate, and a command that mutates without a leading read bring a stopped app up. close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that device skips the runner build, unless --shutdown was requested, the session was recording, the session held a device lease, or the device used a scoped (non-default) simulator set. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. + With --ios-simulator-device-set, runner-backed commands refuse with COMMAND_FAILED and reason xctest_device_set_cleanup_armed when a redirect onto the scoped set finds a shim that would run xcodebuild -runFirstLaunch before simctl or devicectl, which deletes every device in ~/Library/Developer/XCTestDevices, and the runner points that path at the scoped set. A shim is armed for a version mismatch between the selected Xcode and the installed CoreSimulator or CoreDevice framework, or fails closed for one the probe could not verify: a shim it could not locate, shim or version text it could not read, or a probe that ran out of its own 30-second budget; details.xcrunShims names each shim's armedBy for which of these applies, with expected/installed versions filled in only when they were read. A runner phase that runs out of time during the probe fails with reason runner_phase_budget_exhausted instead, not as an armed shim. A scoped set is not protected from a first-launch cleanup that starts outside agent-device (a newly opened Xcode, a manual xcodebuild -runFirstLaunch) while a runner session holds that redirect. Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index d9eb54e337..bc3a9b7e4c 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -204,6 +204,8 @@ agent-device devices --platform android --android-device-allowlist emulator-5554 ``` - `--ios-simulator-device-set ` constrains simulator discovery and simulator command execution via `xcrun simctl --set ...`. +- Runner-backed commands on a scoped set fail with `COMMAND_FAILED` and `details.reason: "xctest_device_set_cleanup_armed"` when redirecting onto the scoped set finds a `simctl` or `devicectl` shim that would run `xcodebuild -runFirstLaunch` first, which deletes every device in `~/Library/Developer/XCTestDevices`, where the runner points the scoped set. A shim is armed for a version mismatch between the selected Xcode and the installed CoreSimulator or CoreDevice framework, or fails closed for one the probe could not verify (a shim it could not locate, shim or version text it could not read, or a probe that ran out of its own 30-second budget). `details.xcrunShims` names each shim's `armedBy` for which of these applies (`version_mismatch`, `version_unreadable`, `shim_unreadable`, `shim_not_located`, or `probe_out_of_budget`), with expected/installed versions filled in only when they were read. A runner phase that runs out of time during the probe fails with `details.reason: "runner_phase_budget_exhausted"` instead, not as an armed shim. For a version mismatch, select the matching Xcode with `xcode-select -s` or `DEVELOPER_DIR`; for an unreadable or timed-out probe, retry once the host is not under load. +- A scoped set is not protected from a first-launch cleanup that starts outside agent-device (opening a newly installed Xcode, or running `xcodebuild -runFirstLaunch` by hand) while a runner session holds that redirect. - `--android-device-allowlist ` constrains Android discovery/selection to comma or space separated serials. - Scope is applied before selectors (`--device`, `--udid`, `--serial`), so out-of-scope selectors fail with `DEVICE_NOT_FOUND`. - With iOS simulator-set scope enabled, iOS physical devices are not enumerated. diff --git a/website/docs/docs/installation.md b/website/docs/docs/installation.md index cd3c749502..8ab9ecbc22 100644 --- a/website/docs/docs/installation.md +++ b/website/docs/docs/installation.md @@ -108,7 +108,7 @@ vega device list - `AGENT_DEVICE_IOS_PROVISIONING_PROFILE` - `AGENT_DEVICE_IOS_BUNDLE_ID` (optional runner bundle-id base override) - Free Apple Developer (Personal Team) accounts can fail with "bundle identifier is not available" for generic IDs; set `AGENT_DEVICE_IOS_BUNDLE_ID` to a unique reverse-DNS value (for example `com.yourname.agentdevice.runner`). -- A runner startup failure is typed, not prose: `error.details.reason` is one of `signing_no_development_team`, `signing_provisioning_profile_missing`, `bundle_identifier_already_registered`, `signing_unspecified`, `devtools_security_developer_mode_disabled` (the Mac's `DevToolsSecurity` setting, which says nothing about the device's Developer Mode toggle), `device_developer_mode_disabled`, `device_developer_disk_image_unavailable`, or `build_failed_unclassified` when nothing proved a cause. Branch on `details.reason` and follow `hint`; the code stays `COMMAND_FAILED` for every reason. +- A runner startup failure is typed, not prose: `error.details.reason` is one of `signing_no_development_team`, `signing_provisioning_profile_missing`, `bundle_identifier_already_registered`, `signing_unspecified`, `devtools_security_developer_mode_disabled` (the Mac's `DevToolsSecurity` setting, which says nothing about the device's Developer Mode toggle), `xctest_device_set_cleanup_armed` (a simulator in an `--ios-simulator-device-set` set where a redirect finds a shim armed to run `xcodebuild -runFirstLaunch` — a version mismatch between the selected Xcode and the installed CoreSimulator or CoreDevice framework, or one the probe could not verify), `device_developer_mode_disabled`, `device_developer_disk_image_unavailable`, or `build_failed_unclassified` when nothing proved a cause. Branch on `details.reason` and follow `hint`; the code stays `COMMAND_FAILED` for every reason. - The two `device_*` reasons come from the iPhone itself, read over `xcrun devicectl device info details` before the runner builds: `developerModeStatus` for the Settings toggle and `ddiServicesAvailable` for the developer disk image. They are reported apart on purpose. A phone with Developer Mode off cannot serve its disk image either, so it gets the toggle reason; a phone with the toggle on and only the image down gets the disk-image reason, which is a device-support install that has not finished rather than a setting anyone turned off. - If device setup is slow, keep the device connected and inspect daemon diagnostics after retrying. - If daemon startup reports stale metadata, remove stale files and retry: