fix(apple): read the runner cache toolchain key from Xcode's own files

oxlint.config.ts

@@ -33,6 +33,9 @@ const NODE_OS_PATH = {
   name: 'node:os',
 };
 
+const TOOLCHAIN_IDENTITY_EXEC_MESSAGE =
+  'Read the toolchain identity from the selected Xcode files; an exec here can stall the runner cache decision.';
+
 const CHILD_PROCESS_BAN_ROOTS = ['src', 'packages/host-kit/src'];
 const PROVIDER_BAN_ROOTS = ['src/commands', 'src/cli/commands'];
 
@@ -244,6 +247,27 @@ export default defineConfig({
         'no-restricted-imports': ['error', { paths: [CHILD_PROCESS_PATH] }],
       },
     },
+    {
+      // The runner cache key reads the selected Xcode's files and execs nothing: a hosted macOS
+      // runner can stall an `xcrun` exec longer than any budget (Perf Nightly run 40441065511).
+      files: ['packages/platform-apple/src/runner/runner-toolchain-identity.ts'],
+      rules: {
+        'no-restricted-imports': [
+          'error',
+          {
+            paths: [
+              { ...CHILD_PROCESS_PATH, message: TOOLCHAIN_IDENTITY_EXEC_MESSAGE },
+              { name: '@agent-device/host-kit/command', message: TOOLCHAIN_IDENTITY_EXEC_MESSAGE },
+              {
+                name: './host.ts',
+                allowImportNames: ['commandDeveloperDir'],
+                message: TOOLCHAIN_IDENTITY_EXEC_MESSAGE,
+              },
+            ],
+          },
+        ],
+      },
+    },
     {
       // The host-kit process helpers are the sanctioned node:child_process importer.
       files: ['packages/host-kit/src/internal/exec.ts'],

packages/platform-apple/src/core/runner-host.ts

@@ -7,7 +7,6 @@ import {
   requireExecSuccess,
   runCmdBackground,
   runCmdStreaming,
-  runCmdSync,
 } from '@agent-device/host-kit/command';
 import { shellQuote } from '@agent-device/kernel/device-shell';
 import { AppError } from '@agent-device/kernel/errors';
@@ -31,7 +30,6 @@ import { findProjectRoot, readVersion } from '@agent-device/host-kit/version';
 import { isRecord } from '@agent-device/kernel/record';
 import { parseBooleanLiteral } from '@agent-device/kernel/source-value';
 import type { HostEnvironment } from '@agent-device/kernel/source-value';
-import { createTtlMemo } from '@agent-device/kernel/ttl-memo';
 
 import {
   getRequestSignal,
@@ -70,7 +68,6 @@ import {
  */
 export const appleRunnerHost: AppleRunnerHost = {
   runCmdStreaming,
-  runCmdSync,
   runCmdBackground,
   requireExecSuccess,
   isCommandTimeoutError,
@@ -104,7 +101,6 @@ export const appleRunnerHost: AppleRunnerHost = {
   publishFileSync,
   classifyOwnerLiveness,
   readHostCpuArchSync,
-  createTtlMemo,
   parseBooleanLiteral,
   isRecord,
   emitRequestProgress,

packages/platform-apple/src/foldable/fold-helper-cache.ts

@@ -4,11 +4,13 @@ import { execFailureDetails } from '@agent-device/host-kit/command';
 import { hostHomeDirectory } from '@agent-device/host-kit/host-file';
 import { findProjectRoot } from '@agent-device/host-kit/version';
 import { runAppleToolCommand } from '../core/tool-provider.ts';
-import { COLD_TOOLCHAIN_PROBE_TIMEOUT_MS } from '../runner/apple-runner-platform.ts';
 import { createNativeBuildDeadline, type NativeBuildDeadline } from '../native-build/deadline.ts';
 import { NativeBuildError } from '../native-build/errors.ts';
 import { createNativeBuildHost, type NativeBuildHost } from '../native-build/host.ts';
-import { readHostToolchainIdentity } from '../native-build/toolchain-identity.ts';
+import {
+  COLD_TOOLCHAIN_PROBE_TIMEOUT_MS,
+  readHostToolchainIdentity,
+} from '../native-build/toolchain-identity.ts';
 import {
   ensureNativeBuildCacheEntry,
   execNativeBuildClang,

packages/platform-apple/src/native-build/toolchain-identity.ts

@@ -1,10 +1,19 @@
 import { isCommandTimeoutError, type ExecResult } from '@agent-device/host-kit/command';
 import { isRequestCanceledError } from '@agent-device/kernel/errors';
-import { COLD_TOOLCHAIN_PROBE_TIMEOUT_MS } from '../runner/apple-runner-platform.ts';
 import { nativeBuildError, NativeBuildError } from './errors.ts';
 import { remainingNativeBuildMs, type NativeBuildDeadline } from './deadline.ts';
 import type { NativeBuildHost } from './host.ts';
 
+/**
+ * Ceiling on one toolchain probe attempt (`xcodebuild -version`, run through the Apple tool
+ * provider). On a fresh macOS host Apple's syspolicyd signature scan blocks the first
+ * `xcodebuild`/`xcrun` exec after boot for roughly 18 to 19 seconds at 0% CPU, and the next exec of
+ * the same tool is instant; a budget sized for a warm toolchain (the old 10 s / 5 s split) trips on
+ * that stall and reports a toolchain timeout that says nothing about the toolchain (#2422). The
+ * build deadlines around this probe add it for the same reason.
+ */
+export const COLD_TOOLCHAIN_PROBE_TIMEOUT_MS = 30_000;
+
 /**
  * The host's active toolchain, independent of any simulator runtime: which Xcode `xcrun` resolves
  * against, the macOS build it runs on, and its architecture. Shared by every runtime clang build in

packages/platform-apple/src/runner/__tests__/apple-toolchain-fixtures.ts

@@ -1,9 +1,8 @@
 import { beforeEach, vi } from 'vitest';
 import fs from 'node:fs';
-import { AppError } from '@agent-device/kernel/errors';
-import { appleRunnerTestHost } from '../test-host.ts';
-import { XCODE_SELECT_LINK_PATH } from '../apple-runner-platform.ts';
-import type { ExecOptions, ExecResult } from '@agent-device/host-kit/command';
+import path from 'node:path';
+import { XCODE_SELECT_LINK_PATH } from '../runner-toolchain-identity.ts';
+import { mkdtempForTestSync } from './tmp-dir.ts';
 
 export const STUBBED_APPLE_TOOLCHAIN = {
   xcodeVersion: '26.2',
@@ -12,120 +11,92 @@ export const STUBBED_APPLE_TOOLCHAIN = {
   sdkBuildVersion: '23C53',
 } as const;
 
-const STUBBED_SELECTED_DEVELOPER_DIR = '/Applications/Xcode-Stub.app/Contents/Developer';
+type StubbedToolchain = { [Key in keyof typeof STUBBED_APPLE_TOOLCHAIN]: string };
 
-export function selectedDeveloperDirResult(developerDir: string): ExecResult {
-  return { exitCode: 0, stdout: `${developerDir}\n`, stderr: '' };
-}
+const SDK_DIRECTORY_NAMES = [
+  'iPhoneSimulator',
+  'iPhoneOS',
+  'AppleTVSimulator',
+  'AppleTVOS',
+  'MacOSX',
+  'XRSimulator',
+  'XROS',
+];
 
-export function appleToolchainProbeResult(command: string, args: readonly string[]): ExecResult {
-  if (command === 'xcode-select' && args[0] === '-p') {
-    return selectedDeveloperDirResult(STUBBED_SELECTED_DEVELOPER_DIR);
-  }
-  if (command === 'xcodebuild' && args[0] === '-version') {
-    return {
-      exitCode: 0,
-      stdout: `Xcode ${STUBBED_APPLE_TOOLCHAIN.xcodeVersion}\nBuild version ${STUBBED_APPLE_TOOLCHAIN.xcodeBuildVersion}\n`,
-      stderr: '',
-    };
-  }
-  if (command === 'xcrun' && args.includes('--show-sdk-build-version')) {
-    return { exitCode: 0, stdout: `${STUBBED_APPLE_TOOLCHAIN.sdkBuildVersion}\n`, stderr: '' };
-  }
-  if (command === 'xcrun' && args.includes('--show-sdk-version')) {
-    return { exitCode: 0, stdout: `${STUBBED_APPLE_TOOLCHAIN.sdkVersion}\n`, stderr: '' };
-  }
-  throw new Error(`Unexpected Apple toolchain probe: ${command} ${args.join(' ')}`);
+function plist(entries: Record<string, string>): string {
+  const body = Object.entries(entries)
+    .map(([key, value]) => `\t<key>${key}</key>\n\t<string>${value}</string>`)
+    .join('\n');
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<plist version="1.0">\n<dict>\n${body}\n</dict>\n</plist>\n`;
 }
 
 /**
- * Answers the runner cache's toolchain probes from a fixed toolchain, so cases that key the cache
- * neither read the host's Xcode nor depend on one existing. No client chooses a `DEVELOPER_DIR`, so
- * the selection a key probe resolves is the one the case scripts. Returns the mock so a case can
- * make a probe fail.
+ * An `Xcode.app` under `root` carrying the files the toolchain identity reads, for every SDK the
+ * runner keys on. Returns its developer dir.
  */
-export function stubAppleToolchainProbes(): ReturnType<typeof vi.fn> {
-  stubAppleSelectionLink();
-  const runCmdSync = vi.fn(appleToolchainProbeResult);
-  beforeEach(() => {
-    runCmdSync.mockImplementation(appleToolchainProbeResult);
-    appleRunnerTestHost.update({
-      runCmdSync,
-      commandDeveloperDir: () => undefined,
-    });
-  });
-  return runCmdSync;
+export function writeXcodeFixture(
+  root: string,
+  toolchain: StubbedToolchain = STUBBED_APPLE_TOOLCHAIN,
+): string {
+  const contents = path.join(root, 'Xcode.app', 'Contents');
+  const developerDir = path.join(contents, 'Developer');
+  fs.mkdirSync(developerDir, { recursive: true });
+  fs.writeFileSync(
+    path.join(contents, 'version.plist'),
+    plist({
+      CFBundleShortVersionString: toolchain.xcodeVersion,
+      ProductBuildVersion: toolchain.xcodeBuildVersion,
+    }),
+  );
+  for (const name of SDK_DIRECTORY_NAMES) {
+    const sdk = path.join(developerDir, 'Platforms', `${name}.platform/Developer/SDKs/${name}.sdk`);
+    fs.mkdirSync(path.join(sdk, 'System/Library/CoreServices'), { recursive: true });
+    fs.writeFileSync(
+      path.join(sdk, 'SDKSettings.json'),
+      JSON.stringify({ Version: toolchain.sdkVersion }),
+    );
+    fs.writeFileSync(
+      path.join(sdk, 'System/Library/CoreServices/SystemVersion.plist'),
+      plist({ ProductBuildVersion: toolchain.sdkBuildVersion }),
+    );
+  }
+  return developerDir;
+}
+
+let stubbedDeveloperDir: string | undefined;
+
+/** One {@link STUBBED_APPLE_TOOLCHAIN} Xcode per worker, so a case never reads the host's Xcode. */
+export function stubbedAppleDeveloperDir(): string {
+  stubbedDeveloperDir ??= writeXcodeFixture(mkdtempForTestSync('apple-toolchain-'));
+  return stubbedDeveloperDir;
 }
 
 /**
- * The host's default selection link as the Apple toolchain probes see it. Undefined keeps the link
- * unreadable, which sends every read down the `xcode-select -p` fallback a case already scripts; a
- * case that owns a link points it at a path it created.
+ * The host's default selection link as the toolchain identity sees it. Undefined keeps the link
+ * unreadable; a case that owns a link points it at a path it created.
  */
 export const appleSelectionLink: { target: string | undefined } = { target: undefined };
 
 /**
  * Hides the selection link of the machine running the suite, so a case decides the host's selection
- * rather than inheriting it. The real host has one, and where it points is not this suite's business.
+ * rather than inheriting it.
  */
 export function stubAppleSelectionLink(): void {
   const realRealpathSync = fs.realpathSync;
+  const realReadlinkSync = fs.readlinkSync;
+  const absent = () =>
+    Object.assign(new Error('ENOENT: no such file or directory'), { code: 'ENOENT' });
   beforeEach(() => {
     appleSelectionLink.target = undefined;
+    vi.spyOn(fs, 'readlinkSync').mockImplementation(((target: fs.PathLike) => {
+      if (String(target) !== XCODE_SELECT_LINK_PATH) return realReadlinkSync(target);
+      if (appleSelectionLink.target === undefined) throw absent();
+      return appleSelectionLink.target;
+    }) as typeof fs.readlinkSync);
     vi.spyOn(fs, 'realpathSync').mockImplementation((target: fs.PathLike) => {
       if (String(target) !== XCODE_SELECT_LINK_PATH) return realRealpathSync(target);
-      if (appleSelectionLink.target === undefined) {
-        throw Object.assign(new Error('ENOENT: no such file or directory'), { code: 'ENOENT' });
-      }
+      if (appleSelectionLink.target === undefined) throw absent();
       return realRealpathSync(appleSelectionLink.target);
     });
   });
 }
-
-/**
- * The fingerprint budget's clock, advanced only by an exec that blocks for the whole timeout it was
- * handed. Without it a mock that throws immediately proves nothing about a deadline: no time passes,
- * so every budget looks untouched however many attempts run.
- */
-export function installToolchainProbeClock(): { nowMs: number } {
-  const clock = { nowMs: 0 };
-  appleRunnerTestHost.update({
-    deadlineFromTimeoutMs: (timeoutMs: number) => {
-      const startedAtMs = clock.nowMs;
-      const expiresAtMs = startedAtMs + Math.max(0, timeoutMs);
-      return {
-        remainingMs: () => Math.max(0, expiresAtMs - clock.nowMs),
-        elapsedMs: () => Math.max(0, clock.nowMs - startedAtMs),
-        isExpired: () => expiresAtMs - clock.nowMs <= 0,
-      };
-    },
-  });
-  return clock;
-}
-
-/** An exec that blocked for its whole timeout and was then killed, as the exec layer reports it. */
-export function blockForWholeTimeout(
-  clock: { nowMs: number },
-  command: string,
-  args: readonly string[],
-  options: ExecOptions,
-): AppError {
-  const timeoutMs = options.timeoutMs ?? 0;
-  clock.nowMs += timeoutMs;
-  return new AppError('COMMAND_FAILED', `${command} timed out after ${timeoutMs}ms`, {
-    cmd: command,
-    args,
-    timeoutMs,
-  });
-}
-
-/**
- * The execs that read the toolchain itself, as opposed to the `xcode-select -p` a read runs to
- * decide which fingerprint entry it belongs to. A case that says a cache hit must not re-read the
- * toolchain says nothing about that one exec, so it asserts through this.
- */
-export function valueProbeCommands(calls: readonly (readonly unknown[])[]): string[] {
-  return calls
-    .filter(([command]) => command !== 'xcode-select')
-    .map(([command]) => String(command));
-}

packages/platform-apple/src/runner/__tests__/host-setup.ts

@@ -6,6 +6,7 @@ import {
   loadAppleRunnerHost,
 } from '../test-host.ts';
 import { setAppleRunnerHostEnvironment } from '../../core/runner-owner-state.ts';
+import { stubbedAppleDeveloperDir } from './apple-toolchain-fixtures.ts';
 
 // The apple-runner vitest project runs the package's own suites, which cannot
 // import root utilities directly (R11). This setup installs the real root host
@@ -19,6 +20,9 @@ bindTestCommandHost();
 
 beforeEach(() => {
   appleRunnerTestHost.reset();
+  // The runner cache key reads the selected Xcode's files. A suite reads a stubbed Xcode rather than
+  // the host's, which CI hosts outside macOS do not have; a case that selects one overrides this.
+  appleRunnerTestHost.update({ commandDeveloperDir: stubbedAppleDeveloperDir });
   // Runner configuration reads take the runtime's host (#3794). A package suite runs outside a
   // composed runtime, so it binds the live process host the command host above carries.
   setAppleRunnerHostEnvironment({ env: process.env, cwd: process.cwd() });

packages/platform-apple/src/runner/__tests__/runner-adoption.test.ts

@@ -3,7 +3,6 @@ import path from 'node:path';
 import { afterEach, beforeEach, expect, test, vi } from 'vitest';
 import type { DeviceInfo } from '@agent-device/kernel/device';
 import { createRequestCanceledError, isRequestCanceledError } from '@agent-device/kernel/errors';
-import { resetAllProcessMemosForTests } from '@agent-device/kernel/ttl-memo';
 import {
   buildDetachedRunnerLease,
   buildRunnerLease,
@@ -20,7 +19,6 @@ import {
 import { resolveRunnerCacheKey } from '../runner-cache-metadata.ts';
 import { appleRunnerTestHost } from '../test-host.ts';
 import type { DiagnosticEventInput } from '@agent-device/host-kit/diagnostics';
-import { appleToolchainProbeResult } from './apple-toolchain-fixtures.ts';
 import { mkdtempForTestSync } from './tmp-dir.ts';
 
 vi.mock('../runner-transport.ts', async (importOriginal) => {
@@ -269,9 +267,8 @@ test('the adopted runner keeps the log file its predecessor opened for it', asyn
   expect(restamped.runnerLogPath).toBe(runnerLogPath);
 });
 
-test('a request canceled during the fingerprint probe fails adoption instead of skipping it, cold or with the fingerprint cache warm', async () => {
-  // The fingerprint check runs the same blocking toolchain probes a fresh
-  // startup would, and it is handed the request's signal. A cancellation from
+test('a request canceled before the fingerprint check fails adoption instead of skipping it', async () => {
+  // The fingerprint check is handed the request's signal. A cancellation from
   // there is not an unresolvable derived path: swallowing it would let startup
   // walk on past a client that is already gone (#2422).
   const lease = writeStaleLease();
@@ -292,18 +289,12 @@ test('a request canceled during the fingerprint probe fails adoption instead of
   expect(mockSendRunnerCommandOnce).not.toHaveBeenCalled();
 
   // The mock above only proves adoption propagates whatever the fingerprint
-  // call throws. Exercise the real production code path too: warm the actual
-  // toolchain fingerprint memo (the module-level cache in
-  // runner-cache-metadata.ts is a singleton the mock above never touches),
-  // then adopt again with an already-aborted signal. Before the fix,
-  // requireRunnerToolchainFingerprint returned the memoized value without
-  // checking the signal, so a cache hit let adoption go on to probe uptime
-  // and write the lease for an already-canceled request (#2422 round 4).
-  resetAllProcessMemosForTests();
+  // call throws. Exercise the real production code path too: with an
+  // already-aborted signal, the real fingerprint read cancels before it reads
+  // anything, so adoption never probes uptime or writes the lease (#2422).
   const { resolveExpectedRunnerCacheMetadata: actualResolveExpectedRunnerCacheMetadata } =
     await vi.importActual<typeof import('../runner-xctestrun.ts')>('../runner-xctestrun.ts');
-  appleRunnerTestHost.update({ runCmdSync: vi.fn(appleToolchainProbeResult) });
-  actualResolveExpectedRunnerCacheMetadata(simulator);
+  appleRunnerTestHost.update({});
 
   const warmRequest = new AbortController();
   warmRequest.abort();

packages/platform-apple/src/runner/__tests__/runner-artifact-build-location.test.ts

@@ -6,7 +6,6 @@ import { resetAllProcessMemosForTests } from '@agent-device/kernel/ttl-memo';
 import { appleRunnerTestHost } from '../test-host.ts';
 import type { ExecResult } from '@agent-device/host-kit/command';
 import { ensureXctestrunArtifact } from '../runner-xctestrun.ts';
-import { appleToolchainProbeResult } from './apple-toolchain-fixtures.ts';
 import { IOS_SIMULATOR } from './device-fixtures.ts';
 import { seedRunnerProductBundle } from './runner-xctestrun.fixtures.ts';
 import { mkdtempForTestSync } from './tmp-dir.ts';
@@ -31,7 +30,6 @@ beforeEach(() => {
   process.env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH = derived;
   runCmdStreaming.mockReset().mockImplementation(xcodebuildHonoringCustomBuildLocation);
   appleRunnerTestHost.update({
-    runCmdSync: vi.fn().mockImplementation(appleToolchainProbeResult),
     runCmdStreaming,
     findProjectRoot: () => projectRoot,
     readVersion: () => '0.0.0-test',