From 037a424750298cb2baa0cd2da1913c521b65cd4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 21:09:16 +0200 Subject: [PATCH 1/2] ci(macos): run the darwin-only Vitest files on a lane that reaches them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `.github/workflows/macos.yml:168` is the only Vitest invocation in the repository that runs on macOS. It was a hand-written two-file list, so a test guarded on `process.platform === 'darwin'` had no lane that could execute it: the sole job that collects `packages/*/src/**/*.test.ts` is the `unit-ci` gate on `ubuntu-latest`, where the whole file skips. Two files were dying that way, six tests between them: - `packages/platform-apple/src/foldable/simulator-hid-native.test.ts` — the only consumer of the `samples` column of `contracts/fixtures/fold-keyframes.json`. The ObjC interpolation in `apple/fold-helper/Fold.m` is the thing those samples check, and the TypeScript twin (`device-rotation-fold-input.test.ts`) reads only `valid` and `keyframesJson`. Proven: nudging one `angle` from 160 to 163 reddens this file ("opening reversal at 1667ms") while the TS legs that Linux CI actually runs report 14 passed. The column was unguarded on every lane. - `packages/capture-kit/src/recording/__tests__/recording-scripts.test.ts` — `swiftc -typecheck` over `apple/runner/AgentDeviceRunner/RecordingScripts`. Both join the existing conditional step rather than a new unconditional one, because `selectAppleBridgeProof` (`scripts/apple-ci-impact.ts:43`) already fires on any change under `apple/` or `packages/platform-apple/src/` — which is exactly the input set of both tests — so they run when their subject moved and cost nothing when it did not. The step is renamed to say what it now covers. All four files pass together here: 58 tests. One gap left on purpose: editing the recording-scripts test file alone does not trigger the step, since `packages/capture-kit/` is outside the classifier's prefixes. Widening `selectAppleBridgeProof` is affected-check model territory with its own tests, not a workflow edit; noted rather than folded in. --- .github/workflows/macos.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 8093217e6c..9d683a3c02 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -161,13 +161,15 @@ jobs: git fetch origin "$BASE_SHA" --depth=1 || echo 'Base fetch failed; bridge proof selection will fail open.' node --experimental-strip-types scripts/apple-ci-impact.ts bridge - - name: Verify clean-installed Simulator snapshot bridge preparation and the fold-helper -Werror gate + - name: Verify the darwin-only native source proofs (snapshot bridge, fold-helper -Werror, recording scripts) if: github.event_name == 'pull_request' && steps.bridge-impact.outputs.run != 'false' run: | pnpm build pnpm exec vitest run \ packages/platform-apple/src/snapshot-source/native-runtime.test.ts \ - packages/platform-apple/src/foldable/fold-helper-cache.test.ts + packages/platform-apple/src/foldable/fold-helper-cache.test.ts \ + packages/platform-apple/src/foldable/simulator-hid-native.test.ts \ + packages/capture-kit/src/recording/__tests__/recording-scripts.test.ts pnpm check:package -- --verify-snapshot-bridge-preparation - name: Upload macOS artifacts From 115021a4f702eb9d591dde00aac3d97490290db5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 25 Sep 2026 08:00:25 +0200 Subject: [PATCH 2/2] test(ci): order the macOS proof by command, and move swiftc out of test wall time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both failures on this PR traced to my own diff. `scripts/__tests__/apple-ci-impact.test.ts` pinned the literal step title `- name: Verify clean-installed Simulator snapshot bridge preparation`, which I renamed, so `indexOf` returned -1 and the Coverage job died on "expected -1 to be greater than 6134". Patching the string would just re-pin the same prose, so the assertion moved to what the steps actually run: the replay step is found by `with.gate === 'replay-macos'`, the clean-install proof by the `--verify-snapshot-bridge-preparation` flag it passes (unique in the file, and named by `scripts/check-package.ts:42`). The lookup walks jobs in file order through the `yaml` parser this test already imports, so a reindented `run:` block or a multi-line `with:` no longer reads as a sequencing change. Proven non-vacuous: retagging the replay step so the proof no longer follows it fails with "expected -1 to be greater than -1". `recording-scripts.test.ts` called `assertSwiftScriptTypechecks` from inside `test()`, so the 7.97s `swiftc` launch landed on the per-case clock, blew through 2x the `packages/**` budget, and aborted the step before `pnpm check:package` ran. The three compiles now run in one `beforeAll`, following the note at `fold-helper-cache.test.ts:164` — one SDK probe, both shared-source compiles paid once instead of per case. The cases drop to 1ms/0ms/0ms while the `describe.skipIf` keeps them darwin-only, which also removes the per-test `t.skip` branches. Detection is unchanged and was checked three ways against real sources: a type error in `recording-inspect.swift` fails the inspect case alone; one in `recording-overlay.swift` fails the overlay case alone; one in the shared `RecordingExportSupport.swift` fails exactly the two cases that compile it. All fixtures reverted. --- .../__tests__/recording-scripts.test.ts | 123 +++++++++--------- scripts/__tests__/apple-ci-impact.test.ts | 25 +++- 2 files changed, 80 insertions(+), 68 deletions(-) diff --git a/packages/capture-kit/src/recording/__tests__/recording-scripts.test.ts b/packages/capture-kit/src/recording/__tests__/recording-scripts.test.ts index 466627d398..50b7e8af63 100644 --- a/packages/capture-kit/src/recording/__tests__/recording-scripts.test.ts +++ b/packages/capture-kit/src/recording/__tests__/recording-scripts.test.ts @@ -1,4 +1,4 @@ -import { beforeAll, test } from 'vitest'; +import { beforeAll, describe, test } from 'vitest'; import assert from 'node:assert/strict'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -15,78 +15,71 @@ const recordingScriptsDir = path.resolve( ); const recordingTestSupportDir = path.resolve(__dirname, '../../../../../test/integration/support'); const SWIFT_TYPECHECK_TIMEOUT_MS = 60_000; -let swiftCompilerPath = 'swiftc'; -let swiftSdkPath = ''; -async function assertSwiftScriptTypechecks( - scriptPath: string, - extraSourcePaths: string[] = [], -): Promise { - const result = await runCmd( - swiftCompilerPath, - ['-sdk', swiftSdkPath, '-typecheck', scriptPath, ...extraSourcePaths], - { - allowFailure: true, - }, - ); - assert.equal( - result.exitCode, - 0, - `${path.basename(scriptPath)} should typecheck\n${result.stderr}`, - ); -} +type TypecheckOutcome = { exitCode: number; stderr: string; source: string }; -beforeAll(async () => { - if (process.platform !== 'darwin') return; - const [compilerResult, sdkResult] = await Promise.all([ - runCmd('xcrun', ['--find', 'swiftc']), - runCmd('xcrun', ['--show-sdk-path', '--sdk', 'macosx']), - ]); - swiftCompilerPath = compilerResult.stdout.trim(); - swiftSdkPath = sdkResult.stdout.trim(); -}); +// The three `swiftc -typecheck` invocations run in beforeAll, not in the test bodies: each is a +// real compiler launch, and the unit slow-test gate budgets `packages/**` test cases far below one +// compile (see the budget note in docs/agents/testing.md). The snapshot-bridge and fold-helper +// gates in platform-apple keep their native compiles out of test-case wall time the same way. +// One beforeAll covers all three so the SDK probe and the two shared-source compiles are paid once +// per file rather than once per case. +describe.skipIf(process.platform !== 'darwin')('recording Swift scripts typecheck', () => { + let outcomes: TypecheckOutcome[] = []; -test( - 'recording inspect Swift script typechecks', - async (t) => { - if (process.platform !== 'darwin') { - t.skip('Swift recording scripts are only validated on macOS'); - } + beforeAll( + async () => { + const [compiler, sdk] = await Promise.all([ + runCmd('xcrun', ['--find', 'swiftc']), + runCmd('xcrun', ['--show-sdk-path', '--sdk', 'macosx']), + ]); + const swiftCompilerPath = compiler.stdout.trim() || 'swiftc'; + const swiftSdkPath = sdk.stdout.trim(); + const sharedSupport = path.join(recordingScriptsDir, 'RecordingExportSupport.swift'); + const targets: Array<{ source: string; extraSources: string[] }> = [ + { source: path.join(recordingTestSupportDir, 'recording-inspect.swift'), extraSources: [] }, + { + source: path.join(recordingScriptsDir, 'recording-overlay.swift'), + extraSources: [sharedSupport], + }, + { + source: path.join(recordingScriptsDir, 'recording-frames.swift'), + extraSources: [sharedSupport], + }, + ]; - await assertSwiftScriptTypechecks( - path.join(recordingTestSupportDir, 'recording-inspect.swift'), - ); - }, - SWIFT_TYPECHECK_TIMEOUT_MS, -); + outcomes = await Promise.all( + targets.map(async ({ source, extraSources }): Promise => { + const result = await runCmd( + swiftCompilerPath, + ['-sdk', swiftSdkPath, '-typecheck', source, ...extraSources], + { allowFailure: true, timeoutMs: SWIFT_TYPECHECK_TIMEOUT_MS }, + ); + return { source, exitCode: result.exitCode, stderr: result.stderr }; + }), + ); + }, + SWIFT_TYPECHECK_TIMEOUT_MS * 3 + 30_000, + ); -test( - 'recording overlay Swift script typechecks', - async (t) => { - if (process.platform !== 'darwin') { - t.skip('Swift recording scripts are only validated on macOS'); - } + test('recording inspect Swift script typechecks', () => { + assertTypechecked(outcomes, 'recording-inspect.swift'); + }); - await assertSwiftScriptTypechecks(path.join(recordingScriptsDir, 'recording-overlay.swift'), [ - path.join(recordingScriptsDir, 'RecordingExportSupport.swift'), - ]); - }, - SWIFT_TYPECHECK_TIMEOUT_MS, -); + test('recording overlay Swift script typechecks', () => { + assertTypechecked(outcomes, 'recording-overlay.swift'); + }); -test( - 'recording frames Swift script typechecks', - async (t) => { - if (process.platform !== 'darwin') { - t.skip('Swift recording scripts are only validated on macOS'); - } + test('recording frames Swift script typechecks', () => { + assertTypechecked(outcomes, 'recording-frames.swift'); + }); +}); - await assertSwiftScriptTypechecks(path.join(recordingScriptsDir, 'recording-frames.swift'), [ - path.join(recordingScriptsDir, 'RecordingExportSupport.swift'), - ]); - }, - SWIFT_TYPECHECK_TIMEOUT_MS, -); +function assertTypechecked(outcomes: readonly TypecheckOutcome[], basename: string): void { + const outcome = outcomes.find((entry) => path.basename(entry.source) === basename); + assert.ok(outcome, `${basename} was never typechecked`); + assert.equal(outcome.exitCode, 0, `${basename} should typecheck\n${outcome.stderr}`); +} test('recording overlays are explicitly unsupported on non-macOS hosts', () => { assert.equal( diff --git a/scripts/__tests__/apple-ci-impact.test.ts b/scripts/__tests__/apple-ci-impact.test.ts index ea0834195d..7fa5a1a86d 100644 --- a/scripts/__tests__/apple-ci-impact.test.ts +++ b/scripts/__tests__/apple-ci-impact.test.ts @@ -19,6 +19,21 @@ function cacheInputs(action: string): string[] { ); } +type WorkflowStep = { run?: string; with?: { gate?: string } }; + +/** + * Index of the first workflow step matching a predicate, counted across every job in file order, + * or -1. Steps are located by the command they run or the gate they invoke rather than by their + * title, and through the same YAML parser the assertions below already use, so a step renamed for + * scope reasons — or a `run:` block reindented — is not mistaken for a sequencing regression. + */ +function workflowStepIndex(workflow: string, matches: (step: WorkflowStep) => boolean): number { + const doc = parse(workflow) as { jobs?: Record }; + return Object.values(doc.jobs ?? {}) + .flatMap((job) => job.steps ?? []) + .findIndex(matches); +} + test('every runner build-cache input triggers the PR XCTest lane', () => { const action = fs.readFileSync( path.join(repoRoot, '.github/actions/setup-apple-runner-build/action.yml'), @@ -58,10 +73,14 @@ test('the PR workflow applies the impact decision to the XCTest step', () => { }); test('macOS clean-install proof follows live UI replay', () => { + // Ordered by what each step runs, not by its title: the clean-install proof must not fire + // before the replay that can raise local-network permission UI, and a step renamed for scope + // reasons is not a sequencing regression. const workflow = fs.readFileSync(path.join(repoRoot, '.github/workflows/macos.yml'), 'utf8'); - const replay = workflow.indexOf('- name: Run macOS integration test'); - const proof = workflow.indexOf( - '- name: Verify clean-installed Simulator snapshot bridge preparation', + const replay = workflowStepIndex(workflow, (step) => step.with?.gate === 'replay-macos'); + const proof = workflowStepIndex( + workflow, + (step) => step.run?.includes('--verify-snapshot-bridge-preparation') ?? false, ); expect(replay).toBeGreaterThan(-1); expect(proof).toBeGreaterThan(replay);