diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55457c9dbf..c449060c9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -531,23 +531,19 @@ jobs: - name: CLI help smoke run: bun run src/cli/index.ts help - # macOS is focused on dev pushes and relevant pull requests, while main, - # preview, and dispatch retain the full unsharded control. This keeps Darwin - # process coverage on the fast path without dropping the release control. + # Dev pushes and relevant pull requests keep a focused Darwin/process lane. + # Release refs and main-targeting changes use the timing-balanced full-suite + # matrix below; nightly-macos retains the unsharded scheduler/load control. platform-macos: name: macos needs: changes if: >- - (github.event_name != 'pull_request' && github.event_name != 'merge_group') || + (github.event_name == 'push' && github.ref == 'refs/heads/dev') || (github.event_name == 'pull_request' && github.base_ref == 'main') || (github.event_name == 'merge_group' && github.event.merge_group.base_ref == 'refs/heads/main') || - needs.changes.outputs.macos == 'true' + ((github.event_name == 'pull_request' || github.event_name == 'merge_group') && needs.changes.outputs.macos == 'true') runs-on: macos-latest - # The unsharded control for the sharded Linux lane: the only place the whole - # suite runs in one pool, so it is the place that catches what sharding - # hides. The flakes it keeps surfacing are timing, not logic, and the fix - # is the tests, not a fourth lane. - timeout-minutes: 30 + timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -619,15 +615,84 @@ jobs: tests/storage/storage-worker-teardown-isolate.test.ts \ tests/ci-workflows/test-runner.test.ts - - name: Full macOS suite - if: >- - github.event_name == 'workflow_dispatch' || - (github.event_name == 'pull_request' && github.base_ref == 'main') || - (github.event_name == 'merge_group' && - github.event.merge_group.base_ref == 'refs/heads/main') || - (github.event_name == 'push' && - (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/preview')) - run: bash scripts/ci/run-bun-with-crash-retry.sh -- bun scripts/test.ts --timeout 60000 + - name: CLI help smoke + run: bun run src/cli/index.ts help + + # Full macOS coverage is split by measured file duration. Every test remains + # in exactly one platform lane; load-sensitive files run one per fresh Bun + # process, matching the full-suite planner. The daily nightly job deliberately + # stays unsharded so it can still expose whole-pool scheduling interactions. + platform-macos-full: + name: macos full ${{ matrix.shard }}/2 + needs: changes + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && github.base_ref == 'main') || + (github.event_name == 'merge_group' && + github.event.merge_group.base_ref == 'refs/heads/main') || + (github.event_name == 'push' && + (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/preview')) + runs-on: macos-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + shard: [1, 2] + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + fetch-tags: true + + - name: Setup project Bun + uses: ./.github/actions/setup-project-bun + + - name: Install dependencies + run: | + bun install --frozen-lockfile + cd gui + bun install --frozen-lockfile + + - name: Build GUI + run: | + cd gui + bun run build + + - name: Restore canonical test timings + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: .bun-timings.json + key: ocx-test-timings-dev-${{ github.sha }} + restore-keys: | + ocx-test-timings-dev- + + - name: Validate restored timing data + if: hashFiles('.bun-timings.json') != '' + run: bun scripts/ci/validate-timings.ts .bun-timings.json --discard-invalid + + - name: Full macOS shard with isolated load-sensitive files + shell: bash + env: + TEST_SHARD: ${{ matrix.shard }}/2 + run: | + set -euo pipefail + main_list="$(mktemp -t ocx-macos-main.XXXXXX)" + serial_list="$(mktemp -t ocx-macos-serial.XXXXXX)" + cleanup() { rm -f -- "$main_list" "$serial_list"; } + trap cleanup EXIT + bun scripts/ci/test-lanes.ts --lane platform-main --timings .bun-timings.json --shard "$TEST_SHARD" > "$main_list" + bun scripts/ci/test-lanes.ts --lane platform-serial --timings .bun-timings.json --shard "$TEST_SHARD" > "$serial_list" + main_files=() + while IFS= read -r file; do main_files+=("$file"); done < "$main_list" + test "${#main_files[@]}" -gt 0 + bash scripts/ci/run-bun-with-crash-retry.sh -- bun scripts/test.ts --isolate --timeout 60000 "${main_files[@]}" + serial_files=() + while IFS= read -r file; do serial_files+=("$file"); done < "$serial_list" + test "${#serial_files[@]}" -gt 0 + for file in "${serial_files[@]}"; do + bash scripts/ci/run-bun-with-crash-retry.sh -- bun scripts/test.ts --isolate --parallel=1 --timeout 60000 "$file" + done - name: CLI help smoke run: bun run src/cli/index.ts help @@ -944,7 +1009,7 @@ jobs: name: ci if: always() # Every producer, including the ones that only feed other jobs. - needs: [changes, test, storage-policy, api-usage, serial-load-sensitive, publish-test-timings, gates, platform-macos, platform-windows, keyring-smoke, npm-global-smoke] + needs: [changes, test, storage-policy, api-usage, serial-load-sensitive, publish-test-timings, gates, platform-macos, platform-macos-full, platform-windows, keyring-smoke, npm-global-smoke] runs-on: ubuntu-latest timeout-minutes: 5 steps: diff --git a/scripts/ci/test-lanes.ts b/scripts/ci/test-lanes.ts index 47130594a7..68932b770e 100644 --- a/scripts/ci/test-lanes.ts +++ b/scripts/ci/test-lanes.ts @@ -113,7 +113,15 @@ export const DEDICATED_TEST_FILES = [ export const STORAGE_TEST_FILES = DEDICATED_TEST_FILES.slice(0, 6); export const API_TEST_FILES = ["tests/server/api-usage.test.ts"] as const; -const laneNames = ["general", "serial", "dedicated", "dedicated-storage", "dedicated-api"] as const; +const laneNames = [ + "general", + "serial", + "dedicated", + "dedicated-storage", + "dedicated-api", + "platform-main", + "platform-serial", +] as const; export type TestLane = (typeof laneNames)[number]; type DirectoryReader = (directory: string) => Dirent[]; @@ -159,7 +167,7 @@ export function validateLaneManifest(inventory: string[]): { const normalized = [...inventory].sort(); if (new Set(normalized).size !== normalized.length) throw new Error("lane manifest contains duplicate paths"); for (const path of normalized) { - if (!path.startsWith("tests/") || path.includes("..")) { + if (!/^tests\/[A-Za-z0-9._/-]+$/.test(path) || path.includes("..")) { throw new Error(`lane manifest contains path outside tests: ${path}`); } } @@ -192,6 +200,10 @@ export function laneFiles(lane: TestLane, root = process.cwd()): string[] { const lanes = validateLaneManifest(discoverTestFiles(root)); if (lane === "dedicated-storage") return [...STORAGE_TEST_FILES]; if (lane === "dedicated-api") return [...API_TEST_FILES]; + // Match the full-suite planner: both serial and dedicated files receive + // their own process on macOS, outside the general worker pool. + if (lane === "platform-main") return lanes.general; + if (lane === "platform-serial") return [...lanes.serial, ...lanes.dedicated]; return lanes[lane]; } @@ -240,6 +252,6 @@ if (import.meta.main) { for (const path of allocateFilesByTiming(files, count, durations)[index - 1]!) console.log(path); } else for (const path of files) console.log(path); } else { - throw new Error("usage: --lane general|serial|dedicated|dedicated-storage|dedicated-api"); + throw new Error(`usage: --lane ${laneNames.join("|")}`); } } diff --git a/tests/ci-workflows/ci-workflows.test.ts b/tests/ci-workflows/ci-workflows.test.ts index c0dc889fe9..76a92928b6 100644 --- a/tests/ci-workflows/ci-workflows.test.ts +++ b/tests/ci-workflows/ci-workflows.test.ts @@ -141,7 +141,8 @@ describe("GitHub Actions hardening", () => { expect(ci.jobs?.["select-windows-runner"]).toBeUndefined(); expect(ci.jobs?.test?.["timeout-minutes"]).toBe(15); expect(ci.jobs?.gates?.["timeout-minutes"]).toBe(15); - expect(ci.jobs?.["platform-macos"]?.["timeout-minutes"]).toBe(30); + expect(ci.jobs?.["platform-macos"]?.["timeout-minutes"]).toBe(15); + expect(ci.jobs?.["platform-macos-full"]?.["timeout-minutes"]).toBe(20); expect(ci.jobs?.["macos-control"]).toBeUndefined(); // Higher than the Linux shards on purpose: at 15 the Windows leg cancelled a // shard mid-suite, which reports as neither pass nor fail (#2152). @@ -215,7 +216,7 @@ describe("GitHub Actions hardening", () => { // how the first cut of that test shipped, so pin the flag rather than trusting a // comment. Asserted per job so a future edit cannot drop it from one leg while // the other still carries it. - for (const jobName of ["test", "platform-macos", "platform-windows"]) { + for (const jobName of ["test", "platform-macos", "platform-macos-full", "platform-windows"]) { const steps = (ci.jobs?.[jobName] as { steps?: Array<{ uses?: string; with?: Record }> })?.steps ?? []; const checkout = steps.find(step => typeof step.uses === "string" && step.uses.includes("actions/checkout")); expect(`${jobName}:${String(checkout?.with?.["fetch-tags"])}`).toBe(`${jobName}:true`); @@ -275,7 +276,8 @@ describe("GitHub Actions hardening", () => { expect(hasExactShellCommand(gatesGuiRun, "cd gui && bun test --isolate tests")).toBe(true); expect(hasExactShellCommand(gatesGuiRun, "cd gui && bun test tests")).toBe(false); - // macOS is focused on dev and relevant PRs; main/preview keep full control. + // macOS is focused on dev and relevant PRs; promotion events use two + // timing-balanced full-suite runners while nightly keeps the pool control. const macosSteps = (ci.jobs?.["platform-macos"] as { steps?: { name?: string; if?: string; run?: string }[]; })?.steps ?? []; @@ -285,8 +287,23 @@ describe("GitHub Actions hardening", () => { expect(macosSteps.some(step => step.run?.includes("--shard"))).toBe(false); const focusedMacos = macosSteps.find(step => step.name === "Focused Darwin/process lifecycle tests"); expect(focusedMacos?.run).toContain("tests/codex-integration/codex-prompt-text-probe.test.ts"); - const fullMacos = macosSteps.find(step => step.name === "Full macOS suite"); - expect(fullMacos?.if).toContain("github.event_name == 'pull_request' && github.base_ref == 'main'"); + expect(macosSteps.find(step => step.name === "Full macOS suite")).toBeUndefined(); + const macosFull = ci.jobs?.["platform-macos-full"] as { + if?: string; + name?: string; + strategy?: { matrix?: { shard?: number[] } }; + steps?: Array<{ name?: string; env?: Record; run?: string }>; + } | undefined; + expect(macosFull?.strategy?.matrix?.shard).toEqual([1, 2]); + expect(macosFull?.name).toBe("macos full ${{ matrix.shard }}/2"); + expect(macosFull?.if).toContain("github.event_name == 'pull_request' && github.base_ref == 'main'"); + const macosFullStep = macosFull?.steps?.find(step => + step.name === "Full macOS shard with isolated load-sensitive files" + ); + expect(macosFullStep?.env?.TEST_SHARD).toBe("${{ matrix.shard }}/2"); + expect(macosFullStep?.run).toContain("--lane platform-main"); + expect(macosFullStep?.run).toContain("--lane platform-serial"); + expect(macosFullStep?.run).toContain("--parallel=1"); const pathPolicy = Bun.YAML.parse(await readText(".github/policies/ci-paths.yml")) as { macos?: string[]; swift?: string[]; @@ -335,7 +352,7 @@ describe("GitHub Actions hardening", () => { expect(crashRetry).not.toContain("while true"); expect((ci.jobs?.["platform-macos"] as { needs?: string; if?: string })?.needs).toBe("changes"); expect((ci.jobs?.["platform-macos"] as { if?: string })?.if) - .toBe("(github.event_name != 'pull_request' && github.event_name != 'merge_group') || (github.event_name == 'pull_request' && github.base_ref == 'main') || (github.event_name == 'merge_group' && github.event.merge_group.base_ref == 'refs/heads/main') || needs.changes.outputs.macos == 'true'"); + .toBe("(github.event_name == 'push' && github.ref == 'refs/heads/dev') || (github.event_name == 'pull_request' && github.base_ref == 'main') || (github.event_name == 'merge_group' && github.event.merge_group.base_ref == 'refs/heads/main') || ((github.event_name == 'pull_request' || github.event_name == 'merge_group') && needs.changes.outputs.macos == 'true')"); // Windows is required for every integration push and CI-relevant PR. The // changes dependency keeps documentation-only PRs cheap without letting a @@ -436,7 +453,7 @@ describe("GitHub Actions hardening", () => { // accident, because the same job also ran the GUI build — splitting the suite // away from the gates removed that coincidence, and the shards went red on a // pull request before this pin existed. - for (const jobName of ["test", "platform-macos", "platform-windows"]) { + for (const jobName of ["test", "platform-macos", "platform-macos-full", "platform-windows"]) { const steps = (ci.jobs?.[jobName] as { steps?: { if?: string; run?: string }[] })?.steps ?? []; const build = steps.find(step => step.run?.includes("bun run build")); expect(`${jobName}:${build === undefined}`).toBe(`${jobName}:false`); @@ -678,7 +695,12 @@ describe("GitHub Actions hardening", () => { } const macosJob = ci.jobs?.["platform-macos"] as { needs?: string; if?: string } | undefined; expect(`platform-macos:${macosJob?.needs}`).toBe("platform-macos:changes"); - expect(`platform-macos:${macosJob?.if}`).toBe("platform-macos:(github.event_name != 'pull_request' && github.event_name != 'merge_group') || (github.event_name == 'pull_request' && github.base_ref == 'main') || (github.event_name == 'merge_group' && github.event.merge_group.base_ref == 'refs/heads/main') || needs.changes.outputs.macos == 'true'"); + expect(`platform-macos:${macosJob?.if}`).toBe("platform-macos:(github.event_name == 'push' && github.ref == 'refs/heads/dev') || (github.event_name == 'pull_request' && github.base_ref == 'main') || (github.event_name == 'merge_group' && github.event.merge_group.base_ref == 'refs/heads/main') || ((github.event_name == 'pull_request' || github.event_name == 'merge_group') && needs.changes.outputs.macos == 'true')"); + const macosFullJob = ci.jobs?.["platform-macos-full"] as { needs?: string; if?: string } | undefined; + expect(`platform-macos-full:${macosFullJob?.needs}`).toBe("platform-macos-full:changes"); + expect(macosFullJob?.if).toContain("github.event_name == 'workflow_dispatch'"); + expect(macosFullJob?.if).toContain("refs/heads/main"); + expect(macosFullJob?.if).toContain("refs/heads/preview"); }); test("cross-platform CI keeps the GUI lint and build gates", async () => { diff --git a/tests/ci-workflows/zz-ci-timing.test.ts b/tests/ci-workflows/zz-ci-timing.test.ts index e1742ec402..4feaf84668 100644 --- a/tests/ci-workflows/zz-ci-timing.test.ts +++ b/tests/ci-workflows/zz-ci-timing.test.ts @@ -63,6 +63,19 @@ describe("CI lane manifest", () => { test("never accepts a lane path outside tests", () => { expect(() => validateLaneManifest(["tests/good.test.ts", "src/not-a-test.ts"])) .toThrow(/outside tests/); + expect(() => validateLaneManifest(["tests/good.test.ts\n--shard=1/1"])) + .toThrow(/outside tests/); + }); + + test("platform lanes preserve full-suite coverage and singleton isolation", () => { + const inventory = discoverTestFiles(process.cwd()); + const main = laneFiles("platform-main", process.cwd()); + const serial = laneFiles("platform-serial", process.cwd()); + expect(new Set([...main, ...serial])).toEqual(new Set(inventory)); + expect(main.length + serial.length).toBe(inventory.length); + expect(new Set(main).intersection(new Set(serial)).size).toBe(0); + expect(serial).toEqual([...SERIAL_TEST_FILES, ...DEDICATED_TEST_FILES]); + for (const dedicated of DEDICATED_TEST_FILES) expect(main).not.toContain(dedicated); }); test("never drops a nested test that shares a quarantined basename", () => { @@ -214,6 +227,13 @@ test("only trusted dev shards publish the canonical timing cache", async () => { step.uses === "actions/cache/save@caa296126883cff596d87d8935842f9db880ef25", )).toBe(true); expect(ci.jobs["platform-macos"].steps?.some(step => step.uses?.startsWith("actions/cache/save@"))).toBe(false); + const macosFull = ci.jobs["platform-macos-full"] as typeof ci.jobs[string] & { + strategy?: { matrix?: { shard?: number[] } }; + }; + expect(macosFull.strategy?.matrix?.shard).toEqual([1, 2]); + expect(macosFull.steps?.some(step => step.uses?.startsWith("actions/cache/save@"))).toBe(false); + expect(macosFull.steps?.some(step => step.run?.includes("--lane platform-main"))).toBe(true); + expect(macosFull.steps?.some(step => step.run?.includes("--lane platform-serial"))).toBe(true); const windows = ci.jobs["platform-windows"]; const windowsRestore = windows.steps?.find(step => step.uses?.startsWith("actions/cache/restore@")); expect(windowsRestore?.with?.key).toContain("ocx-test-timings-dev-"); @@ -231,9 +251,7 @@ test("only trusted dev shards publish the canonical timing cache", async () => { expect(focused?.run).not.toContain("tests/adapters/google/aistudio-native-webkit.test.ts"); expect(focused?.run).toContain("tests/ci-workflows/test-runner.test.ts"); expect(focused?.run).toContain("tests/providers/cursor/cursor-native-exec-shell.test.ts"); - const full = ci.jobs["platform-macos"].steps?.find(step => step.name === "Full macOS suite"); - expect(full?.run).toContain("bun scripts/test.ts --timeout 60000"); - expect(full?.run).not.toContain("--update-timings"); + expect(ci.jobs["platform-macos"].steps?.find(step => step.name === "Full macOS suite")).toBeUndefined(); }); test("nightly macOS is timed at 08:17 UTC and names its timing file", async () => {