Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 85 additions & 20 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -531,23 +531,19 @@ jobs:
- name: CLI help smoke
run: bun run src/cli/index.ts help

# macOS is focused on dev pushes and relevant pull requests, while main,
# preview, and dispatch retain the full unsharded control. This keeps Darwin
# process coverage on the fast path without dropping the release control.
# Dev pushes and relevant pull requests keep a focused Darwin/process lane.
# Release refs and main-targeting changes use the timing-balanced full-suite
# matrix below; nightly-macos retains the unsharded scheduler/load control.
platform-macos:
name: macos
needs: changes
if: >-
(github.event_name != 'pull_request' && github.event_name != 'merge_group') ||
(github.event_name == 'push' && github.ref == 'refs/heads/dev') ||
(github.event_name == 'pull_request' && github.base_ref == 'main') ||
(github.event_name == 'merge_group' && github.event.merge_group.base_ref == 'refs/heads/main') ||
needs.changes.outputs.macos == 'true'
((github.event_name == 'pull_request' || github.event_name == 'merge_group') && needs.changes.outputs.macos == 'true')
runs-on: macos-latest
# The unsharded control for the sharded Linux lane: the only place the whole
# suite runs in one pool, so it is the place that catches what sharding
# hides. The flakes it keeps surfacing are timing, not logic, and the fix
# is the tests, not a fourth lane.
timeout-minutes: 30
timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
Expand Down Expand Up @@ -619,15 +615,84 @@ jobs:
tests/storage/storage-worker-teardown-isolate.test.ts \
tests/ci-workflows/test-runner.test.ts

- name: Full macOS suite
if: >-
github.event_name == 'workflow_dispatch' ||
(github.event_name == 'pull_request' && github.base_ref == 'main') ||
(github.event_name == 'merge_group' &&
github.event.merge_group.base_ref == 'refs/heads/main') ||
(github.event_name == 'push' &&
(github.ref == 'refs/heads/main' || github.ref == 'refs/heads/preview'))
run: bash scripts/ci/run-bun-with-crash-retry.sh -- bun scripts/test.ts --timeout 60000
- name: CLI help smoke
run: bun run src/cli/index.ts help

# Full macOS coverage is split by measured file duration. Every test remains
# in exactly one platform lane; load-sensitive files run one per fresh Bun
# process, matching the full-suite planner. The daily nightly job deliberately
# stays unsharded so it can still expose whole-pool scheduling interactions.
platform-macos-full:
name: macos full ${{ matrix.shard }}/2
needs: changes
if: >-
github.event_name == 'workflow_dispatch' ||
(github.event_name == 'pull_request' && github.base_ref == 'main') ||
(github.event_name == 'merge_group' &&
github.event.merge_group.base_ref == 'refs/heads/main') ||
(github.event_name == 'push' &&
(github.ref == 'refs/heads/main' || github.ref == 'refs/heads/preview'))
runs-on: macos-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
shard: [1, 2]
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
fetch-tags: true

- name: Setup project Bun
uses: ./.github/actions/setup-project-bun

- name: Install dependencies
run: |
bun install --frozen-lockfile
cd gui
bun install --frozen-lockfile

- name: Build GUI
run: |
cd gui
bun run build

- name: Restore canonical test timings
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
with:
path: .bun-timings.json
key: ocx-test-timings-dev-${{ github.sha }}
restore-keys: |
ocx-test-timings-dev-

- name: Validate restored timing data
if: hashFiles('.bun-timings.json') != ''
run: bun scripts/ci/validate-timings.ts .bun-timings.json --discard-invalid

- name: Full macOS shard with isolated load-sensitive files
shell: bash
env:
TEST_SHARD: ${{ matrix.shard }}/2
run: |
set -euo pipefail
main_list="$(mktemp -t ocx-macos-main.XXXXXX)"
serial_list="$(mktemp -t ocx-macos-serial.XXXXXX)"
cleanup() { rm -f -- "$main_list" "$serial_list"; }
trap cleanup EXIT
bun scripts/ci/test-lanes.ts --lane platform-main --timings .bun-timings.json --shard "$TEST_SHARD" > "$main_list"
bun scripts/ci/test-lanes.ts --lane platform-serial --timings .bun-timings.json --shard "$TEST_SHARD" > "$serial_list"
main_files=()
while IFS= read -r file; do main_files+=("$file"); done < "$main_list"
test "${#main_files[@]}" -gt 0
bash scripts/ci/run-bun-with-crash-retry.sh -- bun scripts/test.ts --isolate --timeout 60000 "${main_files[@]}"
serial_files=()
while IFS= read -r file; do serial_files+=("$file"); done < "$serial_list"
test "${#serial_files[@]}" -gt 0
for file in "${serial_files[@]}"; do
bash scripts/ci/run-bun-with-crash-retry.sh -- bun scripts/test.ts --isolate --parallel=1 --timeout 60000 "$file"
done

- name: CLI help smoke
run: bun run src/cli/index.ts help
Expand Down Expand Up @@ -944,7 +1009,7 @@ jobs:
name: ci
if: always()
# Every producer, including the ones that only feed other jobs.
needs: [changes, test, storage-policy, api-usage, serial-load-sensitive, publish-test-timings, gates, platform-macos, platform-windows, keyring-smoke, npm-global-smoke]
needs: [changes, test, storage-policy, api-usage, serial-load-sensitive, publish-test-timings, gates, platform-macos, platform-macos-full, platform-windows, keyring-smoke, npm-global-smoke]
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
Expand Down
18 changes: 15 additions & 3 deletions scripts/ci/test-lanes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,15 @@ export const DEDICATED_TEST_FILES = [
export const STORAGE_TEST_FILES = DEDICATED_TEST_FILES.slice(0, 6);
export const API_TEST_FILES = ["tests/server/api-usage.test.ts"] as const;

const laneNames = ["general", "serial", "dedicated", "dedicated-storage", "dedicated-api"] as const;
const laneNames = [
"general",
"serial",
"dedicated",
"dedicated-storage",
"dedicated-api",
"platform-main",
"platform-serial",
] as const;
export type TestLane = (typeof laneNames)[number];

type DirectoryReader = (directory: string) => Dirent[];
Expand Down Expand Up @@ -159,7 +167,7 @@ export function validateLaneManifest(inventory: string[]): {
const normalized = [...inventory].sort();
if (new Set(normalized).size !== normalized.length) throw new Error("lane manifest contains duplicate paths");
for (const path of normalized) {
if (!path.startsWith("tests/") || path.includes("..")) {
if (!/^tests\/[A-Za-z0-9._/-]+$/.test(path) || path.includes("..")) {
throw new Error(`lane manifest contains path outside tests: ${path}`);
}
}
Expand Down Expand Up @@ -192,6 +200,10 @@ export function laneFiles(lane: TestLane, root = process.cwd()): string[] {
const lanes = validateLaneManifest(discoverTestFiles(root));
if (lane === "dedicated-storage") return [...STORAGE_TEST_FILES];
if (lane === "dedicated-api") return [...API_TEST_FILES];
// Match the full-suite planner: both serial and dedicated files receive
// their own process on macOS, outside the general worker pool.
if (lane === "platform-main") return lanes.general;
if (lane === "platform-serial") return [...lanes.serial, ...lanes.dedicated];
return lanes[lane];
}

Expand Down Expand Up @@ -240,6 +252,6 @@ if (import.meta.main) {
for (const path of allocateFilesByTiming(files, count, durations)[index - 1]!) console.log(path);
} else for (const path of files) console.log(path);
} else {
throw new Error("usage: --lane general|serial|dedicated|dedicated-storage|dedicated-api");
throw new Error(`usage: --lane ${laneNames.join("|")}`);
}
}
38 changes: 30 additions & 8 deletions tests/ci-workflows/ci-workflows.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,8 @@ describe("GitHub Actions hardening", () => {
expect(ci.jobs?.["select-windows-runner"]).toBeUndefined();
expect(ci.jobs?.test?.["timeout-minutes"]).toBe(15);
expect(ci.jobs?.gates?.["timeout-minutes"]).toBe(15);
expect(ci.jobs?.["platform-macos"]?.["timeout-minutes"]).toBe(30);
expect(ci.jobs?.["platform-macos"]?.["timeout-minutes"]).toBe(15);
expect(ci.jobs?.["platform-macos-full"]?.["timeout-minutes"]).toBe(20);
expect(ci.jobs?.["macos-control"]).toBeUndefined();
// Higher than the Linux shards on purpose: at 15 the Windows leg cancelled a
// shard mid-suite, which reports as neither pass nor fail (#2152).
Expand Down Expand Up @@ -215,7 +216,7 @@ describe("GitHub Actions hardening", () => {
// how the first cut of that test shipped, so pin the flag rather than trusting a
// comment. Asserted per job so a future edit cannot drop it from one leg while
// the other still carries it.
for (const jobName of ["test", "platform-macos", "platform-windows"]) {
for (const jobName of ["test", "platform-macos", "platform-macos-full", "platform-windows"]) {
const steps = (ci.jobs?.[jobName] as { steps?: Array<{ uses?: string; with?: Record<string, unknown> }> })?.steps ?? [];
const checkout = steps.find(step => typeof step.uses === "string" && step.uses.includes("actions/checkout"));
expect(`${jobName}:${String(checkout?.with?.["fetch-tags"])}`).toBe(`${jobName}:true`);
Expand Down Expand Up @@ -275,7 +276,8 @@ describe("GitHub Actions hardening", () => {
expect(hasExactShellCommand(gatesGuiRun, "cd gui && bun test --isolate tests")).toBe(true);
expect(hasExactShellCommand(gatesGuiRun, "cd gui && bun test tests")).toBe(false);

// macOS is focused on dev and relevant PRs; main/preview keep full control.
// macOS is focused on dev and relevant PRs; promotion events use two
// timing-balanced full-suite runners while nightly keeps the pool control.
const macosSteps = (ci.jobs?.["platform-macos"] as {
steps?: { name?: string; if?: string; run?: string }[];
})?.steps ?? [];
Expand All @@ -285,8 +287,23 @@ describe("GitHub Actions hardening", () => {
expect(macosSteps.some(step => step.run?.includes("--shard"))).toBe(false);
const focusedMacos = macosSteps.find(step => step.name === "Focused Darwin/process lifecycle tests");
expect(focusedMacos?.run).toContain("tests/codex-integration/codex-prompt-text-probe.test.ts");
const fullMacos = macosSteps.find(step => step.name === "Full macOS suite");
expect(fullMacos?.if).toContain("github.event_name == 'pull_request' && github.base_ref == 'main'");
expect(macosSteps.find(step => step.name === "Full macOS suite")).toBeUndefined();
const macosFull = ci.jobs?.["platform-macos-full"] as {
if?: string;
name?: string;
strategy?: { matrix?: { shard?: number[] } };
steps?: Array<{ name?: string; env?: Record<string, string>; run?: string }>;
} | undefined;
expect(macosFull?.strategy?.matrix?.shard).toEqual([1, 2]);
expect(macosFull?.name).toBe("macos full ${{ matrix.shard }}/2");
expect(macosFull?.if).toContain("github.event_name == 'pull_request' && github.base_ref == 'main'");
const macosFullStep = macosFull?.steps?.find(step =>
step.name === "Full macOS shard with isolated load-sensitive files"
);
expect(macosFullStep?.env?.TEST_SHARD).toBe("${{ matrix.shard }}/2");
expect(macosFullStep?.run).toContain("--lane platform-main");
expect(macosFullStep?.run).toContain("--lane platform-serial");
expect(macosFullStep?.run).toContain("--parallel=1");
const pathPolicy = Bun.YAML.parse(await readText(".github/policies/ci-paths.yml")) as {
macos?: string[];
swift?: string[];
Expand Down Expand Up @@ -335,7 +352,7 @@ describe("GitHub Actions hardening", () => {
expect(crashRetry).not.toContain("while true");
expect((ci.jobs?.["platform-macos"] as { needs?: string; if?: string })?.needs).toBe("changes");
expect((ci.jobs?.["platform-macos"] as { if?: string })?.if)
.toBe("(github.event_name != 'pull_request' && github.event_name != 'merge_group') || (github.event_name == 'pull_request' && github.base_ref == 'main') || (github.event_name == 'merge_group' && github.event.merge_group.base_ref == 'refs/heads/main') || needs.changes.outputs.macos == 'true'");
.toBe("(github.event_name == 'push' && github.ref == 'refs/heads/dev') || (github.event_name == 'pull_request' && github.base_ref == 'main') || (github.event_name == 'merge_group' && github.event.merge_group.base_ref == 'refs/heads/main') || ((github.event_name == 'pull_request' || github.event_name == 'merge_group') && needs.changes.outputs.macos == 'true')");

// Windows is required for every integration push and CI-relevant PR. The
// changes dependency keeps documentation-only PRs cheap without letting a
Expand Down Expand Up @@ -436,7 +453,7 @@ describe("GitHub Actions hardening", () => {
// accident, because the same job also ran the GUI build — splitting the suite
// away from the gates removed that coincidence, and the shards went red on a
// pull request before this pin existed.
for (const jobName of ["test", "platform-macos", "platform-windows"]) {
for (const jobName of ["test", "platform-macos", "platform-macos-full", "platform-windows"]) {
const steps = (ci.jobs?.[jobName] as { steps?: { if?: string; run?: string }[] })?.steps ?? [];
const build = steps.find(step => step.run?.includes("bun run build"));
expect(`${jobName}:${build === undefined}`).toBe(`${jobName}:false`);
Expand Down Expand Up @@ -678,7 +695,12 @@ describe("GitHub Actions hardening", () => {
}
const macosJob = ci.jobs?.["platform-macos"] as { needs?: string; if?: string } | undefined;
expect(`platform-macos:${macosJob?.needs}`).toBe("platform-macos:changes");
expect(`platform-macos:${macosJob?.if}`).toBe("platform-macos:(github.event_name != 'pull_request' && github.event_name != 'merge_group') || (github.event_name == 'pull_request' && github.base_ref == 'main') || (github.event_name == 'merge_group' && github.event.merge_group.base_ref == 'refs/heads/main') || needs.changes.outputs.macos == 'true'");
expect(`platform-macos:${macosJob?.if}`).toBe("platform-macos:(github.event_name == 'push' && github.ref == 'refs/heads/dev') || (github.event_name == 'pull_request' && github.base_ref == 'main') || (github.event_name == 'merge_group' && github.event.merge_group.base_ref == 'refs/heads/main') || ((github.event_name == 'pull_request' || github.event_name == 'merge_group') && needs.changes.outputs.macos == 'true')");
const macosFullJob = ci.jobs?.["platform-macos-full"] as { needs?: string; if?: string } | undefined;
expect(`platform-macos-full:${macosFullJob?.needs}`).toBe("platform-macos-full:changes");
expect(macosFullJob?.if).toContain("github.event_name == 'workflow_dispatch'");
expect(macosFullJob?.if).toContain("refs/heads/main");
expect(macosFullJob?.if).toContain("refs/heads/preview");
});

test("cross-platform CI keeps the GUI lint and build gates", async () => {
Expand Down
24 changes: 21 additions & 3 deletions tests/ci-workflows/zz-ci-timing.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,19 @@ describe("CI lane manifest", () => {
test("never accepts a lane path outside tests", () => {
expect(() => validateLaneManifest(["tests/good.test.ts", "src/not-a-test.ts"]))
.toThrow(/outside tests/);
expect(() => validateLaneManifest(["tests/good.test.ts\n--shard=1/1"]))
.toThrow(/outside tests/);
});

test("platform lanes preserve full-suite coverage and singleton isolation", () => {
const inventory = discoverTestFiles(process.cwd());
const main = laneFiles("platform-main", process.cwd());
const serial = laneFiles("platform-serial", process.cwd());
expect(new Set([...main, ...serial])).toEqual(new Set(inventory));
expect(main.length + serial.length).toBe(inventory.length);
expect(new Set(main).intersection(new Set(serial)).size).toBe(0);
expect(serial).toEqual([...SERIAL_TEST_FILES, ...DEDICATED_TEST_FILES]);
for (const dedicated of DEDICATED_TEST_FILES) expect(main).not.toContain(dedicated);
});

test("never drops a nested test that shares a quarantined basename", () => {
Expand Down Expand Up @@ -214,6 +227,13 @@ test("only trusted dev shards publish the canonical timing cache", async () => {
step.uses === "actions/cache/save@caa296126883cff596d87d8935842f9db880ef25",
)).toBe(true);
expect(ci.jobs["platform-macos"].steps?.some(step => step.uses?.startsWith("actions/cache/save@"))).toBe(false);
const macosFull = ci.jobs["platform-macos-full"] as typeof ci.jobs[string] & {
strategy?: { matrix?: { shard?: number[] } };
};
expect(macosFull.strategy?.matrix?.shard).toEqual([1, 2]);
expect(macosFull.steps?.some(step => step.uses?.startsWith("actions/cache/save@"))).toBe(false);
expect(macosFull.steps?.some(step => step.run?.includes("--lane platform-main"))).toBe(true);
expect(macosFull.steps?.some(step => step.run?.includes("--lane platform-serial"))).toBe(true);
const windows = ci.jobs["platform-windows"];
const windowsRestore = windows.steps?.find(step => step.uses?.startsWith("actions/cache/restore@"));
expect(windowsRestore?.with?.key).toContain("ocx-test-timings-dev-");
Expand All @@ -231,9 +251,7 @@ test("only trusted dev shards publish the canonical timing cache", async () => {
expect(focused?.run).not.toContain("tests/adapters/google/aistudio-native-webkit.test.ts");
expect(focused?.run).toContain("tests/ci-workflows/test-runner.test.ts");
expect(focused?.run).toContain("tests/providers/cursor/cursor-native-exec-shell.test.ts");
const full = ci.jobs["platform-macos"].steps?.find(step => step.name === "Full macOS suite");
expect(full?.run).toContain("bun scripts/test.ts --timeout 60000");
expect(full?.run).not.toContain("--update-timings");
expect(ci.jobs["platform-macos"].steps?.find(step => step.name === "Full macOS suite")).toBeUndefined();
});

test("nightly macOS is timed at 08:17 UTC and names its timing file", async () => {
Expand Down
Loading