ci: balance test shards and stabilize CI checks

.github/workflows/node-ci.yml

@@ -145,8 +145,6 @@ jobs:
         uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
         with:
           node-version: ${{ matrix.node }}
-          cache: npm
-          cache-dependency-path: sdk/typescript/pnpm-lock.yaml
 
       - name: Set up Bun
         uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
@@ -172,7 +170,8 @@ jobs:
           TMP: ${{ runner.temp }}
           TMPDIR: ${{ runner.temp }}
           CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"
-        run: pnpm --dir sdk/typescript run ${{ matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' && 'test:ci' || 'test' }}
+        working-directory: sdk/typescript
+        run: ${{ matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' && 'pnpm run test:ci' || 'node scripts/run-ci-tests.mjs' }}
 
       - name: Upload test reports
         if: always() && matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
@@ -182,15 +181,59 @@ jobs:
           name: node-22-test-reports
           overwrite: true
           path: |
-            sdk/typescript/reports/junit.xml
-            sdk/typescript/coverage/lcov.info
+            sdk/typescript/reports/junit-*.xml
+            sdk/typescript/coverage/shard-*/lcov.info
           if-no-files-found: warn
           retention-days: 14
 
       - name: Check formatting
         if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
         run: pnpm --dir sdk/typescript run format
 
+  unix-verify:
+    name: ${{ matrix.os }} / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / verify
+    needs: validate-title
+    if: needs.validate-title.outputs.ci-mode == 'full'
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        node: ["22.13.0"]
+        include:
+          - os: ubuntu-latest
+            node: "24.0.0"
+          - os: ubuntu-latest
+            node: "24"
+          - os: ubuntu-latest
+            node: "26.0.0"
+          - os: ubuntu-latest
+            node: "26"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Set up pnpm
+        uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
+        with:
+          package_json_file: sdk/typescript/package.json
+          cache: true
+          cache_dependency_path: sdk/typescript/pnpm-lock.yaml
+
+      - name: Set up Node.js
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
+        with:
+          node-version: ${{ matrix.node }}
+          cache: npm
+          cache-dependency-path: sdk/typescript/pnpm-lock.yaml
+
+      - name: Install dependencies
+        run: pnpm --dir sdk/typescript install --frozen-lockfile
+
       - name: Pack
         working-directory: sdk/typescript
         run: pnpm pack --pack-destination ../../dist
@@ -202,7 +245,7 @@ jobs:
 
   required-test:
     name: ${{ matrix.os }} / node-22
-    needs: [validate-title, test]
+    needs: [validate-title, test, unix-verify]
     if: always()
     runs-on: ubuntu-latest
     strategy:
@@ -212,7 +255,7 @@ jobs:
 
     steps:
       - name: Require every Unix coverage job
-        if: needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && needs.test.result != 'success') || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')
+        if: needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.test.result != 'success' || needs.unix-verify.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')
         run: exit 1
 
   windows-test:
@@ -238,8 +281,42 @@ jobs:
         with:
           node-version: ${{ matrix.node }}
 
+      - name: Select pnpm version
+        id: pnpm-version
+        shell: bash
+        run: |
+          pnpm_pin=$(node -p 'require("./sdk/typescript/package.json").packageManager')
+          printf 'pin=%s\npackage=%s\n' "$pnpm_pin" "${pnpm_pin%%+*}" >> "$GITHUB_OUTPUT"
+          printf '%s/pnpm\n' "$RUNNER_TEMP" >> "$GITHUB_PATH"
+
+      - name: Cache pnpm executable
+        id: pnpm-tool
+        continue-on-error: true
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ runner.temp }}/pnpm
+          key: pnpm-tool-${{ runner.os }}-${{ runner.arch }}-${{ steps.pnpm-version.outputs.pin }}
+
       - name: Set up pnpm
-        run: npm install --global pnpm@11.9.0 --no-audit --no-fund
+        id: pnpm
+        shell: bash
+        env:
+          PNPM_PACKAGE: ${{ steps.pnpm-version.outputs.package }}
+          PNPM_CACHE_HIT: ${{ steps.pnpm-tool.outputs.cache-hit }}
+        run: |
+          if [[ "$PNPM_CACHE_HIT" != 'true' ]]; then
+            npm install --global --prefix "$RUNNER_TEMP/pnpm" "$PNPM_PACKAGE" --prefer-offline --no-audit --no-fund
+          fi
+          pnpm_store=$(pnpm store path --silent)
+          pnpm_arch=$(node -p 'process.arch')
+          printf 'store-path=%s\narch=%s\n' "$pnpm_store" "$pnpm_arch" >> "$GITHUB_OUTPUT"
+
+      - name: Cache pnpm store
+        continue-on-error: true
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ steps.pnpm.outputs.store-path }}
+          key: pnpm-cache-${{ runner.os }}-${{ steps.pnpm.outputs.arch }}-${{ hashFiles('sdk/typescript/pnpm-lock.yaml') }}
 
       - name: Set up Bun
         uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
@@ -262,7 +339,7 @@ jobs:
           TMP: ${{ steps.windows-temp.outputs.path }}
           TMPDIR: ${{ steps.windows-temp.outputs.path }}
           CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"
-        run: node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}
+        run: node sdk/typescript/scripts/run-ci-tests.mjs ${{ matrix.shard }}/7
 
       - name: Test machine-wide PowerShell policy
         if: matrix.shard == 3 && runner.environment == 'github-hosted'
@@ -275,6 +352,17 @@ jobs:
           CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "true"
         run: bun test --timeout 120000 ./tests-ts/windows-machine-policy.test.ts
 
+      - name: Upload Windows test report
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: windows-test-report-node-${{ matrix.node }}-${{ matrix.shard }}
+          overwrite: true
+          path: sdk/typescript/reports/junit-*.xml
+          if-no-files-found: warn
+          retention-days: 14
+
   windows-verify:
     name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / verify
     needs: validate-title
@@ -299,8 +387,42 @@ jobs:
           cache: npm
           cache-dependency-path: sdk/typescript/pnpm-lock.yaml
 
+      - name: Select pnpm version
+        id: pnpm-version
+        shell: bash
+        run: |
+          pnpm_pin=$(node -p 'require("./sdk/typescript/package.json").packageManager')
+          printf 'pin=%s\npackage=%s\n' "$pnpm_pin" "${pnpm_pin%%+*}" >> "$GITHUB_OUTPUT"
+          printf '%s/pnpm\n' "$RUNNER_TEMP" >> "$GITHUB_PATH"
+
+      - name: Cache pnpm executable
+        id: pnpm-tool
+        continue-on-error: true
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ runner.temp }}/pnpm
+          key: pnpm-tool-${{ runner.os }}-${{ runner.arch }}-${{ steps.pnpm-version.outputs.pin }}
+
       - name: Set up pnpm
-        run: npm install --global pnpm@11.9.0 --no-audit --no-fund
+        id: pnpm
+        shell: bash
+        env:
+          PNPM_PACKAGE: ${{ steps.pnpm-version.outputs.package }}
+          PNPM_CACHE_HIT: ${{ steps.pnpm-tool.outputs.cache-hit }}
+        run: |
+          if [[ "$PNPM_CACHE_HIT" != 'true' ]]; then
+            npm install --global --prefix "$RUNNER_TEMP/pnpm" "$PNPM_PACKAGE" --prefer-offline --no-audit --no-fund
+          fi
+          pnpm_store=$(pnpm store path --silent)
+          pnpm_arch=$(node -p 'process.arch')
+          printf 'store-path=%s\narch=%s\n' "$pnpm_store" "$pnpm_arch" >> "$GITHUB_OUTPUT"
+
+      - name: Cache pnpm store
+        continue-on-error: true
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ steps.pnpm.outputs.store-path }}
+          key: pnpm-cache-${{ runner.os }}-${{ steps.pnpm.outputs.arch }}-${{ hashFiles('sdk/typescript/pnpm-lock.yaml') }}
 
       - name: Install dependencies
         working-directory: sdk/typescript
@@ -310,9 +432,20 @@ jobs:
         working-directory: sdk/typescript
         run: pnpm pack --pack-destination ../../dist
 
+      - name: Prepare private Windows test root
+        id: windows-temp
+        shell: pwsh
+        run: ./sdk/typescript/scripts/prepare-windows-test-root.ps1
+
       - name: Inspect package
         working-directory: sdk/typescript
         shell: bash
+        env:
+          TEMP: ${{ steps.windows-temp.outputs.path }}
+          TMP: ${{ steps.windows-temp.outputs.path }}
+          npm_config_timing: "true"
+          npm_config_loglevel: http
+          UV_THREADPOOL_SIZE: "16"
         run: pnpm run check:package ../../dist/*.tgz
 
   windows:

sdk/typescript/TESTING.md

@@ -22,12 +22,18 @@ Bun's summary to `pnpm run test --seed 12345`.
 
 The local test commands pass a 30-second per-test timeout explicitly. Windows
 CI and the Windows runner experiment allow 120 seconds for slower native
-credential and document checks. `test:ci` writes `reports/junit.xml` and
-`coverage/lcov.info`. Coverage measures loaded
+credential and document checks. `test:ci` runs four independent Bun processes
+and writes `reports/junit-*.xml` and `coverage/shard-*/lcov.info`. These are
+per-shard reports, not four measurements of the full suite. Coverage measures loaded
 JavaScript and TypeScript, not the Python helpers or child processes. It is
 diagnostic for now. Use several successful CI runs to establish a baseline
 before proposing a coverage floor.
 
+The scan-history Python fixtures use direct interpreter lookup and a 30-second
+child deadline, raised from 10 seconds after repeated hosted Windows timeouts.
+They still run real Python processes and fail if Python is unavailable.
+Production interpreter discovery and its deadlines are unchanged.
+
 ## Writing tests
 
 - Test observable results, failures, cancellation, and cleanup. Prefer a
@@ -60,10 +66,72 @@ default to 100 cases; filesystem contract properties default to 20.
 
 `node-ci` retains the required `ubuntu-latest / node-22`,
 `macos-latest / node-22`, and `windows-latest / node-22` checks. Its Ubuntu
-Node 22 job runs static checks and uploads JUnit and LCOV. All supported runtime
-lanes still test and inspect an installed package. Package inspection includes
+Node 22 job runs static checks and uploads JUnit and LCOV. Every existing OS and
+Node runtime lane still runs the full suite in full CI: tests also launch Node subprocesses,
+so changing Node can affect more than the Bun test runtime. All supported runtime
+lanes still inspect an installed package. Package inspection includes
 a strict NodeNext TypeScript consumer and the actual installed CLI. Failed
-tests block CI; a failed diagnostic upload does not.
+tests block CI; a failed diagnostic upload does not. Unix package verification
+runs in six separate jobs alongside the six test lanes, with the same OS and
+Node versions. Required checks depend on both matrices. This raises full CI
+from 27 to 33 jobs and duplicates checkout and dependency setup, trading more
+runner resources for less sequential work; it does not duplicate the test suite.
+
+Pull-request validation classifies changes before starting test and package
+jobs. Markdown-only diffs run changed-file formatting instead; pushes, base
+retargets, empty diffs, and changes to other file types run full CI. Required
+check names remain stable and reject failed validation or an unknown CI mode.
+Every pull-request edit is reclassified and shares the same cancellation group,
+so editing the body of a code pull request can restart its full CI run.
+
+`scripts/run-ci-tests.mjs` discovers the test files and balances them using
+rounded timings in `scripts/test-shards.mjs`. Unix estimates average measured
+Linux and macOS file timings. Unix jobs run four processes;
+Windows runs seven separate jobs with `node scripts/run-ci-tests.mjs 1/7`
+(substitute the shard number), each using up to two Bun processes with separate
+reports such as `junit-1-1.xml` and `junit-1-2.xml`. Each file runs once. New files
+are included automatically; stale timing estimates can affect balance, but not coverage.
+Credential-home locking and ACL checks live in `runtime-credentials.test.ts` so
+they can run independently of the plugin, output-directory, and Python checks
+in `runtime.test.ts`.
+The machine-wide Windows policy test still runs separately and serially.
+Windows caches the exact pnpm version from `packageManager` in a dedicated
+runner-temp prefix. A cache miss or cache error installs it with npm; only an
+exact hit skips installation. The resolved pnpm store is cached separately.
+In full CI, only package-verification jobs restore the npm download cache for
+the fresh consumer; test jobs on either platform do not restore it. Cache failures do not suppress
+installation failures. Unix keeps its pinned pnpm
+setup action. Windows package inspection enables npm's native phase timings to
+diagnose installation delays without changing its failure or timeout behavior.
+It also logs npm cache/fetch activity and uses a 16-thread libuv pool for
+filesystem-heavy package extraction. The fresh Windows npm consumer uses
+GitHub's `RUNNER_TEMP` when available, otherwise the normal temporary directory.
+Its `TEMP` and `TMP` use the same private Windows root as the tests; the
+installed credential fixture gets a separate temporary state directory there
+so its ancestor ACL checks remain enabled. Both temporary directories are
+removed after use.
+These settings apply only to Windows package inspection; the install arguments,
+assertions, and timeouts are unchanged.
+
+When forwarding `--test-name-pattern` to a sharded run, also pass Bun's
+`--pass-with-no-tests` if some workers may have no matching tests. Normal CI
+does not enable that option, and empty file partitions are never launched.
+
+To compare the serial and CI runners on the same machine, commit, and seed:
+
+```sh
+mkdir -p reports
+pnpm run test --seed 12345 --reporter=junit --reporter-outfile=reports/baseline.xml
+node scripts/run-ci-tests.mjs --seed 12345
+python3 scripts/compare-test-reports.py reports/baseline.xml 'reports/junit-*.xml'
+```
+
+Use a clean reports directory and compare wall time as well as the test
+identities and outcomes. Run both commands with the same coverage options
+when measuring coverage overhead. The Windows serial baseline includes a
+skipped machine-policy case that the shard runner excludes; compare that
+case separately. Update the timing estimates only when reports show a
+meaningful imbalance, not on every timing fluctuation.
 
 The separate `test-quality` workflow runs weekly, can be dispatched manually,
 and runs on pull requests that change its workflow file. It compares Bun's
@@ -80,7 +148,7 @@ that breaks the Ink UI tests under isolation. Keep the trial pin until a newer
 release passes the full SDK suite in every mode. Required CI and the mutation
 trial remain on Bun 1.3.14.
 
-Keep the current file-balanced Windows runner until the native runner has
+Keep the file-balanced CI runner until the native runner has
 matching inventories and acceptable Windows timings. Before promotion, compare
 native and file-balanced shards using the same commit and Bun version.
 Keep the machine-policy test serial. Do not replace the full required suite

sdk/typescript/package.json

@@ -49,7 +49,7 @@
     "lint": "tsc --noEmit",
     "prepack": "node --run build",
     "test": "bun test --timeout 30000 ./tests-ts",
-    "test:ci": "node -e \"require('node:fs').mkdirSync('reports',{recursive:true})\" && pnpm run test --coverage --coverage-reporter=text --coverage-reporter=lcov --reporter=junit --reporter-outfile=reports/junit.xml",
+    "test:ci": "node scripts/run-ci-tests.mjs --coverage --coverage-reporter=text --coverage-reporter=lcov",
     "test:mutation": "stryker run",
     "test:package": "node scripts/smoke-package.mjs",
     "types": "pnpm run generate:models:check && tsc --noEmit"