autoresearch: speed up Test CI job

.auto/ideas.md

@@ -0,0 +1,8 @@
+# Ideas
+
+- Try 75% and 100% CI `maxWorkers`; confirm any win with repeated hosted runs because the current 50% cap was added for Postgres/PGlite stability.
+- Inspect Vitest's per-project timing from CI logs to identify heavy projects and whether project-level scheduling leaves cores idle.
+- Evaluate splitting package coverage and example tests into concurrent processes inside the same job only if CPU/memory contention does not erase the wall-clock gain; preserve a single job and all checks.
+- Explore safe coverage sharding plus Istanbul JSON merge if one Vitest coordinator cannot keep the runner busy, but do not weaken per-package coverage ownership or thresholds.
+- Determine whether the cloudflare-worker Postgres startup can overlap dependency linking/build setup without changing readiness or teardown guarantees.
+- Selectively disable file isolation only in demonstrably stateless projects (high-file-count candidates: cli, sql-orm-client, framework-components, sql-contract-ts, migration). Global `isolate: false` passed twice but violates Supabase's explicit per-file isolation requirement and is too broad to ship.

.auto/measure.sh

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo="prisma/prisma"
+workflow="CI (PR)"
+branch="$(git branch --show-current)"
+
+if [[ "$branch" != autoresearch/* ]]; then
+  echo "Refusing to benchmark from non-autoresearch branch: $branch" >&2
+  exit 1
+fi
+
+# Each hosted-CI measurement needs a distinct head SHA. Include the current
+# candidate and the prior experiment log in that SHA, then push it to the
+# temporary draft PR.
+git add -A
+git commit --signoff --allow-empty -m "autoresearch: measure Test CI $(date -u +%Y%m%dT%H%M%SZ)" >/dev/null
+sha="$(git rev-parse HEAD)"
+pushed=false
+for _ in $(seq 1 3); do
+  if timeout 120 git push --set-upstream origin "$branch" >/dev/null; then
+    pushed=true
+    break
+  fi
+  sleep 10
+done
+[[ "$pushed" == true ]] || { echo "Could not push experiment after three attempts" >&2; exit 1; }
+
+pr_number=""
+for _ in $(seq 1 6); do
+  pr_number="$(gh pr list --repo "$repo" --state open --head "$branch" --json number --jq '.[0].number // empty' 2>/dev/null || true)"
+  [[ -n "$pr_number" ]] && break
+  sleep 10
+done
+if [[ -z "$pr_number" ]]; then
+  pr_url="$(gh pr create --repo "$repo" --base main --head "$branch" --draft --title "autoresearch: speed up Test CI job" --body $'Temporary draft PR for hosted-CI performance experiments.\n\nDo not review or merge. The final result will be prepared separately.')"
+  pr_number="${pr_url##*/}"
+fi
+echo "Benchmarking PR #$pr_number at $sha" >&2
+
+run_id=""
+for _ in $(seq 1 60); do
+  run_id="$(gh run list --repo "$repo" --workflow "$workflow" --event pull_request --commit "$sha" --limit 1 --json databaseId --jq '.[0].databaseId // empty' 2>/dev/null || true)"
+  [[ -n "$run_id" ]] && break
+  sleep 5
+done
+if [[ -z "$run_id" ]]; then
+  echo "Timed out waiting for the CI run for $sha" >&2
+  exit 1
+fi
+echo "CI run: https://github.com/$repo/actions/runs/$run_id" >&2
+
+jobs_file="$(mktemp)"
+job_log="$(mktemp)"
+trap 'rm -f "$jobs_file" "$job_log"' EXIT
+for _ in $(seq 1 180); do
+  if ! gh run view "$run_id" --repo "$repo" --json jobs > "$jobs_file"; then
+    sleep 10
+    continue
+  fi
+  status="$(node -e '
+    const fs = require("node:fs");
+    const jobs = JSON.parse(fs.readFileSync(process.argv[1], "utf8")).jobs;
+    const job = jobs.find(({ name }) => name === "Test");
+    process.stdout.write(job?.status ?? "waiting");
+  ' "$jobs_file")"
+  [[ "$status" == "completed" ]] && break
+  sleep 10
+done
+
+job_id="$(node -e '
+  const fs = require("node:fs");
+  const jobs = JSON.parse(fs.readFileSync(process.argv[1], "utf8")).jobs;
+  process.stdout.write(String(jobs.find(({ name }) => name === "Test")?.databaseId ?? ""));
+' "$jobs_file")"
+for _ in $(seq 1 6); do
+  gh run view "$run_id" --repo "$repo" --job "$job_id" --log > "$job_log" && break
+  sleep 10
+done
+[[ -s "$job_log" ]] || { echo "Could not download Test job log" >&2; exit 1; }
+phase_metric() {
+  grep -o "CI_PHASE $1=[0-9]*" "$job_log" | tail -1 | cut -d= -f2 || true
+}
+export PACKAGES_COVERAGE_SECONDS="$(phase_metric packages_coverage_seconds)"
+export EXAMPLES_SECONDS="$(phase_metric examples_seconds)"
+
+node - "$jobs_file" "$run_id" <<'NODE'
+const fs = require('node:fs');
+const [jobsPath, runId] = process.argv.slice(2);
+const jobs = JSON.parse(fs.readFileSync(jobsPath, 'utf8')).jobs;
+const job = jobs.find(({ name }) => name === 'Test');
+if (!job) throw new Error('The Test job did not appear');
+if (job.status !== 'completed') throw new Error(`The Test job did not complete: ${job.status}`);
+
+const seconds = (start, end) => (Date.parse(end) - Date.parse(start)) / 1000;
+const step = (name) => {
+  const value = job.steps.find(({ name: stepName }) => stepName === name);
+  if (!value?.startedAt || !value?.completedAt) throw new Error(`Missing timing for step: ${name}`);
+  return seconds(value.startedAt, value.completedAt);
+};
+
+if (job.conclusion !== 'success') {
+  console.error(`Test job conclusion: ${job.conclusion}`);
+  for (const value of job.steps.filter(({ conclusion }) => conclusion === 'failure')) {
+    console.error(`Failed step: ${value.name}`);
+  }
+  process.exit(1);
+}
+
+const coverageStep = job.steps.find(({ name }) =>
+  ['Test packages with coverage', 'Test packages with coverage and examples'].includes(name),
+);
+if (!coverageStep?.startedAt) throw new Error('Missing package coverage start time');
+const packagesCoverageSeconds = process.env.PACKAGES_COVERAGE_SECONDS
+  ? Number(process.env.PACKAGES_COVERAGE_SECONDS)
+  : step('Test packages with coverage');
+const examplesSeconds = process.env.EXAMPLES_SECONDS
+  ? Number(process.env.EXAMPLES_SECONDS)
+  : step('Test examples');
+
+console.log(`METRIC ci_test_seconds=${seconds(job.startedAt, job.completedAt)}`);
+console.log(`METRIC packages_coverage_seconds=${packagesCoverageSeconds}`);
+console.log(`METRIC examples_seconds=${examplesSeconds}`);
+console.log(`METRIC startup_seconds=${seconds(job.startedAt, coverageStep.startedAt)}`);
+console.log(`METRIC ci_run_id=${runId}`);
+NODE

.auto/prompt.md

@@ -0,0 +1,55 @@
+# Autoresearch: speed up the Test CI job
+
+## Objective
+
+Reduce the wall-clock duration of the `Test` job in `.github/workflows/ci.yml` on GitHub-hosted CI. The job must continue to execute all existing package tests with coverage and enforce per-package coverage, execute all example tests, use the required Postgres services, and check that the working tree stays clean. Optimize real CI behavior rather than local timings.
+
+## Metrics
+
+- **Primary**: `ci_test_seconds` (seconds, lower is better) — GitHub's elapsed time from the `Test` job's `startedAt` through `completedAt`.
+- **Secondary**: `packages_coverage_seconds`, `examples_seconds`, `startup_seconds`, and `ci_run_id` — phase timing and traceability. The package-coverage phase is the dominant cost, but the primary metric remains the complete job.
+
+## How to Run
+
+`./.auto/measure.sh` commits and pushes the current experiment to the temporary draft PR, waits for that exact SHA's `CI (PR)` run and `Test` job, and emits `METRIC name=value` lines from GitHub timestamps. This intentionally measures hosted CI, not the local machine.
+
+## Files in Scope
+
+- `vitest.config.ts` — root Vitest project orchestration, worker count, pool behavior, and coverage settings.
+- `.github/workflows/ci.yml` — `Test` job structure and safe parallelization of independent phases.
+- `package.json` — package test/coverage commands.
+- `scripts/coverage-config.ts`, `scripts/coverage-report.mjs`, and their tests — coverage collection/reporting if profiling proves they matter.
+- Package Vitest configs and test-support code only when a general, behavior-preserving infrastructure optimization requires them.
+- `.auto/*` — temporary experiment harness and findings; never part of the final product change.
+
+## Off Limits
+
+- Do not remove, skip, narrow, or weaken tests, source coverage collection, coverage thresholds, coverage reporting, database-backed behavior, or clean-tree verification.
+- Do not classify executable/test-affecting changes as inert.
+- Do not optimize only for the temporary PR or GitHub cache state.
+- Do not modify production behavior merely to make tests faster.
+- Do not use local elapsed time as the primary metric.
+
+## Constraints
+
+- Preserve the semantics and pass/fail guarantees of the current `Test` job.
+- Every experiment runs on a real GitHub-hosted runner through a temporary draft PR.
+- Treat CI timing as noisy. Historical successful runs on the predecessor PR ranged from 723 to 854 seconds, with package coverage taking 549 to 650 seconds. Prefer substantial, repeatable improvements and confirm promising results.
+- The runner must remain stable: no dropped Postgres sockets, PGlite timeouts, flaky tests, or resource exhaustion.
+- Follow repository rules: use pnpm, do not weaken lint/type safety, and keep tests current when changing behavior.
+- Do not overfit or cheat the benchmark.
+
+## What's Been Tried
+
+- Before this session, package unit tests and package coverage were combined into one coverage-enabled Vitest pass, eliminating duplicate execution. That landed in PR #30082 and is the current baseline.
+- The root config currently caps CI workers at 50% to avoid oversubscribing PGlite-heavy suites and the Postgres service. Worker-count experiments are promising but must prove stability.
+- Historical predecessor-PR `Test` job durations were 723s, 799s, 828s, and 854s. Package coverage dominated at 549s, 608s, 612s, and 650s; example tests took 110–143s.
+- Increasing serial package coverage from 50% to 75% workers produced successful 773s and 787s jobs, but a repeat ended with a language-server teardown rejection after every test passed. This was not a database failure, but the configuration needs caution.
+- Running package coverage and examples concurrently looked fast but was rejected after a real `ECONNRESET`/non-queryable Postgres failure. Do not retry workload overlap.
+- Full worker reuse (`isolate: false`) passed at 50% and 75%, but it is too broad: Supabase's config explicitly requires per-file isolation to prevent pg mocks leaking into integration files. Consider only targeted reuse in proven stateless projects.
+- Hosted-runner timing has large outliers. Compare repeated medians and stability rather than trusting minima; a 548s overlap run was followed by 739s and 735s unchanged runs.
+- Selected candidate: CI uses `pool: 'vmThreads'` with `maxWorkers: '100%'`. VM contexts preserve file-level isolation while reusable threads avoid fork churn; Supabase retains its explicit `pool: 'forks'` and one-worker override. Three initial all-core VM runs passed at 741s, 761s, and 752s (median 752s), versus a 767s median at 75% and 814s at 50%. Later exact-final runs passed at 704s, 770s, 782s, and 749s across varying runner load.
+- The language-server test harness had an existing intermittent teardown rejection across multiple pools. Normal teardown now awaits the standard LSP ShutdownRequest before disposal, while immediate disposal tests keep their old path. Three post-fix all-core jobs passed, followed by a 704s favorable-runner result.
+- JSON-only coverage output showed no measurable phase-tail saving, so the text reporter was restored. Deferring Cloudflare Postgres startup did not improve coverage, so the original workflow order was restored. Five workers produced fast points but failed a legitimate 200ms test on the third sample, so do not oversubscribe. Plain threads had a 764s three-sample median versus 752s for VM threads.
+- Retain a job-local `NODE_COMPILE_CACHE` under `/tmp`: three runs passed at 630s, 774s, and 734s (median 734s), with package coverage phases of 457s, 527s, and 553s (median 527s). The broad no-cache all-core VM distribution centers around ~765s complete and ~580s package coverage. The cache is fresh per hosted job and outside the checkout.
+- The intended product diff is limited to root Vitest pool/concurrency, Test-job compile-cache configuration plus its workflow contract test, and language-server harness teardown tests.

.github/workflows/ci.yml

@@ -167,6 +167,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       TEST_TIMEOUT_MULTIPLIER: 2
+      NODE_COMPILE_CACHE: /tmp/prisma-test-node-compile-cache
       # Used by examples/prisma-8-cloudflare-worker's vitest-pool-workers
       # integration test. Mirrors the .env.example pattern; the container is
       # brought up by `pnpm db:up` below (docker-compose, not a service

packages/1-framework/3-tooling/language-server/test/server.test.ts

@@ -44,6 +44,7 @@ import {
   type RegistrationParams,
   RegistrationRequest,
   type SemanticTokens,
+  ShutdownRequest,
   StreamMessageReader,
   StreamMessageWriter,
   type TextEdit,
@@ -207,6 +208,7 @@ interface Harness {
   readonly notifyConfigChanged: (uri?: string) => void;
   readonly getDocumentAst: (uri: string) => DocumentArtifacts | undefined;
   readonly getProjectSymbolTable: (uri: string) => SymbolTable | undefined;
+  readonly shutdown: () => Promise<void>;
   dispose: () => void;
 }
 
@@ -418,6 +420,13 @@ function startHarness(
     },
     getDocumentAst: (uri) => server.getDocumentAst(uri),
     getProjectSymbolTable: (uri) => server.getProjectSymbolTable(uri),
+    shutdown: async () => {
+      await client.sendRequest(ShutdownRequest.type);
+      server.dispose();
+      client.dispose();
+      clientToServer.end();
+      serverToClient.end();
+    },
     dispose: () => {
       // Dispose the server first, so its connection is gone before the
       // transport dies. Sends made after that point — an in-flight `publish`
@@ -560,14 +569,7 @@ function deferredSettleable<T>(): {
 let harness: Harness | undefined;
 
 afterEach(async () => {
-  // The harness disposes the server before the client (see `dispose` above),
-  // so the server's `disposed` guard is raised before the transport dies and
-  // an in-flight `publish` can never log through a dead connection. This tick
-  // is a separate concern: it lets any in-flight JSON-RPC request/response
-  // write flush before the streams are torn down, so vscode-jsonrpc's own
-  // internal error logging doesn't reject a notification mid-transmission.
-  await new Promise((resolve) => setTimeout(resolve, 0));
-  harness?.dispose();
+  await harness?.shutdown();
   harness = undefined;
   configResolutionMock.resolveConfigInputs.mockReset();
   configLoaderMock.findNearestConfigPathForFile.mockReset();
@@ -2046,6 +2048,28 @@ describe('language server preserved artifacts', { timeout: timeouts.databaseOper
 });
 
 describe('language server disposal', { timeout: timeouts.databaseOperation }, () => {
+  it('gracefully shuts down after a pull-diagnostics request', async () => {
+    const unhandled: unknown[] = [];
+    const onUnhandled = (reason: unknown): void => {
+      unhandled.push(reason);
+    };
+    process.on('unhandledRejection', onUnhandled);
+    try {
+      harness = startHarness(resolveToSchema, pullDiagnosticsCapabilities);
+      await harness.initialize();
+      openDocument(harness, schemaUri, duplicateModelSource);
+      await requestPullDiagnostics(harness, schemaUri);
+
+      await harness.shutdown();
+      harness = undefined;
+      await new Promise((resolve) => setTimeout(resolve, 0));
+
+      expect(unhandled).toEqual([]);
+    } finally {
+      process.off('unhandledRejection', onUnhandled);
+    }
+  });
+
   async function assertNoUnhandledRejection(
     settle: (load: {
       readonly resolve: (value: ConfigResolution) => void;

scripts/coverage-config.test.mjs

@@ -270,6 +270,7 @@ describe('coverage config', () => {
     assert.doesNotMatch(rootVitestConfig, /from ['"]\.\/scripts\/coverage-config\.[^'"]+['"]/);
     assert.match(rootVitestConfig, /provider:\s*['"]v8['"]/);
     assert.match(rootVitestConfig, /reportOnFailure:\s*true/);
+    assert.match(rootVitestConfig, /vmMemoryLimit: process\.env\['CI'\] \? '2GB' : undefined/);
   });
 
   it('combines package tests and coverage in one CI job', async () => {
@@ -280,6 +281,7 @@ describe('coverage config', () => {
 
     assert.ok(testJob);
     assert.match(testJob, /^ {4}name: Test$/m);
+    assert.match(testJob, /^ {6}NODE_COMPILE_CACHE: \/tmp\/prisma-test-node-compile-cache$/m);
     assert.match(
       testJob,
       /run: pnpm coverage:packages\n {6}- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/,

vitest.config.ts

@@ -7,14 +7,11 @@ const coveragePolicy = composeCoverageConfig(import.meta.dirname);
 export default defineConfig({
   test: {
     projects: ['packages/**/vitest.config.ts'],
-    // Cap fork concurrency on CI so the PGlite-WASM-heavy package suites
-    // (cli, sql runtime, postgres/supabase extensions, postgres adapter +
-    // driver) don't all peak at once. Uncapped, vitest runs ~one fork per
-    // core; several CPU-hungry PGlite forks plus the postgres service
-    // container then oversubscribe the runner, stalling a fork's event loop
-    // long enough to drop its postgres socket ("Client ... is not
-    // queryable"). 50% leaves cores for the container and orchestrator.
-    maxWorkers: process.env['CI'] ? '50%' : undefined,
+    // Reuse all CI runner cores while keeping a fresh VM context per test file.
+    // Stateful projects can override this default, as the Supabase suite does.
+    maxWorkers: process.env['CI'] ? '100%' : undefined,
+    pool: process.env['CI'] ? 'vmThreads' : undefined,
+    vmMemoryLimit: process.env['CI'] ? '2GB' : undefined,
     // Hard-suppress telemetry across every package test suite. The CLI's
     // `program.hook('preAction', …)` would otherwise fork the sender
     // child every time a test invokes the CLI in-process.