From d996f4cd173bc287e0c80db8fd1a301c6b09ee85 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 09:37:56 +0000 Subject: [PATCH 01/80] chore: set up Test CI autoresearch Signed-off-by: Steven McClankerton --- .auto/ideas.md | 7 ++++ .auto/measure.sh | 85 ++++++++++++++++++++++++++++++++++++++++++++++++ .auto/prompt.md | 46 ++++++++++++++++++++++++++ 3 files changed, 138 insertions(+) create mode 100644 .auto/ideas.md create mode 100755 .auto/measure.sh create mode 100644 .auto/prompt.md diff --git a/.auto/ideas.md b/.auto/ideas.md new file mode 100644 index 000000000000..e09ad388bd89 --- /dev/null +++ b/.auto/ideas.md @@ -0,0 +1,7 @@ +# Ideas + +- Try 75% and 100% CI `maxWorkers`; confirm any win with repeated hosted runs because the current 50% cap was added for Postgres/PGlite stability. +- Inspect Vitest's per-project timing from CI logs to identify heavy projects and whether project-level scheduling leaves cores idle. +- Evaluate splitting package coverage and example tests into concurrent processes inside the same job only if CPU/memory contention does not erase the wall-clock gain; preserve a single job and all checks. +- Explore safe coverage sharding plus Istanbul JSON merge if one Vitest coordinator cannot keep the runner busy, but do not weaken per-package coverage ownership or thresholds. +- Determine whether the cloudflare-worker Postgres startup can overlap dependency linking/build setup without changing readiness or teardown guarantees. diff --git a/.auto/measure.sh b/.auto/measure.sh new file mode 100755 index 000000000000..8bb2dec336d6 --- /dev/null +++ b/.auto/measure.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo="prisma/prisma" +workflow="CI (PR)" +branch="$(git branch --show-current)" + +if [[ "$branch" != autoresearch/* ]]; then + echo "Refusing to benchmark from non-autoresearch branch: $branch" >&2 + exit 1 +fi + +# Each hosted-CI measurement needs a distinct head SHA. Include the current +# candidate and the prior experiment log in that SHA, then push it to the +# temporary draft PR. +git add -A +git commit --signoff --allow-empty -m "autoresearch: measure Test CI $(date -u +%Y%m%dT%H%M%SZ)" >/dev/null +sha="$(git rev-parse HEAD)" +git push --set-upstream origin "$branch" >/dev/null + +pr_number="$(gh pr list --repo "$repo" --state open --head "$branch" --json number --jq '.[0].number // empty')" +if [[ -z "$pr_number" ]]; then + pr_url="$(gh pr create --repo "$repo" --base main --head "$branch" --draft --title "autoresearch: speed up Test CI job" --body $'Temporary draft PR for hosted-CI performance experiments.\n\nDo not review or merge. The final result will be prepared separately.')" + pr_number="${pr_url##*/}" +fi +echo "Benchmarking PR #$pr_number at $sha" >&2 + +run_id="" +for _ in $(seq 1 60); do + run_id="$(gh run list --repo "$repo" --workflow "$workflow" --event pull_request --commit "$sha" --limit 1 --json databaseId --jq '.[0].databaseId // empty')" + [[ -n "$run_id" ]] && break + sleep 5 +done +if [[ -z "$run_id" ]]; then + echo "Timed out waiting for the CI run for $sha" >&2 + exit 1 +fi +echo "CI run: https://github.com/$repo/actions/runs/$run_id" >&2 + +jobs_file="$(mktemp)" +trap 'rm -f "$jobs_file"' EXIT +for _ in $(seq 1 180); do + gh run view "$run_id" --repo "$repo" --json jobs > "$jobs_file" + status="$(node -e ' + const fs = require("node:fs"); + const jobs = JSON.parse(fs.readFileSync(process.argv[1], "utf8")).jobs; + const job = jobs.find(({ name }) => name === "Test"); + process.stdout.write(job?.status ?? "waiting"); + ' "$jobs_file")" + [[ "$status" == "completed" ]] && break + sleep 10 +done + +node - "$jobs_file" "$run_id" <<'NODE' +const fs = require('node:fs'); +const [jobsPath, runId] = process.argv.slice(2); +const jobs = JSON.parse(fs.readFileSync(jobsPath, 'utf8')).jobs; +const job = jobs.find(({ name }) => name === 'Test'); +if (!job) throw new Error('The Test job did not appear'); +if (job.status !== 'completed') throw new Error(`The Test job did not complete: ${job.status}`); + +const seconds = (start, end) => (Date.parse(end) - Date.parse(start)) / 1000; +const step = (name) => { + const value = job.steps.find(({ name: stepName }) => stepName === name); + if (!value?.startedAt || !value?.completedAt) throw new Error(`Missing timing for step: ${name}`); + return seconds(value.startedAt, value.completedAt); +}; + +if (job.conclusion !== 'success') { + console.error(`Test job conclusion: ${job.conclusion}`); + for (const value of job.steps.filter(({ conclusion }) => conclusion === 'failure')) { + console.error(`Failed step: ${value.name}`); + } + process.exit(1); +} + +const coverageStart = job.steps.find(({ name }) => name === 'Test packages with coverage')?.startedAt; +if (!coverageStart) throw new Error('Missing package coverage start time'); + +console.log(`METRIC ci_test_seconds=${seconds(job.startedAt, job.completedAt)}`); +console.log(`METRIC packages_coverage_seconds=${step('Test packages with coverage')}`); +console.log(`METRIC examples_seconds=${step('Test examples')}`); +console.log(`METRIC startup_seconds=${seconds(job.startedAt, coverageStart)}`); +console.log(`METRIC ci_run_id=${runId}`); +NODE diff --git a/.auto/prompt.md b/.auto/prompt.md new file mode 100644 index 000000000000..b3e8d124f198 --- /dev/null +++ b/.auto/prompt.md @@ -0,0 +1,46 @@ +# Autoresearch: speed up the Test CI job + +## Objective + +Reduce the wall-clock duration of the `Test` job in `.github/workflows/ci.yml` on GitHub-hosted CI. The job must continue to execute all existing package tests with coverage and enforce per-package coverage, execute all example tests, use the required Postgres services, and check that the working tree stays clean. Optimize real CI behavior rather than local timings. + +## Metrics + +- **Primary**: `ci_test_seconds` (seconds, lower is better) — GitHub's elapsed time from the `Test` job's `startedAt` through `completedAt`. +- **Secondary**: `packages_coverage_seconds`, `examples_seconds`, `startup_seconds`, and `ci_run_id` — phase timing and traceability. The package-coverage phase is the dominant cost, but the primary metric remains the complete job. + +## How to Run + +`./.auto/measure.sh` commits and pushes the current experiment to the temporary draft PR, waits for that exact SHA's `CI (PR)` run and `Test` job, and emits `METRIC name=value` lines from GitHub timestamps. This intentionally measures hosted CI, not the local machine. + +## Files in Scope + +- `vitest.config.ts` — root Vitest project orchestration, worker count, pool behavior, and coverage settings. +- `.github/workflows/ci.yml` — `Test` job structure and safe parallelization of independent phases. +- `package.json` — package test/coverage commands. +- `scripts/coverage-config.ts`, `scripts/coverage-report.mjs`, and their tests — coverage collection/reporting if profiling proves they matter. +- Package Vitest configs and test-support code only when a general, behavior-preserving infrastructure optimization requires them. +- `.auto/*` — temporary experiment harness and findings; never part of the final product change. + +## Off Limits + +- Do not remove, skip, narrow, or weaken tests, source coverage collection, coverage thresholds, coverage reporting, database-backed behavior, or clean-tree verification. +- Do not classify executable/test-affecting changes as inert. +- Do not optimize only for the temporary PR or GitHub cache state. +- Do not modify production behavior merely to make tests faster. +- Do not use local elapsed time as the primary metric. + +## Constraints + +- Preserve the semantics and pass/fail guarantees of the current `Test` job. +- Every experiment runs on a real GitHub-hosted runner through a temporary draft PR. +- Treat CI timing as noisy. Historical successful runs on the predecessor PR ranged from 723 to 854 seconds, with package coverage taking 549 to 650 seconds. Prefer substantial, repeatable improvements and confirm promising results. +- The runner must remain stable: no dropped Postgres sockets, PGlite timeouts, flaky tests, or resource exhaustion. +- Follow repository rules: use pnpm, do not weaken lint/type safety, and keep tests current when changing behavior. +- Do not overfit or cheat the benchmark. + +## What's Been Tried + +- Before this session, package unit tests and package coverage were combined into one coverage-enabled Vitest pass, eliminating duplicate execution. That landed in PR #30082 and is the current baseline. +- The root config currently caps CI workers at 50% to avoid oversubscribing PGlite-heavy suites and the Postgres service. Worker-count experiments are promising but must prove stability. +- Historical predecessor-PR `Test` job durations were 723s, 799s, 828s, and 854s. Package coverage dominated at 549s, 608s, 612s, and 650s; example tests took 110–143s. From a4fac3f3c1530971ce86afffff7aafd543c83de1 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 09:38:05 +0000 Subject: [PATCH 02/80] autoresearch: measure Test CI 20260821T093805Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) create mode 100644 .auto/log.jsonl diff --git a/.auto/log.jsonl b/.auto/log.jsonl new file mode 100644 index 000000000000..002f305beb26 --- /dev/null +++ b/.auto/log.jsonl @@ -0,0 +1 @@ +{"type":"config","name":"Speed up Test CI job on GitHub-hosted runners","metricName":"ci_test_seconds","metricUnit":"s","bestDirection":"lower"} From 41b37c33c0536c16e247e013b4d564aa42a2e314 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 09:55:21 +0000 Subject: [PATCH 03/80] autoresearch: measure Test CI 20260821T095521Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 9 +++------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 002f305beb26..51a689607860 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -1 +1,2 @@ {"type":"config","name":"Speed up Test CI job on GitHub-hosted runners","metricName":"ci_test_seconds","metricUnit":"s","bestDirection":"lower"} +{"run":1,"commit":"a4fac3f","metric":847,"metrics":{"packages_coverage_seconds":668,"examples_seconds":114,"startup_seconds":59,"ci_run_id":32468842242},"status":"keep","description":"Baseline hosted-CI Test job on current main with CI maxWorkers at 50%","timestamp":1787306108587,"segment":0,"confidence":null,"asi":{"hypothesis":"Establish a fresh hosted-runner baseline for the complete Test job before changing worker concurrency.","historical_context":"The 847s baseline is within the predecessor PR's 723-854s range; package coverage remains dominant at 668s."}} diff --git a/vitest.config.ts b/vitest.config.ts index d0419b708b28..6d067b7f22cb 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,12 +9,9 @@ export default defineConfig({ projects: ['packages/**/vitest.config.ts'], // Cap fork concurrency on CI so the PGlite-WASM-heavy package suites // (cli, sql runtime, postgres/supabase extensions, postgres adapter + - // driver) don't all peak at once. Uncapped, vitest runs ~one fork per - // core; several CPU-hungry PGlite forks plus the postgres service - // container then oversubscribe the runner, stalling a fork's event loop - // long enough to drop its postgres socket ("Client ... is not - // queryable"). 50% leaves cores for the container and orchestrator. - maxWorkers: process.env['CI'] ? '50%' : undefined, + // driver) don't all peak at once. Leave one of the four hosted-runner + // cores for the Postgres container and Vitest orchestrator. + maxWorkers: process.env['CI'] ? '75%' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From e832d00338b52ef8e92ac08506e5b5a84d57d8f3 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 10:11:55 +0000 Subject: [PATCH 04/80] autoresearch: measure Test CI 20260821T101155Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 51a689607860..8b3d25a7b236 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -1,2 +1,3 @@ {"type":"config","name":"Speed up Test CI job on GitHub-hosted runners","metricName":"ci_test_seconds","metricUnit":"s","bestDirection":"lower"} {"run":1,"commit":"a4fac3f","metric":847,"metrics":{"packages_coverage_seconds":668,"examples_seconds":114,"startup_seconds":59,"ci_run_id":32468842242},"status":"keep","description":"Baseline hosted-CI Test job on current main with CI maxWorkers at 50%","timestamp":1787306108587,"segment":0,"confidence":null,"asi":{"hypothesis":"Establish a fresh hosted-runner baseline for the complete Test job before changing worker concurrency.","historical_context":"The 847s baseline is within the predecessor PR's 723-854s range; package coverage remains dominant at 668s."}} +{"run":2,"commit":"41b37c3","metric":787,"metrics":{"packages_coverage_seconds":607,"examples_seconds":114,"startup_seconds":60,"ci_run_id":32470205923},"status":"keep","description":"Raise CI Vitest workers from 50% to 75%, leaving one hosted-runner core free","timestamp":1787307106027,"segment":0,"confidence":null,"asi":{"hypothesis":"Using three of four hosted-runner cores should shorten the dominant coverage phase while preserving one core for Postgres and orchestration.","result":"The complete Test job improved by 60s (7.1%); the package coverage phase improved by 61s while examples and startup were unchanged. No instability appeared in this run.","next_action_hint":"Test 100% workers to find whether the fourth worker helps or recreates oversubscription."}} diff --git a/vitest.config.ts b/vitest.config.ts index 6d067b7f22cb..1844a49de294 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,9 +9,9 @@ export default defineConfig({ projects: ['packages/**/vitest.config.ts'], // Cap fork concurrency on CI so the PGlite-WASM-heavy package suites // (cli, sql runtime, postgres/supabase extensions, postgres adapter + - // driver) don't all peak at once. Leave one of the four hosted-runner - // cores for the Postgres container and Vitest orchestrator. - maxWorkers: process.env['CI'] ? '75%' : undefined, + // driver) don't all peak at once. Keep concurrency explicit so results + // remain comparable if Vitest changes its default worker policy. + maxWorkers: process.env['CI'] ? '100%' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From d60d50003d920ebefa64abbdfbdc0249dd6a350c Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 10:27:22 +0000 Subject: [PATCH 05/80] autoresearch: measure Test CI 20260821T102722Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 8b3d25a7b236..21a3a65bbb7b 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -1,3 +1,4 @@ {"type":"config","name":"Speed up Test CI job on GitHub-hosted runners","metricName":"ci_test_seconds","metricUnit":"s","bestDirection":"lower"} {"run":1,"commit":"a4fac3f","metric":847,"metrics":{"packages_coverage_seconds":668,"examples_seconds":114,"startup_seconds":59,"ci_run_id":32468842242},"status":"keep","description":"Baseline hosted-CI Test job on current main with CI maxWorkers at 50%","timestamp":1787306108587,"segment":0,"confidence":null,"asi":{"hypothesis":"Establish a fresh hosted-runner baseline for the complete Test job before changing worker concurrency.","historical_context":"The 847s baseline is within the predecessor PR's 723-854s range; package coverage remains dominant at 668s."}} {"run":2,"commit":"41b37c3","metric":787,"metrics":{"packages_coverage_seconds":607,"examples_seconds":114,"startup_seconds":60,"ci_run_id":32470205923},"status":"keep","description":"Raise CI Vitest workers from 50% to 75%, leaving one hosted-runner core free","timestamp":1787307106027,"segment":0,"confidence":null,"asi":{"hypothesis":"Using three of four hosted-runner cores should shorten the dominant coverage phase while preserving one core for Postgres and orchestration.","result":"The complete Test job improved by 60s (7.1%); the package coverage phase improved by 61s while examples and startup were unchanged. No instability appeared in this run.","next_action_hint":"Test 100% workers to find whether the fourth worker helps or recreates oversubscription."}} +{"run":3,"commit":"e832d00","metric":794,"metrics":{"packages_coverage_seconds":604,"examples_seconds":115,"startup_seconds":69,"ci_run_id":32471525771},"status":"discard","description":"Use all four hosted-runner cores for CI Vitest workers","timestamp":1787307984401,"segment":0,"confidence":8.571428571428571,"asi":{"hypothesis":"A fourth Vitest worker might further reduce the package coverage phase.","result":"Coverage was only 3s faster than 75%, while startup rose 9s and the complete job regressed 7s. The fourth worker provides no primary-metric benefit and removes resource headroom.","rollback_reason":"Primary metric regressed from the 787s best to 794s and full concurrency has higher stability risk.","next_action_hint":"Restore 75%; inspect CI logs for per-project durations or test independent phase overlap."}} diff --git a/vitest.config.ts b/vitest.config.ts index 1844a49de294..abdddee02ee9 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,9 +9,10 @@ export default defineConfig({ projects: ['packages/**/vitest.config.ts'], // Cap fork concurrency on CI so the PGlite-WASM-heavy package suites // (cli, sql runtime, postgres/supabase extensions, postgres adapter + - // driver) don't all peak at once. Keep concurrency explicit so results - // remain comparable if Vitest changes its default worker policy. - maxWorkers: process.env['CI'] ? '100%' : undefined, + // driver) don't all peak at once. Leave one of the four hosted-runner + // cores for the Postgres container and Vitest orchestrator. + maxWorkers: process.env['CI'] ? '75%' : undefined, + pool: process.env['CI'] ? 'threads' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 008d681b93d59c1c66308526e40cb21eae749f98 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 10:43:38 +0000 Subject: [PATCH 06/80] autoresearch: measure Test CI 20260821T104338Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .auto/measure.sh | 33 ++++++++++++++++++++++++++------ .github/workflows/ci.yml | 28 ++++++++++++++++++++++----- scripts/coverage-config.test.mjs | 15 +++++++-------- vitest.config.ts | 1 - 5 files changed, 58 insertions(+), 20 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 21a3a65bbb7b..503a3424e115 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -2,3 +2,4 @@ {"run":1,"commit":"a4fac3f","metric":847,"metrics":{"packages_coverage_seconds":668,"examples_seconds":114,"startup_seconds":59,"ci_run_id":32468842242},"status":"keep","description":"Baseline hosted-CI Test job on current main with CI maxWorkers at 50%","timestamp":1787306108587,"segment":0,"confidence":null,"asi":{"hypothesis":"Establish a fresh hosted-runner baseline for the complete Test job before changing worker concurrency.","historical_context":"The 847s baseline is within the predecessor PR's 723-854s range; package coverage remains dominant at 668s."}} {"run":2,"commit":"41b37c3","metric":787,"metrics":{"packages_coverage_seconds":607,"examples_seconds":114,"startup_seconds":60,"ci_run_id":32470205923},"status":"keep","description":"Raise CI Vitest workers from 50% to 75%, leaving one hosted-runner core free","timestamp":1787307106027,"segment":0,"confidence":null,"asi":{"hypothesis":"Using three of four hosted-runner cores should shorten the dominant coverage phase while preserving one core for Postgres and orchestration.","result":"The complete Test job improved by 60s (7.1%); the package coverage phase improved by 61s while examples and startup were unchanged. No instability appeared in this run.","next_action_hint":"Test 100% workers to find whether the fourth worker helps or recreates oversubscription."}} {"run":3,"commit":"e832d00","metric":794,"metrics":{"packages_coverage_seconds":604,"examples_seconds":115,"startup_seconds":69,"ci_run_id":32471525771},"status":"discard","description":"Use all four hosted-runner cores for CI Vitest workers","timestamp":1787307984401,"segment":0,"confidence":8.571428571428571,"asi":{"hypothesis":"A fourth Vitest worker might further reduce the package coverage phase.","result":"Coverage was only 3s faster than 75%, while startup rose 9s and the complete job regressed 7s. The fourth worker provides no primary-metric benefit and removes resource headroom.","rollback_reason":"Primary metric regressed from the 787s best to 794s and full concurrency has higher stability risk.","next_action_hint":"Restore 75%; inspect CI logs for per-project durations or test independent phase overlap."}} +{"run":4,"commit":"d60d500","metric":790,"metrics":{"packages_coverage_seconds":613,"examples_seconds":114,"startup_seconds":57,"ci_run_id":32472735070},"status":"discard","description":"Run CI Vitest projects in worker threads at the winning 75% concurrency","timestamp":1787308914829,"segment":0,"confidence":17.142857142857142,"asi":{"hypothesis":"Threads could reduce process startup and module-transfer overhead across 1,164 test files while retaining isolation.","result":"The suite passed, but the complete job was 3s slower and coverage 6s slower than the 75%-fork run; thread startup savings did not offset execution cost.","rollback_reason":"No primary metric improvement over 75% with the default fork pool.","next_action_hint":"Restore 75% forks; test overlap of independent package coverage and example workloads or inspect typecheck/import dominance."}} diff --git a/.auto/measure.sh b/.auto/measure.sh index 8bb2dec336d6..6197983b3c81 100755 --- a/.auto/measure.sh +++ b/.auto/measure.sh @@ -38,7 +38,8 @@ fi echo "CI run: https://github.com/$repo/actions/runs/$run_id" >&2 jobs_file="$(mktemp)" -trap 'rm -f "$jobs_file"' EXIT +job_log="$(mktemp)" +trap 'rm -f "$jobs_file" "$job_log"' EXIT for _ in $(seq 1 180); do gh run view "$run_id" --repo "$repo" --json jobs > "$jobs_file" status="$(node -e ' @@ -51,6 +52,18 @@ for _ in $(seq 1 180); do sleep 10 done +job_id="$(node -e ' + const fs = require("node:fs"); + const jobs = JSON.parse(fs.readFileSync(process.argv[1], "utf8")).jobs; + process.stdout.write(String(jobs.find(({ name }) => name === "Test")?.databaseId ?? "")); +' "$jobs_file")" +gh run view "$run_id" --repo "$repo" --job "$job_id" --log > "$job_log" +phase_metric() { + grep -o "CI_PHASE $1=[0-9]*" "$job_log" | tail -1 | cut -d= -f2 || true +} +export PACKAGES_COVERAGE_SECONDS="$(phase_metric packages_coverage_seconds)" +export EXAMPLES_SECONDS="$(phase_metric examples_seconds)" + node - "$jobs_file" "$run_id" <<'NODE' const fs = require('node:fs'); const [jobsPath, runId] = process.argv.slice(2); @@ -74,12 +87,20 @@ if (job.conclusion !== 'success') { process.exit(1); } -const coverageStart = job.steps.find(({ name }) => name === 'Test packages with coverage')?.startedAt; -if (!coverageStart) throw new Error('Missing package coverage start time'); +const coverageStep = job.steps.find(({ name }) => + ['Test packages with coverage', 'Test packages with coverage and examples'].includes(name), +); +if (!coverageStep?.startedAt) throw new Error('Missing package coverage start time'); +const packagesCoverageSeconds = process.env.PACKAGES_COVERAGE_SECONDS + ? Number(process.env.PACKAGES_COVERAGE_SECONDS) + : step('Test packages with coverage'); +const examplesSeconds = process.env.EXAMPLES_SECONDS + ? Number(process.env.EXAMPLES_SECONDS) + : step('Test examples'); console.log(`METRIC ci_test_seconds=${seconds(job.startedAt, job.completedAt)}`); -console.log(`METRIC packages_coverage_seconds=${step('Test packages with coverage')}`); -console.log(`METRIC examples_seconds=${step('Test examples')}`); -console.log(`METRIC startup_seconds=${seconds(job.startedAt, coverageStart)}`); +console.log(`METRIC packages_coverage_seconds=${packagesCoverageSeconds}`); +console.log(`METRIC examples_seconds=${examplesSeconds}`); +console.log(`METRIC startup_seconds=${seconds(job.startedAt, coverageStep.startedAt)}`); console.log(`METRIC ci_run_id=${runId}`); NODE diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac0e6125ad0e..af402fe1367e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -204,15 +204,33 @@ jobs: - name: Start cloudflare-worker Postgres (5433, pg_stat_statements) if: needs.changes.outputs.inert != 'true' run: pnpm --filter prisma-8-cloudflare-worker db:up - - name: Test packages with coverage + - name: Test packages with coverage and examples if: needs.changes.outputs.inert != 'true' - run: pnpm coverage:packages + run: | + set +e + run_timed() { + local metric="$1" + shift + local started_at="$SECONDS" + "$@" + local status="$?" + echo "CI_PHASE $metric=$((SECONDS - started_at))" + return "$status" + } + + run_timed packages_coverage_seconds pnpm coverage:packages & + coverage_pid="$!" + run_timed examples_seconds pnpm test:examples & + examples_pid="$!" + + wait "$coverage_pid" + coverage_status="$?" + wait "$examples_pid" + examples_status="$?" + ((coverage_status == 0 && examples_status == 0)) - name: Report package coverage if: ${{ !cancelled() && needs.changes.outputs.inert != 'true' }} run: pnpm coverage:report - - name: Test examples - if: ${{ !cancelled() && needs.changes.outputs.inert != 'true' }} - run: pnpm test:examples - name: Check working tree is clean if: needs.changes.outputs.inert != 'true' run: pnpm check:clean-tree diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 138f155d9fff..646c26215686 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -272,7 +272,7 @@ describe('coverage config', () => { assert.match(rootVitestConfig, /reportOnFailure:\s*true/); }); - it('combines package tests and coverage in one CI job', async () => { + it('runs package coverage and example tests concurrently in one CI job', async () => { const repositoryRoot = join(import.meta.dirname, '..'); const workflow = await readFile(join(repositoryRoot, '.github/workflows/ci.yml'), 'utf8'); const testJob = workflow.match(/\n {2}test:\n(?[\s\S]*?)(?=\n {2}test-e2e:\n)/)?.groups @@ -280,16 +280,15 @@ describe('coverage config', () => { assert.ok(testJob); assert.match(testJob, /^ {4}name: Test$/m); + assert.match(testJob, /- name: Test packages with coverage and examples/); + assert.match(testJob, /run_timed packages_coverage_seconds pnpm coverage:packages &/); + assert.match(testJob, /run_timed examples_seconds pnpm test:examples &/); assert.match( testJob, - /run: pnpm coverage:packages\n {6}- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, - ); - assert.match( - testJob, - /- name: Test examples\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm test:examples/, + /- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, ); assert.doesNotMatch(workflow, /\n {2}coverage:\n/); - assert.equal(workflow.match(/run: pnpm coverage:packages/g)?.length, 1); - assert.equal(workflow.match(/run: pnpm test:examples/g)?.length, 1); + assert.equal(workflow.match(/pnpm coverage:packages/g)?.length, 1); + assert.equal(workflow.match(/pnpm test:examples/g)?.length, 1); }); }); diff --git a/vitest.config.ts b/vitest.config.ts index abdddee02ee9..6d067b7f22cb 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -12,7 +12,6 @@ export default defineConfig({ // driver) don't all peak at once. Leave one of the four hosted-runner // cores for the Postgres container and Vitest orchestrator. maxWorkers: process.env['CI'] ? '75%' : undefined, - pool: process.env['CI'] ? 'threads' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 864dcdae407ebf2507065e92359d3cf7d5717800 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 10:56:44 +0000 Subject: [PATCH 07/80] autoresearch: measure Test CI 20260821T105643Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 503a3424e115..08ce7c4b0b04 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -3,3 +3,4 @@ {"run":2,"commit":"41b37c3","metric":787,"metrics":{"packages_coverage_seconds":607,"examples_seconds":114,"startup_seconds":60,"ci_run_id":32470205923},"status":"keep","description":"Raise CI Vitest workers from 50% to 75%, leaving one hosted-runner core free","timestamp":1787307106027,"segment":0,"confidence":null,"asi":{"hypothesis":"Using three of four hosted-runner cores should shorten the dominant coverage phase while preserving one core for Postgres and orchestration.","result":"The complete Test job improved by 60s (7.1%); the package coverage phase improved by 61s while examples and startup were unchanged. No instability appeared in this run.","next_action_hint":"Test 100% workers to find whether the fourth worker helps or recreates oversubscription."}} {"run":3,"commit":"e832d00","metric":794,"metrics":{"packages_coverage_seconds":604,"examples_seconds":115,"startup_seconds":69,"ci_run_id":32471525771},"status":"discard","description":"Use all four hosted-runner cores for CI Vitest workers","timestamp":1787307984401,"segment":0,"confidence":8.571428571428571,"asi":{"hypothesis":"A fourth Vitest worker might further reduce the package coverage phase.","result":"Coverage was only 3s faster than 75%, while startup rose 9s and the complete job regressed 7s. The fourth worker provides no primary-metric benefit and removes resource headroom.","rollback_reason":"Primary metric regressed from the 787s best to 794s and full concurrency has higher stability risk.","next_action_hint":"Restore 75%; inspect CI logs for per-project durations or test independent phase overlap."}} {"run":4,"commit":"d60d500","metric":790,"metrics":{"packages_coverage_seconds":613,"examples_seconds":114,"startup_seconds":57,"ci_run_id":32472735070},"status":"discard","description":"Run CI Vitest projects in worker threads at the winning 75% concurrency","timestamp":1787308914829,"segment":0,"confidence":17.142857142857142,"asi":{"hypothesis":"Threads could reduce process startup and module-transfer overhead across 1,164 test files while retaining isolation.","result":"The suite passed, but the complete job was 3s slower and coverage 6s slower than the 75%-fork run; thread startup savings did not offset execution cost.","rollback_reason":"No primary metric improvement over 75% with the default fork pool.","next_action_hint":"Restore 75% forks; test overlap of independent package coverage and example workloads or inspect typecheck/import dominance."}} +{"run":5,"commit":"008d681","metric":703,"metrics":{"packages_coverage_seconds":638,"examples_seconds":191,"startup_seconds":59,"ci_run_id":32473956663},"status":"keep","description":"Run package coverage and example tests concurrently inside the Test job","timestamp":1787309794515,"segment":0,"confidence":36,"asi":{"hypothesis":"Overlapping the independent package-coverage and example-test commands should reduce wall time despite CPU contention, while preserving one Test job and all pass/fail guarantees.","result":"The full job improved from 787s to 703s (10.7%). Contention slowed coverage from 607s to 638s and examples from 114s to 191s, but 191s of overlap more than compensated. Both workloads and coverage reporting passed.","correctness":"The shell waits for both PIDs, preserves either exit status, emits phase timing, and the existing coverage report still runs under !cancelled(). The workflow contract test was updated and passes.","next_action_hint":"Try 50% coverage workers during overlap to reduce contention, or confirm the 703s result before increasing complexity."}} diff --git a/vitest.config.ts b/vitest.config.ts index 6d067b7f22cb..615008531699 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,9 +9,9 @@ export default defineConfig({ projects: ['packages/**/vitest.config.ts'], // Cap fork concurrency on CI so the PGlite-WASM-heavy package suites // (cli, sql runtime, postgres/supabase extensions, postgres adapter + - // driver) don't all peak at once. Leave one of the four hosted-runner - // cores for the Postgres container and Vitest orchestrator. - maxWorkers: process.env['CI'] ? '75%' : undefined, + // driver) don't all peak at once. Leave half of the hosted-runner cores + // for the concurrently running example tests and Postgres container. + maxWorkers: process.env['CI'] ? '50%' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 66751167b5ae0b6e00127895ad43a865bc461fd9 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 11:10:38 +0000 Subject: [PATCH 08/80] autoresearch: measure Test CI 20260821T111038Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 08ce7c4b0b04..1dde3f354b3a 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -4,3 +4,4 @@ {"run":3,"commit":"e832d00","metric":794,"metrics":{"packages_coverage_seconds":604,"examples_seconds":115,"startup_seconds":69,"ci_run_id":32471525771},"status":"discard","description":"Use all four hosted-runner cores for CI Vitest workers","timestamp":1787307984401,"segment":0,"confidence":8.571428571428571,"asi":{"hypothesis":"A fourth Vitest worker might further reduce the package coverage phase.","result":"Coverage was only 3s faster than 75%, while startup rose 9s and the complete job regressed 7s. The fourth worker provides no primary-metric benefit and removes resource headroom.","rollback_reason":"Primary metric regressed from the 787s best to 794s and full concurrency has higher stability risk.","next_action_hint":"Restore 75%; inspect CI logs for per-project durations or test independent phase overlap."}} {"run":4,"commit":"d60d500","metric":790,"metrics":{"packages_coverage_seconds":613,"examples_seconds":114,"startup_seconds":57,"ci_run_id":32472735070},"status":"discard","description":"Run CI Vitest projects in worker threads at the winning 75% concurrency","timestamp":1787308914829,"segment":0,"confidence":17.142857142857142,"asi":{"hypothesis":"Threads could reduce process startup and module-transfer overhead across 1,164 test files while retaining isolation.","result":"The suite passed, but the complete job was 3s slower and coverage 6s slower than the 75%-fork run; thread startup savings did not offset execution cost.","rollback_reason":"No primary metric improvement over 75% with the default fork pool.","next_action_hint":"Restore 75% forks; test overlap of independent package coverage and example workloads or inspect typecheck/import dominance."}} {"run":5,"commit":"008d681","metric":703,"metrics":{"packages_coverage_seconds":638,"examples_seconds":191,"startup_seconds":59,"ci_run_id":32473956663},"status":"keep","description":"Run package coverage and example tests concurrently inside the Test job","timestamp":1787309794515,"segment":0,"confidence":36,"asi":{"hypothesis":"Overlapping the independent package-coverage and example-test commands should reduce wall time despite CPU contention, while preserving one Test job and all pass/fail guarantees.","result":"The full job improved from 787s to 703s (10.7%). Contention slowed coverage from 607s to 638s and examples from 114s to 191s, but 191s of overlap more than compensated. Both workloads and coverage reporting passed.","correctness":"The shell waits for both PIDs, preserves either exit status, emits phase timing, and the existing coverage report still runs under !cancelled(). The workflow contract test was updated and passes.","next_action_hint":"Try 50% coverage workers during overlap to reduce contention, or confirm the 703s result before increasing complexity."}} +{"run":6,"commit":"864dcda","metric":748,"metrics":{"packages_coverage_seconds":668,"examples_seconds":242,"startup_seconds":75,"ci_run_id":32474948672},"status":"discard","description":"Reserve two hosted-runner cores for example tests during workload overlap","timestamp":1787310627479,"segment":0,"confidence":6.260869565217392,"asi":{"hypothesis":"Reducing package Vitest from three to two workers during overlap might lessen contention enough to shorten the combined critical path.","result":"Both workloads slowed: coverage rose to 668s and examples to 242s, producing a 748s job, 45s worse than 75% overlap.","rollback_reason":"Primary metric regressed 6.4%; reserving an extra core did not improve effective throughput.","next_action_hint":"Restore 75% overlap. Try 100% overlap only as a boundary check, though prior serial 100% had no benefit."}} diff --git a/vitest.config.ts b/vitest.config.ts index 615008531699..1844a49de294 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,9 +9,9 @@ export default defineConfig({ projects: ['packages/**/vitest.config.ts'], // Cap fork concurrency on CI so the PGlite-WASM-heavy package suites // (cli, sql runtime, postgres/supabase extensions, postgres adapter + - // driver) don't all peak at once. Leave half of the hosted-runner cores - // for the concurrently running example tests and Postgres container. - maxWorkers: process.env['CI'] ? '50%' : undefined, + // driver) don't all peak at once. Keep concurrency explicit so results + // remain comparable if Vitest changes its default worker policy. + maxWorkers: process.env['CI'] ? '100%' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 6bc719fa5d525ab8820c32a001a6af57ca04fceb Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 11:21:28 +0000 Subject: [PATCH 09/80] autoresearch: measure Test CI 20260821T112128Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 1dde3f354b3a..16bfcbaa1300 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -5,3 +5,4 @@ {"run":4,"commit":"d60d500","metric":790,"metrics":{"packages_coverage_seconds":613,"examples_seconds":114,"startup_seconds":57,"ci_run_id":32472735070},"status":"discard","description":"Run CI Vitest projects in worker threads at the winning 75% concurrency","timestamp":1787308914829,"segment":0,"confidence":17.142857142857142,"asi":{"hypothesis":"Threads could reduce process startup and module-transfer overhead across 1,164 test files while retaining isolation.","result":"The suite passed, but the complete job was 3s slower and coverage 6s slower than the 75%-fork run; thread startup savings did not offset execution cost.","rollback_reason":"No primary metric improvement over 75% with the default fork pool.","next_action_hint":"Restore 75% forks; test overlap of independent package coverage and example workloads or inspect typecheck/import dominance."}} {"run":5,"commit":"008d681","metric":703,"metrics":{"packages_coverage_seconds":638,"examples_seconds":191,"startup_seconds":59,"ci_run_id":32473956663},"status":"keep","description":"Run package coverage and example tests concurrently inside the Test job","timestamp":1787309794515,"segment":0,"confidence":36,"asi":{"hypothesis":"Overlapping the independent package-coverage and example-test commands should reduce wall time despite CPU contention, while preserving one Test job and all pass/fail guarantees.","result":"The full job improved from 787s to 703s (10.7%). Contention slowed coverage from 607s to 638s and examples from 114s to 191s, but 191s of overlap more than compensated. Both workloads and coverage reporting passed.","correctness":"The shell waits for both PIDs, preserves either exit status, emits phase timing, and the existing coverage report still runs under !cancelled(). The workflow contract test was updated and passes.","next_action_hint":"Try 50% coverage workers during overlap to reduce contention, or confirm the 703s result before increasing complexity."}} {"run":6,"commit":"864dcda","metric":748,"metrics":{"packages_coverage_seconds":668,"examples_seconds":242,"startup_seconds":75,"ci_run_id":32474948672},"status":"discard","description":"Reserve two hosted-runner cores for example tests during workload overlap","timestamp":1787310627479,"segment":0,"confidence":6.260869565217392,"asi":{"hypothesis":"Reducing package Vitest from three to two workers during overlap might lessen contention enough to shorten the combined critical path.","result":"Both workloads slowed: coverage rose to 668s and examples to 242s, producing a 748s job, 45s worse than 75% overlap.","rollback_reason":"Primary metric regressed 6.4%; reserving an extra core did not improve effective throughput.","next_action_hint":"Restore 75% overlap. Try 100% overlap only as a boundary check, though prior serial 100% had no benefit."}} +{"run":7,"commit":"6675116","metric":548,"metrics":{"packages_coverage_seconds":475,"examples_seconds":196,"startup_seconds":66,"ci_run_id":32476018995},"status":"keep","description":"Use full CI Vitest concurrency while package coverage overlaps example tests","timestamp":1787311274030,"segment":0,"confidence":7.666666666666667,"asi":{"hypothesis":"With examples already consuming otherwise idle intervals, allowing Vitest its fourth worker may increase total throughput despite oversubscription.","result":"The job dropped from 703s to 548s (22.0%); coverage fell from 638s to 475s while examples stayed near 196s. All 1,164 package files, examples, coverage report, and clean-tree check passed.","risk":"This is unexpectedly much better than both 75%-overlap and serial 100%. Full concurrency previously carried Postgres/PGlite instability risk, so repeat before trusting.","next_action_hint":"Repeat the identical 100%-overlap configuration to test stability and timing reproducibility."}} From f97053674754648e8c33d844c440edc2e7dfaed1 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 11:35:08 +0000 Subject: [PATCH 10/80] autoresearch: measure Test CI 20260821T113508Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 16bfcbaa1300..b22b2018fa1a 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -6,3 +6,4 @@ {"run":5,"commit":"008d681","metric":703,"metrics":{"packages_coverage_seconds":638,"examples_seconds":191,"startup_seconds":59,"ci_run_id":32473956663},"status":"keep","description":"Run package coverage and example tests concurrently inside the Test job","timestamp":1787309794515,"segment":0,"confidence":36,"asi":{"hypothesis":"Overlapping the independent package-coverage and example-test commands should reduce wall time despite CPU contention, while preserving one Test job and all pass/fail guarantees.","result":"The full job improved from 787s to 703s (10.7%). Contention slowed coverage from 607s to 638s and examples from 114s to 191s, but 191s of overlap more than compensated. Both workloads and coverage reporting passed.","correctness":"The shell waits for both PIDs, preserves either exit status, emits phase timing, and the existing coverage report still runs under !cancelled(). The workflow contract test was updated and passes.","next_action_hint":"Try 50% coverage workers during overlap to reduce contention, or confirm the 703s result before increasing complexity."}} {"run":6,"commit":"864dcda","metric":748,"metrics":{"packages_coverage_seconds":668,"examples_seconds":242,"startup_seconds":75,"ci_run_id":32474948672},"status":"discard","description":"Reserve two hosted-runner cores for example tests during workload overlap","timestamp":1787310627479,"segment":0,"confidence":6.260869565217392,"asi":{"hypothesis":"Reducing package Vitest from three to two workers during overlap might lessen contention enough to shorten the combined critical path.","result":"Both workloads slowed: coverage rose to 668s and examples to 242s, producing a 748s job, 45s worse than 75% overlap.","rollback_reason":"Primary metric regressed 6.4%; reserving an extra core did not improve effective throughput.","next_action_hint":"Restore 75% overlap. Try 100% overlap only as a boundary check, though prior serial 100% had no benefit."}} {"run":7,"commit":"6675116","metric":548,"metrics":{"packages_coverage_seconds":475,"examples_seconds":196,"startup_seconds":66,"ci_run_id":32476018995},"status":"keep","description":"Use full CI Vitest concurrency while package coverage overlaps example tests","timestamp":1787311274030,"segment":0,"confidence":7.666666666666667,"asi":{"hypothesis":"With examples already consuming otherwise idle intervals, allowing Vitest its fourth worker may increase total throughput despite oversubscription.","result":"The job dropped from 703s to 548s (22.0%); coverage fell from 638s to 475s while examples stayed near 196s. All 1,164 package files, examples, coverage report, and clean-tree check passed.","risk":"This is unexpectedly much better than both 75%-overlap and serial 100%. Full concurrency previously carried Postgres/PGlite instability risk, so repeat before trusting.","next_action_hint":"Repeat the identical 100%-overlap configuration to test stability and timing reproducibility."}} +{"run":8,"commit":"6bc719f","metric":739,"metrics":{"packages_coverage_seconds":663,"examples_seconds":248,"startup_seconds":69,"ci_run_id":32476845242},"status":"discard","description":"Repeat full-concurrency overlap configuration for reproducibility","timestamp":1787312103481,"segment":0,"confidence":10.872727272727273,"asi":{"hypothesis":"An unchanged rerun would confirm the 548s result and full-concurrency stability.","result":"The rerun passed but took 739s; coverage varied from 475s to 663s and examples from 196s to 248s. Hosted-runner variance is much larger than expected, so the 548s point is not representative by itself.","rollback_reason":"The repeated measurement regressed 191s versus the prior point; no code delta exists to retain or revert.","next_action_hint":"Run another unchanged 100%-overlap sample, then compare a small sample distribution against repeated 75%-overlap rather than single minima."}} From fe0ea57bffd1be27dcf6be518f3c8cdbd2fccf4b Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 11:49:00 +0000 Subject: [PATCH 11/80] autoresearch: measure Test CI 20260821T114900Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index b22b2018fa1a..a65949e9ed03 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -7,3 +7,4 @@ {"run":6,"commit":"864dcda","metric":748,"metrics":{"packages_coverage_seconds":668,"examples_seconds":242,"startup_seconds":75,"ci_run_id":32474948672},"status":"discard","description":"Reserve two hosted-runner cores for example tests during workload overlap","timestamp":1787310627479,"segment":0,"confidence":6.260869565217392,"asi":{"hypothesis":"Reducing package Vitest from three to two workers during overlap might lessen contention enough to shorten the combined critical path.","result":"Both workloads slowed: coverage rose to 668s and examples to 242s, producing a 748s job, 45s worse than 75% overlap.","rollback_reason":"Primary metric regressed 6.4%; reserving an extra core did not improve effective throughput.","next_action_hint":"Restore 75% overlap. Try 100% overlap only as a boundary check, though prior serial 100% had no benefit."}} {"run":7,"commit":"6675116","metric":548,"metrics":{"packages_coverage_seconds":475,"examples_seconds":196,"startup_seconds":66,"ci_run_id":32476018995},"status":"keep","description":"Use full CI Vitest concurrency while package coverage overlaps example tests","timestamp":1787311274030,"segment":0,"confidence":7.666666666666667,"asi":{"hypothesis":"With examples already consuming otherwise idle intervals, allowing Vitest its fourth worker may increase total throughput despite oversubscription.","result":"The job dropped from 703s to 548s (22.0%); coverage fell from 638s to 475s while examples stayed near 196s. All 1,164 package files, examples, coverage report, and clean-tree check passed.","risk":"This is unexpectedly much better than both 75%-overlap and serial 100%. Full concurrency previously carried Postgres/PGlite instability risk, so repeat before trusting.","next_action_hint":"Repeat the identical 100%-overlap configuration to test stability and timing reproducibility."}} {"run":8,"commit":"6bc719f","metric":739,"metrics":{"packages_coverage_seconds":663,"examples_seconds":248,"startup_seconds":69,"ci_run_id":32476845242},"status":"discard","description":"Repeat full-concurrency overlap configuration for reproducibility","timestamp":1787312103481,"segment":0,"confidence":10.872727272727273,"asi":{"hypothesis":"An unchanged rerun would confirm the 548s result and full-concurrency stability.","result":"The rerun passed but took 739s; coverage varied from 475s to 663s and examples from 196s to 248s. Hosted-runner variance is much larger than expected, so the 548s point is not representative by itself.","rollback_reason":"The repeated measurement regressed 191s versus the prior point; no code delta exists to retain or revert.","next_action_hint":"Run another unchanged 100%-overlap sample, then compare a small sample distribution against repeated 75%-overlap rather than single minima."}} +{"run":9,"commit":"f970536","metric":735,"metrics":{"packages_coverage_seconds":663,"examples_seconds":246,"startup_seconds":65,"ci_run_id":32477882654},"status":"discard","description":"Third full-concurrency overlap measurement","timestamp":1787312922740,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"A third sample would distinguish the 548s outlier from typical 100%-overlap performance.","result":"The job passed at 735s, nearly matching the 739s repeat. Typical full-concurrency overlap is therefore about 737s; the 548s run was a favorable hosted-runner outlier.","rollback_reason":"No code delta; this repeated point is worse than both the retained minimum and the sole 75%-overlap sample.","next_action_hint":"Switch back to 75%-overlap and collect two more samples for a fair median comparison."}} diff --git a/vitest.config.ts b/vitest.config.ts index 1844a49de294..8373da70ae72 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,9 +9,9 @@ export default defineConfig({ projects: ['packages/**/vitest.config.ts'], // Cap fork concurrency on CI so the PGlite-WASM-heavy package suites // (cli, sql runtime, postgres/supabase extensions, postgres adapter + - // driver) don't all peak at once. Keep concurrency explicit so results - // remain comparable if Vitest changes its default worker policy. - maxWorkers: process.env['CI'] ? '100%' : undefined, + // driver) don't all peak at once. Leave one of the four hosted-runner + // cores for the concurrently running example tests and Postgres container. + maxWorkers: process.env['CI'] ? '75%' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 9fca75b6d85cb2e0e2436c2f92e857f439f90c02 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 12:02:37 +0000 Subject: [PATCH 12/80] autoresearch: measure Test CI 20260821T120237Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index a65949e9ed03..b62860441784 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -8,3 +8,4 @@ {"run":7,"commit":"6675116","metric":548,"metrics":{"packages_coverage_seconds":475,"examples_seconds":196,"startup_seconds":66,"ci_run_id":32476018995},"status":"keep","description":"Use full CI Vitest concurrency while package coverage overlaps example tests","timestamp":1787311274030,"segment":0,"confidence":7.666666666666667,"asi":{"hypothesis":"With examples already consuming otherwise idle intervals, allowing Vitest its fourth worker may increase total throughput despite oversubscription.","result":"The job dropped from 703s to 548s (22.0%); coverage fell from 638s to 475s while examples stayed near 196s. All 1,164 package files, examples, coverage report, and clean-tree check passed.","risk":"This is unexpectedly much better than both 75%-overlap and serial 100%. Full concurrency previously carried Postgres/PGlite instability risk, so repeat before trusting.","next_action_hint":"Repeat the identical 100%-overlap configuration to test stability and timing reproducibility."}} {"run":8,"commit":"6bc719f","metric":739,"metrics":{"packages_coverage_seconds":663,"examples_seconds":248,"startup_seconds":69,"ci_run_id":32476845242},"status":"discard","description":"Repeat full-concurrency overlap configuration for reproducibility","timestamp":1787312103481,"segment":0,"confidence":10.872727272727273,"asi":{"hypothesis":"An unchanged rerun would confirm the 548s result and full-concurrency stability.","result":"The rerun passed but took 739s; coverage varied from 475s to 663s and examples from 196s to 248s. Hosted-runner variance is much larger than expected, so the 548s point is not representative by itself.","rollback_reason":"The repeated measurement regressed 191s versus the prior point; no code delta exists to retain or revert.","next_action_hint":"Run another unchanged 100%-overlap sample, then compare a small sample distribution against repeated 75%-overlap rather than single minima."}} {"run":9,"commit":"f970536","metric":735,"metrics":{"packages_coverage_seconds":663,"examples_seconds":246,"startup_seconds":65,"ci_run_id":32477882654},"status":"discard","description":"Third full-concurrency overlap measurement","timestamp":1787312922740,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"A third sample would distinguish the 548s outlier from typical 100%-overlap performance.","result":"The job passed at 735s, nearly matching the 739s repeat. Typical full-concurrency overlap is therefore about 737s; the 548s run was a favorable hosted-runner outlier.","rollback_reason":"No code delta; this repeated point is worse than both the retained minimum and the sole 75%-overlap sample.","next_action_hint":"Switch back to 75%-overlap and collect two more samples for a fair median comparison."}} +{"run":10,"commit":"fe0ea57","metric":736,"metrics":{"packages_coverage_seconds":661,"examples_seconds":195,"startup_seconds":68,"ci_run_id":32478951542},"status":"discard","description":"Return to 75%-worker overlap for a second sample","timestamp":1787313751851,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"Leaving one core free should make overlap faster and steadier than full Vitest concurrency.","result":"The second 75% sample passed at 736s versus its first 703s sample. Coverage was 661s and examples 195s. This is effectively tied with the two typical 100% samples at 735s and 739s.","rollback_reason":"The 736s primary metric did not improve the retained best; the tool cannot meaningfully revert already-pushed measurement commits.","next_action_hint":"Collect a third 75% sample; choose based on medians and stability, not the 548s outlier."}} From 16bb93727305f3f76ccd21d3944660cbec63143f Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 12:13:43 +0000 Subject: [PATCH 13/80] autoresearch: measure Test CI 20260821T121343Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .github/workflows/ci.yml | 2 +- scripts/coverage-config.test.mjs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index b62860441784..c7f1853ef8e1 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -9,3 +9,4 @@ {"run":8,"commit":"6bc719f","metric":739,"metrics":{"packages_coverage_seconds":663,"examples_seconds":248,"startup_seconds":69,"ci_run_id":32476845242},"status":"discard","description":"Repeat full-concurrency overlap configuration for reproducibility","timestamp":1787312103481,"segment":0,"confidence":10.872727272727273,"asi":{"hypothesis":"An unchanged rerun would confirm the 548s result and full-concurrency stability.","result":"The rerun passed but took 739s; coverage varied from 475s to 663s and examples from 196s to 248s. Hosted-runner variance is much larger than expected, so the 548s point is not representative by itself.","rollback_reason":"The repeated measurement regressed 191s versus the prior point; no code delta exists to retain or revert.","next_action_hint":"Run another unchanged 100%-overlap sample, then compare a small sample distribution against repeated 75%-overlap rather than single minima."}} {"run":9,"commit":"f970536","metric":735,"metrics":{"packages_coverage_seconds":663,"examples_seconds":246,"startup_seconds":65,"ci_run_id":32477882654},"status":"discard","description":"Third full-concurrency overlap measurement","timestamp":1787312922740,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"A third sample would distinguish the 548s outlier from typical 100%-overlap performance.","result":"The job passed at 735s, nearly matching the 739s repeat. Typical full-concurrency overlap is therefore about 737s; the 548s run was a favorable hosted-runner outlier.","rollback_reason":"No code delta; this repeated point is worse than both the retained minimum and the sole 75%-overlap sample.","next_action_hint":"Switch back to 75%-overlap and collect two more samples for a fair median comparison."}} {"run":10,"commit":"fe0ea57","metric":736,"metrics":{"packages_coverage_seconds":661,"examples_seconds":195,"startup_seconds":68,"ci_run_id":32478951542},"status":"discard","description":"Return to 75%-worker overlap for a second sample","timestamp":1787313751851,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"Leaving one core free should make overlap faster and steadier than full Vitest concurrency.","result":"The second 75% sample passed at 736s versus its first 703s sample. Coverage was 661s and examples 195s. This is effectively tied with the two typical 100% samples at 735s and 739s.","rollback_reason":"The 736s primary metric did not improve the retained best; the tool cannot meaningfully revert already-pushed measurement commits.","next_action_hint":"Collect a third 75% sample; choose based on medians and stability, not the 548s outlier."}} +{"run":11,"commit":"9fca75b","metric":571,"metrics":{"packages_coverage_seconds":510,"examples_seconds":155,"startup_seconds":56,"ci_run_id":32480009364},"status":"discard","description":"Third 75%-worker overlap measurement","timestamp":1787314407118,"segment":0,"confidence":6.229166666666667,"asi":{"hypothesis":"A third 75% sample would establish a robust median for selecting worker concurrency.","result":"The run passed at 571s. Three 75% samples are 571/703/736 (median 703s); three 100% samples are 548/735/739 (median 735s). Choose 75% based on the 32s lower median and resource headroom.","rollback_reason":"This sample does not beat the absolute 548s hosted-runner outlier, though it establishes 75% as the better median configuration.","next_action_hint":"Keep 75% conceptually and cap concurrent example Turbo task concurrency; examples finish far before coverage, so trading example speed for less coverage contention may reduce the critical path."}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af402fe1367e..f1013cabc4ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -220,7 +220,7 @@ jobs: run_timed packages_coverage_seconds pnpm coverage:packages & coverage_pid="$!" - run_timed examples_seconds pnpm test:examples & + run_timed examples_seconds pnpm test:examples --concurrency=1 & examples_pid="$!" wait "$coverage_pid" diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 646c26215686..8bcd8e839dc3 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -282,7 +282,7 @@ describe('coverage config', () => { assert.match(testJob, /^ {4}name: Test$/m); assert.match(testJob, /- name: Test packages with coverage and examples/); assert.match(testJob, /run_timed packages_coverage_seconds pnpm coverage:packages &/); - assert.match(testJob, /run_timed examples_seconds pnpm test:examples &/); + assert.match(testJob, /run_timed examples_seconds pnpm test:examples --concurrency=1 &/); assert.match( testJob, /- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, From ef0c2a3b89894bc91dceb5b7fa849768448f1f5f Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 12:26:11 +0000 Subject: [PATCH 14/80] autoresearch: measure Test CI 20260821T122611Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .github/workflows/ci.yml | 2 +- scripts/coverage-config.test.mjs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index c7f1853ef8e1..94a061ff41e4 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -10,3 +10,4 @@ {"run":9,"commit":"f970536","metric":735,"metrics":{"packages_coverage_seconds":663,"examples_seconds":246,"startup_seconds":65,"ci_run_id":32477882654},"status":"discard","description":"Third full-concurrency overlap measurement","timestamp":1787312922740,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"A third sample would distinguish the 548s outlier from typical 100%-overlap performance.","result":"The job passed at 735s, nearly matching the 739s repeat. Typical full-concurrency overlap is therefore about 737s; the 548s run was a favorable hosted-runner outlier.","rollback_reason":"No code delta; this repeated point is worse than both the retained minimum and the sole 75%-overlap sample.","next_action_hint":"Switch back to 75%-overlap and collect two more samples for a fair median comparison."}} {"run":10,"commit":"fe0ea57","metric":736,"metrics":{"packages_coverage_seconds":661,"examples_seconds":195,"startup_seconds":68,"ci_run_id":32478951542},"status":"discard","description":"Return to 75%-worker overlap for a second sample","timestamp":1787313751851,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"Leaving one core free should make overlap faster and steadier than full Vitest concurrency.","result":"The second 75% sample passed at 736s versus its first 703s sample. Coverage was 661s and examples 195s. This is effectively tied with the two typical 100% samples at 735s and 739s.","rollback_reason":"The 736s primary metric did not improve the retained best; the tool cannot meaningfully revert already-pushed measurement commits.","next_action_hint":"Collect a third 75% sample; choose based on medians and stability, not the 548s outlier."}} {"run":11,"commit":"9fca75b","metric":571,"metrics":{"packages_coverage_seconds":510,"examples_seconds":155,"startup_seconds":56,"ci_run_id":32480009364},"status":"discard","description":"Third 75%-worker overlap measurement","timestamp":1787314407118,"segment":0,"confidence":6.229166666666667,"asi":{"hypothesis":"A third 75% sample would establish a robust median for selecting worker concurrency.","result":"The run passed at 571s. Three 75% samples are 571/703/736 (median 703s); three 100% samples are 548/735/739 (median 735s). Choose 75% based on the 32s lower median and resource headroom.","rollback_reason":"This sample does not beat the absolute 548s hosted-runner outlier, though it establishes 75% as the better median configuration.","next_action_hint":"Keep 75% conceptually and cap concurrent example Turbo task concurrency; examples finish far before coverage, so trading example speed for less coverage contention may reduce the critical path."}} +{"run":12,"commit":"16bb937","metric":674,"metrics":{"packages_coverage_seconds":599,"examples_seconds":333,"startup_seconds":71,"ci_run_id":32480893962},"status":"discard","description":"Limit overlapping example Turbo tasks to one at a time","timestamp":1787315161828,"segment":0,"confidence":5.862745098039215,"asi":{"hypothesis":"Serializing example workspace tasks would free CPU for the coverage critical path while examples still finish before coverage.","result":"The job passed at 674s; examples expanded to 333s but still completed 266s before coverage at 599s. This is 29s below the uncapped 75%-overlap median, but one sample cannot resolve runner noise.","rollback_reason":"The single 674s point does not beat the retained absolute best and requires comparison with a moderate example concurrency.","next_action_hint":"Try --concurrency=2, which may reduce examples oversubscription without serializing all workspace tasks."}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1013cabc4ee..ec6ca9dc0ee9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -220,7 +220,7 @@ jobs: run_timed packages_coverage_seconds pnpm coverage:packages & coverage_pid="$!" - run_timed examples_seconds pnpm test:examples --concurrency=1 & + run_timed examples_seconds pnpm test:examples --concurrency=2 & examples_pid="$!" wait "$coverage_pid" diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 8bcd8e839dc3..f2e1b850def8 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -282,7 +282,7 @@ describe('coverage config', () => { assert.match(testJob, /^ {4}name: Test$/m); assert.match(testJob, /- name: Test packages with coverage and examples/); assert.match(testJob, /run_timed packages_coverage_seconds pnpm coverage:packages &/); - assert.match(testJob, /run_timed examples_seconds pnpm test:examples --concurrency=1 &/); + assert.match(testJob, /run_timed examples_seconds pnpm test:examples --concurrency=2 &/); assert.match( testJob, /- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, From 98b6eec7af37d524ae42001f8a0755bc33148bcf Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 12:40:15 +0000 Subject: [PATCH 15/80] autoresearch: measure Test CI 20260821T124015Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .github/workflows/ci.yml | 2 +- scripts/coverage-config.test.mjs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 94a061ff41e4..31fb0bcc1ed9 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -11,3 +11,4 @@ {"run":10,"commit":"fe0ea57","metric":736,"metrics":{"packages_coverage_seconds":661,"examples_seconds":195,"startup_seconds":68,"ci_run_id":32478951542},"status":"discard","description":"Return to 75%-worker overlap for a second sample","timestamp":1787313751851,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"Leaving one core free should make overlap faster and steadier than full Vitest concurrency.","result":"The second 75% sample passed at 736s versus its first 703s sample. Coverage was 661s and examples 195s. This is effectively tied with the two typical 100% samples at 735s and 739s.","rollback_reason":"The 736s primary metric did not improve the retained best; the tool cannot meaningfully revert already-pushed measurement commits.","next_action_hint":"Collect a third 75% sample; choose based on medians and stability, not the 548s outlier."}} {"run":11,"commit":"9fca75b","metric":571,"metrics":{"packages_coverage_seconds":510,"examples_seconds":155,"startup_seconds":56,"ci_run_id":32480009364},"status":"discard","description":"Third 75%-worker overlap measurement","timestamp":1787314407118,"segment":0,"confidence":6.229166666666667,"asi":{"hypothesis":"A third 75% sample would establish a robust median for selecting worker concurrency.","result":"The run passed at 571s. Three 75% samples are 571/703/736 (median 703s); three 100% samples are 548/735/739 (median 735s). Choose 75% based on the 32s lower median and resource headroom.","rollback_reason":"This sample does not beat the absolute 548s hosted-runner outlier, though it establishes 75% as the better median configuration.","next_action_hint":"Keep 75% conceptually and cap concurrent example Turbo task concurrency; examples finish far before coverage, so trading example speed for less coverage contention may reduce the critical path."}} {"run":12,"commit":"16bb937","metric":674,"metrics":{"packages_coverage_seconds":599,"examples_seconds":333,"startup_seconds":71,"ci_run_id":32480893962},"status":"discard","description":"Limit overlapping example Turbo tasks to one at a time","timestamp":1787315161828,"segment":0,"confidence":5.862745098039215,"asi":{"hypothesis":"Serializing example workspace tasks would free CPU for the coverage critical path while examples still finish before coverage.","result":"The job passed at 674s; examples expanded to 333s but still completed 266s before coverage at 599s. This is 29s below the uncapped 75%-overlap median, but one sample cannot resolve runner noise.","rollback_reason":"The single 674s point does not beat the retained absolute best and requires comparison with a moderate example concurrency.","next_action_hint":"Try --concurrency=2, which may reduce examples oversubscription without serializing all workspace tasks."}} +{"run":13,"commit":"ef0c2a3","metric":763,"metrics":{"packages_coverage_seconds":686,"examples_seconds":246,"startup_seconds":70,"ci_run_id":32481888160},"status":"discard","description":"Limit overlapping example Turbo tasks to two at a time","timestamp":1787316006633,"segment":0,"confidence":6.229166666666667,"asi":{"hypothesis":"Two concurrent example workspace tasks could balance example completion against package-coverage CPU needs.","result":"The job passed but took 763s; coverage was the slowest overlap sample at 686s while examples took 246s. Moderate Turbo concurrency did not produce a useful balance.","rollback_reason":"Primary metric regressed 89s versus example concurrency 1 and 60s versus uncapped 75%-overlap median.","next_action_hint":"Return to concurrency 1 for another sample, or test whether deferring examples until late in coverage can reduce peak contention."}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec6ca9dc0ee9..f1013cabc4ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -220,7 +220,7 @@ jobs: run_timed packages_coverage_seconds pnpm coverage:packages & coverage_pid="$!" - run_timed examples_seconds pnpm test:examples --concurrency=2 & + run_timed examples_seconds pnpm test:examples --concurrency=1 & examples_pid="$!" wait "$coverage_pid" diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index f2e1b850def8..8bcd8e839dc3 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -282,7 +282,7 @@ describe('coverage config', () => { assert.match(testJob, /^ {4}name: Test$/m); assert.match(testJob, /- name: Test packages with coverage and examples/); assert.match(testJob, /run_timed packages_coverage_seconds pnpm coverage:packages &/); - assert.match(testJob, /run_timed examples_seconds pnpm test:examples --concurrency=2 &/); + assert.match(testJob, /run_timed examples_seconds pnpm test:examples --concurrency=1 &/); assert.match( testJob, /- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, From 8ff22aa9b47cafc5e83859b2d484bab3f52426b8 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 12:52:49 +0000 Subject: [PATCH 16/80] autoresearch: measure Test CI 20260821T125249Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .github/workflows/ci.yml | 28 +++++----------------------- scripts/coverage-config.test.mjs | 15 ++++++++------- 3 files changed, 14 insertions(+), 30 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 31fb0bcc1ed9..3266492e285c 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -12,3 +12,4 @@ {"run":11,"commit":"9fca75b","metric":571,"metrics":{"packages_coverage_seconds":510,"examples_seconds":155,"startup_seconds":56,"ci_run_id":32480009364},"status":"discard","description":"Third 75%-worker overlap measurement","timestamp":1787314407118,"segment":0,"confidence":6.229166666666667,"asi":{"hypothesis":"A third 75% sample would establish a robust median for selecting worker concurrency.","result":"The run passed at 571s. Three 75% samples are 571/703/736 (median 703s); three 100% samples are 548/735/739 (median 735s). Choose 75% based on the 32s lower median and resource headroom.","rollback_reason":"This sample does not beat the absolute 548s hosted-runner outlier, though it establishes 75% as the better median configuration.","next_action_hint":"Keep 75% conceptually and cap concurrent example Turbo task concurrency; examples finish far before coverage, so trading example speed for less coverage contention may reduce the critical path."}} {"run":12,"commit":"16bb937","metric":674,"metrics":{"packages_coverage_seconds":599,"examples_seconds":333,"startup_seconds":71,"ci_run_id":32480893962},"status":"discard","description":"Limit overlapping example Turbo tasks to one at a time","timestamp":1787315161828,"segment":0,"confidence":5.862745098039215,"asi":{"hypothesis":"Serializing example workspace tasks would free CPU for the coverage critical path while examples still finish before coverage.","result":"The job passed at 674s; examples expanded to 333s but still completed 266s before coverage at 599s. This is 29s below the uncapped 75%-overlap median, but one sample cannot resolve runner noise.","rollback_reason":"The single 674s point does not beat the retained absolute best and requires comparison with a moderate example concurrency.","next_action_hint":"Try --concurrency=2, which may reduce examples oversubscription without serializing all workspace tasks."}} {"run":13,"commit":"ef0c2a3","metric":763,"metrics":{"packages_coverage_seconds":686,"examples_seconds":246,"startup_seconds":70,"ci_run_id":32481888160},"status":"discard","description":"Limit overlapping example Turbo tasks to two at a time","timestamp":1787316006633,"segment":0,"confidence":6.229166666666667,"asi":{"hypothesis":"Two concurrent example workspace tasks could balance example completion against package-coverage CPU needs.","result":"The job passed but took 763s; coverage was the slowest overlap sample at 686s while examples took 246s. Moderate Turbo concurrency did not produce a useful balance.","rollback_reason":"Primary metric regressed 89s versus example concurrency 1 and 60s versus uncapped 75%-overlap median.","next_action_hint":"Return to concurrency 1 for another sample, or test whether deferring examples until late in coverage can reduce peak contention."}} +{"run":14,"commit":"98b6eec","metric":0,"metrics":{"packages_coverage_seconds":555,"examples_seconds":245,"startup_seconds":0,"ci_run_id":32483003057},"status":"crash","description":"Repeat single-example-task overlap configuration","timestamp":1787316745232,"segment":0,"confidence":6.229166666666667,"asi":{"hypothesis":"A second concurrency-1 sample would confirm its apparent lower overlap median.","result":"The Test job failed: postgres-codec-testkit aggregate defaults lost its database connection (`read ECONNRESET`, then `Client has encountered a connection error and is not queryable`), causing 7 failures. Examples completed in 245s; package coverage failed after 555s.","error_details":"Hosted-run log also showed react-router Postgres pool connection terminations. This matches the resource-starvation failure mode documented by the original 50% worker cap.","rollback_reason":"Parallel package coverage and example tests introduce unacceptable Postgres flakiness even at 75% package workers and example concurrency 1.","next_action_hint":"Abandon workload overlap, restore serial examples, retain 75% package workers, and collect repeated serial 75% measurements."}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1013cabc4ee..ac0e6125ad0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -204,33 +204,15 @@ jobs: - name: Start cloudflare-worker Postgres (5433, pg_stat_statements) if: needs.changes.outputs.inert != 'true' run: pnpm --filter prisma-8-cloudflare-worker db:up - - name: Test packages with coverage and examples + - name: Test packages with coverage if: needs.changes.outputs.inert != 'true' - run: | - set +e - run_timed() { - local metric="$1" - shift - local started_at="$SECONDS" - "$@" - local status="$?" - echo "CI_PHASE $metric=$((SECONDS - started_at))" - return "$status" - } - - run_timed packages_coverage_seconds pnpm coverage:packages & - coverage_pid="$!" - run_timed examples_seconds pnpm test:examples --concurrency=1 & - examples_pid="$!" - - wait "$coverage_pid" - coverage_status="$?" - wait "$examples_pid" - examples_status="$?" - ((coverage_status == 0 && examples_status == 0)) + run: pnpm coverage:packages - name: Report package coverage if: ${{ !cancelled() && needs.changes.outputs.inert != 'true' }} run: pnpm coverage:report + - name: Test examples + if: ${{ !cancelled() && needs.changes.outputs.inert != 'true' }} + run: pnpm test:examples - name: Check working tree is clean if: needs.changes.outputs.inert != 'true' run: pnpm check:clean-tree diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 8bcd8e839dc3..138f155d9fff 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -272,7 +272,7 @@ describe('coverage config', () => { assert.match(rootVitestConfig, /reportOnFailure:\s*true/); }); - it('runs package coverage and example tests concurrently in one CI job', async () => { + it('combines package tests and coverage in one CI job', async () => { const repositoryRoot = join(import.meta.dirname, '..'); const workflow = await readFile(join(repositoryRoot, '.github/workflows/ci.yml'), 'utf8'); const testJob = workflow.match(/\n {2}test:\n(?[\s\S]*?)(?=\n {2}test-e2e:\n)/)?.groups @@ -280,15 +280,16 @@ describe('coverage config', () => { assert.ok(testJob); assert.match(testJob, /^ {4}name: Test$/m); - assert.match(testJob, /- name: Test packages with coverage and examples/); - assert.match(testJob, /run_timed packages_coverage_seconds pnpm coverage:packages &/); - assert.match(testJob, /run_timed examples_seconds pnpm test:examples --concurrency=1 &/); assert.match( testJob, - /- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, + /run: pnpm coverage:packages\n {6}- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, + ); + assert.match( + testJob, + /- name: Test examples\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm test:examples/, ); assert.doesNotMatch(workflow, /\n {2}coverage:\n/); - assert.equal(workflow.match(/pnpm coverage:packages/g)?.length, 1); - assert.equal(workflow.match(/pnpm test:examples/g)?.length, 1); + assert.equal(workflow.match(/run: pnpm coverage:packages/g)?.length, 1); + assert.equal(workflow.match(/run: pnpm test:examples/g)?.length, 1); }); }); From f30dbd2fa3bf4dff41f515b6838a6ec78d770d43 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 13:07:12 +0000 Subject: [PATCH 17/80] autoresearch: measure Test CI 20260821T130712Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 3266492e285c..83bc9cab0854 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -13,3 +13,4 @@ {"run":12,"commit":"16bb937","metric":674,"metrics":{"packages_coverage_seconds":599,"examples_seconds":333,"startup_seconds":71,"ci_run_id":32480893962},"status":"discard","description":"Limit overlapping example Turbo tasks to one at a time","timestamp":1787315161828,"segment":0,"confidence":5.862745098039215,"asi":{"hypothesis":"Serializing example workspace tasks would free CPU for the coverage critical path while examples still finish before coverage.","result":"The job passed at 674s; examples expanded to 333s but still completed 266s before coverage at 599s. This is 29s below the uncapped 75%-overlap median, but one sample cannot resolve runner noise.","rollback_reason":"The single 674s point does not beat the retained absolute best and requires comparison with a moderate example concurrency.","next_action_hint":"Try --concurrency=2, which may reduce examples oversubscription without serializing all workspace tasks."}} {"run":13,"commit":"ef0c2a3","metric":763,"metrics":{"packages_coverage_seconds":686,"examples_seconds":246,"startup_seconds":70,"ci_run_id":32481888160},"status":"discard","description":"Limit overlapping example Turbo tasks to two at a time","timestamp":1787316006633,"segment":0,"confidence":6.229166666666667,"asi":{"hypothesis":"Two concurrent example workspace tasks could balance example completion against package-coverage CPU needs.","result":"The job passed but took 763s; coverage was the slowest overlap sample at 686s while examples took 246s. Moderate Turbo concurrency did not produce a useful balance.","rollback_reason":"Primary metric regressed 89s versus example concurrency 1 and 60s versus uncapped 75%-overlap median.","next_action_hint":"Return to concurrency 1 for another sample, or test whether deferring examples until late in coverage can reduce peak contention."}} {"run":14,"commit":"98b6eec","metric":0,"metrics":{"packages_coverage_seconds":555,"examples_seconds":245,"startup_seconds":0,"ci_run_id":32483003057},"status":"crash","description":"Repeat single-example-task overlap configuration","timestamp":1787316745232,"segment":0,"confidence":6.229166666666667,"asi":{"hypothesis":"A second concurrency-1 sample would confirm its apparent lower overlap median.","result":"The Test job failed: postgres-codec-testkit aggregate defaults lost its database connection (`read ECONNRESET`, then `Client has encountered a connection error and is not queryable`), causing 7 failures. Examples completed in 245s; package coverage failed after 555s.","error_details":"Hosted-run log also showed react-router Postgres pool connection terminations. This matches the resource-starvation failure mode documented by the original 50% worker cap.","rollback_reason":"Parallel package coverage and example tests introduce unacceptable Postgres flakiness even at 75% package workers and example concurrency 1.","next_action_hint":"Abandon workload overlap, restore serial examples, retain 75% package workers, and collect repeated serial 75% measurements."}} +{"run":15,"commit":"8ff22aa","metric":773,"metrics":{"packages_coverage_seconds":585,"examples_seconds":111,"startup_seconds":69,"ci_run_id":32484027897},"status":"keep","description":"Restore serial workloads and retain 75% package Vitest workers after overlap flake","timestamp":1787317628269,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"Serializing examples again removes database contention while three Vitest workers preserve the safe package-coverage speedup.","result":"The full Test job passed in 773s, 74s (8.7%) below the 847s baseline. Package coverage took 585s versus baseline 668s; examples returned to 111s.","correctness":"This removes the overlap configuration that produced ECONNRESET and returns to the established step-level failure semantics.","next_action_hint":"Repeat serial 75% once more for stability, then investigate coverage-specific settings that do not add concurrent database load."}} From dea3311351cdaa71b8e9dbd7027a18953177db93 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 13:22:23 +0000 Subject: [PATCH 18/80] autoresearch: measure Test CI 20260821T132223Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 83bc9cab0854..22dbd70fabd7 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -14,3 +14,4 @@ {"run":13,"commit":"ef0c2a3","metric":763,"metrics":{"packages_coverage_seconds":686,"examples_seconds":246,"startup_seconds":70,"ci_run_id":32481888160},"status":"discard","description":"Limit overlapping example Turbo tasks to two at a time","timestamp":1787316006633,"segment":0,"confidence":6.229166666666667,"asi":{"hypothesis":"Two concurrent example workspace tasks could balance example completion against package-coverage CPU needs.","result":"The job passed but took 763s; coverage was the slowest overlap sample at 686s while examples took 246s. Moderate Turbo concurrency did not produce a useful balance.","rollback_reason":"Primary metric regressed 89s versus example concurrency 1 and 60s versus uncapped 75%-overlap median.","next_action_hint":"Return to concurrency 1 for another sample, or test whether deferring examples until late in coverage can reduce peak contention."}} {"run":14,"commit":"98b6eec","metric":0,"metrics":{"packages_coverage_seconds":555,"examples_seconds":245,"startup_seconds":0,"ci_run_id":32483003057},"status":"crash","description":"Repeat single-example-task overlap configuration","timestamp":1787316745232,"segment":0,"confidence":6.229166666666667,"asi":{"hypothesis":"A second concurrency-1 sample would confirm its apparent lower overlap median.","result":"The Test job failed: postgres-codec-testkit aggregate defaults lost its database connection (`read ECONNRESET`, then `Client has encountered a connection error and is not queryable`), causing 7 failures. Examples completed in 245s; package coverage failed after 555s.","error_details":"Hosted-run log also showed react-router Postgres pool connection terminations. This matches the resource-starvation failure mode documented by the original 50% worker cap.","rollback_reason":"Parallel package coverage and example tests introduce unacceptable Postgres flakiness even at 75% package workers and example concurrency 1.","next_action_hint":"Abandon workload overlap, restore serial examples, retain 75% package workers, and collect repeated serial 75% measurements."}} {"run":15,"commit":"8ff22aa","metric":773,"metrics":{"packages_coverage_seconds":585,"examples_seconds":111,"startup_seconds":69,"ci_run_id":32484027897},"status":"keep","description":"Restore serial workloads and retain 75% package Vitest workers after overlap flake","timestamp":1787317628269,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"Serializing examples again removes database contention while three Vitest workers preserve the safe package-coverage speedup.","result":"The full Test job passed in 773s, 74s (8.7%) below the 847s baseline. Package coverage took 585s versus baseline 668s; examples returned to 111s.","correctness":"This removes the overlap configuration that produced ECONNRESET and returns to the established step-level failure semantics.","next_action_hint":"Repeat serial 75% once more for stability, then investigate coverage-specific settings that do not add concurrent database load."}} +{"run":16,"commit":"f30dbd2","metric":0,"metrics":{"packages_coverage_seconds":599,"examples_seconds":112,"startup_seconds":65,"ci_run_id":32485225250},"status":"crash","description":"Repeat serial 75%-worker configuration for stability","timestamp":1787318531654,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"A repeated serial 75% run would confirm the worker-count speedup without the overlap flake.","result":"All 1,164 files and 15,364 tests passed, but Vitest failed the phase because language-server/server.test.ts produced an unhandled `Connection is disposed` rejection after its final test. Coverage reporting and examples still passed.","error_details":"Unlike the overlap failure, this was not a Postgres connection loss. It is an asynchronous language-server teardown race and may be an existing suite flake, but increased file concurrency could increase its likelihood.","rollback_reason":"A speedup that raises CI flake risk is not acceptable without distinguishing or fixing the teardown race.","next_action_hint":"Test isolate:false at the safe 50% worker cap, using Vitest's own report that 998 worker spawns cost at least ~43.75s; shared-state failures will reveal whether it is viable."}} diff --git a/vitest.config.ts b/vitest.config.ts index 8373da70ae72..eb1bc230dd03 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,9 +9,10 @@ export default defineConfig({ projects: ['packages/**/vitest.config.ts'], // Cap fork concurrency on CI so the PGlite-WASM-heavy package suites // (cli, sql runtime, postgres/supabase extensions, postgres adapter + - // driver) don't all peak at once. Leave one of the four hosted-runner - // cores for the concurrently running example tests and Postgres container. - maxWorkers: process.env['CI'] ? '75%' : undefined, + // driver) don't all peak at once. Uncapped, several CPU-hungry PGlite + // forks plus the Postgres service can oversubscribe the runner. + maxWorkers: process.env['CI'] ? '50%' : undefined, + isolate: process.env['CI'] ? false : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 7817d7ae5155bf79060607766ea72a60e4d9e686 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 13:37:29 +0000 Subject: [PATCH 19/80] autoresearch: measure Test CI 20260821T133729Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 22dbd70fabd7..d1a03f2a4ba2 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -15,3 +15,4 @@ {"run":14,"commit":"98b6eec","metric":0,"metrics":{"packages_coverage_seconds":555,"examples_seconds":245,"startup_seconds":0,"ci_run_id":32483003057},"status":"crash","description":"Repeat single-example-task overlap configuration","timestamp":1787316745232,"segment":0,"confidence":6.229166666666667,"asi":{"hypothesis":"A second concurrency-1 sample would confirm its apparent lower overlap median.","result":"The Test job failed: postgres-codec-testkit aggregate defaults lost its database connection (`read ECONNRESET`, then `Client has encountered a connection error and is not queryable`), causing 7 failures. Examples completed in 245s; package coverage failed after 555s.","error_details":"Hosted-run log also showed react-router Postgres pool connection terminations. This matches the resource-starvation failure mode documented by the original 50% worker cap.","rollback_reason":"Parallel package coverage and example tests introduce unacceptable Postgres flakiness even at 75% package workers and example concurrency 1.","next_action_hint":"Abandon workload overlap, restore serial examples, retain 75% package workers, and collect repeated serial 75% measurements."}} {"run":15,"commit":"8ff22aa","metric":773,"metrics":{"packages_coverage_seconds":585,"examples_seconds":111,"startup_seconds":69,"ci_run_id":32484027897},"status":"keep","description":"Restore serial workloads and retain 75% package Vitest workers after overlap flake","timestamp":1787317628269,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"Serializing examples again removes database contention while three Vitest workers preserve the safe package-coverage speedup.","result":"The full Test job passed in 773s, 74s (8.7%) below the 847s baseline. Package coverage took 585s versus baseline 668s; examples returned to 111s.","correctness":"This removes the overlap configuration that produced ECONNRESET and returns to the established step-level failure semantics.","next_action_hint":"Repeat serial 75% once more for stability, then investigate coverage-specific settings that do not add concurrent database load."}} {"run":16,"commit":"f30dbd2","metric":0,"metrics":{"packages_coverage_seconds":599,"examples_seconds":112,"startup_seconds":65,"ci_run_id":32485225250},"status":"crash","description":"Repeat serial 75%-worker configuration for stability","timestamp":1787318531654,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"A repeated serial 75% run would confirm the worker-count speedup without the overlap flake.","result":"All 1,164 files and 15,364 tests passed, but Vitest failed the phase because language-server/server.test.ts produced an unhandled `Connection is disposed` rejection after its final test. Coverage reporting and examples still passed.","error_details":"Unlike the overlap failure, this was not a Postgres connection loss. It is an asynchronous language-server teardown race and may be an existing suite flake, but increased file concurrency could increase its likelihood.","rollback_reason":"A speedup that raises CI flake risk is not acceptable without distinguishing or fixing the teardown race.","next_action_hint":"Test isolate:false at the safe 50% worker cap, using Vitest's own report that 998 worker spawns cost at least ~43.75s; shared-state failures will reveal whether it is viable."}} +{"run":17,"commit":"dea3311","metric":815,"metrics":{"packages_coverage_seconds":641,"examples_seconds":109,"startup_seconds":60,"ci_run_id":32486508106},"status":"keep","description":"Reuse CI Vitest workers across files at the safe 50% concurrency cap","timestamp":1787319442925,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"Disabling per-file worker isolation should eliminate most of the 998 worker spawns Vitest reported while retaining the stable two-worker resource cap.","result":"The complete job passed in 815s, 32s below baseline; package coverage improved from 668s to 641s. All tests, type tests, coverage thresholds/reporting, examples, and clean-tree checks passed.","risk":"Worker reuse can expose shared module/process state between files, so repeated runs are required even though this run was clean.","next_action_hint":"Repeat isolate:false at 50% to test stability and timing."}} From 3263f081ead14f7e17ae607ed985ff581235ade6 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 13:52:50 +0000 Subject: [PATCH 20/80] autoresearch: measure Test CI 20260821T135250Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index d1a03f2a4ba2..913c92851589 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -16,3 +16,4 @@ {"run":15,"commit":"8ff22aa","metric":773,"metrics":{"packages_coverage_seconds":585,"examples_seconds":111,"startup_seconds":69,"ci_run_id":32484027897},"status":"keep","description":"Restore serial workloads and retain 75% package Vitest workers after overlap flake","timestamp":1787317628269,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"Serializing examples again removes database contention while three Vitest workers preserve the safe package-coverage speedup.","result":"The full Test job passed in 773s, 74s (8.7%) below the 847s baseline. Package coverage took 585s versus baseline 668s; examples returned to 111s.","correctness":"This removes the overlap configuration that produced ECONNRESET and returns to the established step-level failure semantics.","next_action_hint":"Repeat serial 75% once more for stability, then investigate coverage-specific settings that do not add concurrent database load."}} {"run":16,"commit":"f30dbd2","metric":0,"metrics":{"packages_coverage_seconds":599,"examples_seconds":112,"startup_seconds":65,"ci_run_id":32485225250},"status":"crash","description":"Repeat serial 75%-worker configuration for stability","timestamp":1787318531654,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"A repeated serial 75% run would confirm the worker-count speedup without the overlap flake.","result":"All 1,164 files and 15,364 tests passed, but Vitest failed the phase because language-server/server.test.ts produced an unhandled `Connection is disposed` rejection after its final test. Coverage reporting and examples still passed.","error_details":"Unlike the overlap failure, this was not a Postgres connection loss. It is an asynchronous language-server teardown race and may be an existing suite flake, but increased file concurrency could increase its likelihood.","rollback_reason":"A speedup that raises CI flake risk is not acceptable without distinguishing or fixing the teardown race.","next_action_hint":"Test isolate:false at the safe 50% worker cap, using Vitest's own report that 998 worker spawns cost at least ~43.75s; shared-state failures will reveal whether it is viable."}} {"run":17,"commit":"dea3311","metric":815,"metrics":{"packages_coverage_seconds":641,"examples_seconds":109,"startup_seconds":60,"ci_run_id":32486508106},"status":"keep","description":"Reuse CI Vitest workers across files at the safe 50% concurrency cap","timestamp":1787319442925,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"Disabling per-file worker isolation should eliminate most of the 998 worker spawns Vitest reported while retaining the stable two-worker resource cap.","result":"The complete job passed in 815s, 32s below baseline; package coverage improved from 668s to 641s. All tests, type tests, coverage thresholds/reporting, examples, and clean-tree checks passed.","risk":"Worker reuse can expose shared module/process state between files, so repeated runs are required even though this run was clean.","next_action_hint":"Repeat isolate:false at 50% to test stability and timing."}} +{"run":18,"commit":"7817d7a","metric":836,"metrics":{"packages_coverage_seconds":661,"examples_seconds":112,"startup_seconds":57,"ci_run_id":32487830559},"status":"discard","description":"Repeat 50%-worker reuse configuration","timestamp":1787320362900,"segment":0,"confidence":8.191780821917808,"asi":{"hypothesis":"A second isolate:false run at 50% would confirm the spawn-overhead improvement without shared-state failures.","result":"The run passed in 836s. Two samples are 815s and 836s (median 825.5s), only 21.5s below the 847s baseline and within broader hosted-runner variability.","rollback_reason":"The repeated point did not improve the retained 815s result; the small apparent gain does not yet justify global loss of file isolation.","next_action_hint":"Try isolate:false with 75% workers to see whether reuse also prevents the language-server teardown race while retaining higher throughput."}} diff --git a/vitest.config.ts b/vitest.config.ts index eb1bc230dd03..69aac52c1c55 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,7 +11,7 @@ export default defineConfig({ // (cli, sql runtime, postgres/supabase extensions, postgres adapter + // driver) don't all peak at once. Uncapped, several CPU-hungry PGlite // forks plus the Postgres service can oversubscribe the runner. - maxWorkers: process.env['CI'] ? '50%' : undefined, + maxWorkers: process.env['CI'] ? '75%' : undefined, isolate: process.env['CI'] ? false : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender From 31196faf1ffa1c994ba07bfe266616c3ad696d8d Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 14:07:34 +0000 Subject: [PATCH 21/80] autoresearch: measure Test CI 20260821T140734Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 913c92851589..775dc3127ceb 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -17,3 +17,4 @@ {"run":16,"commit":"f30dbd2","metric":0,"metrics":{"packages_coverage_seconds":599,"examples_seconds":112,"startup_seconds":65,"ci_run_id":32485225250},"status":"crash","description":"Repeat serial 75%-worker configuration for stability","timestamp":1787318531654,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"A repeated serial 75% run would confirm the worker-count speedup without the overlap flake.","result":"All 1,164 files and 15,364 tests passed, but Vitest failed the phase because language-server/server.test.ts produced an unhandled `Connection is disposed` rejection after its final test. Coverage reporting and examples still passed.","error_details":"Unlike the overlap failure, this was not a Postgres connection loss. It is an asynchronous language-server teardown race and may be an existing suite flake, but increased file concurrency could increase its likelihood.","rollback_reason":"A speedup that raises CI flake risk is not acceptable without distinguishing or fixing the teardown race.","next_action_hint":"Test isolate:false at the safe 50% worker cap, using Vitest's own report that 998 worker spawns cost at least ~43.75s; shared-state failures will reveal whether it is viable."}} {"run":17,"commit":"dea3311","metric":815,"metrics":{"packages_coverage_seconds":641,"examples_seconds":109,"startup_seconds":60,"ci_run_id":32486508106},"status":"keep","description":"Reuse CI Vitest workers across files at the safe 50% concurrency cap","timestamp":1787319442925,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"Disabling per-file worker isolation should eliminate most of the 998 worker spawns Vitest reported while retaining the stable two-worker resource cap.","result":"The complete job passed in 815s, 32s below baseline; package coverage improved from 668s to 641s. All tests, type tests, coverage thresholds/reporting, examples, and clean-tree checks passed.","risk":"Worker reuse can expose shared module/process state between files, so repeated runs are required even though this run was clean.","next_action_hint":"Repeat isolate:false at 50% to test stability and timing."}} {"run":18,"commit":"7817d7a","metric":836,"metrics":{"packages_coverage_seconds":661,"examples_seconds":112,"startup_seconds":57,"ci_run_id":32487830559},"status":"discard","description":"Repeat 50%-worker reuse configuration","timestamp":1787320362900,"segment":0,"confidence":8.191780821917808,"asi":{"hypothesis":"A second isolate:false run at 50% would confirm the spawn-overhead improvement without shared-state failures.","result":"The run passed in 836s. Two samples are 815s and 836s (median 825.5s), only 21.5s below the 847s baseline and within broader hosted-runner variability.","rollback_reason":"The repeated point did not improve the retained 815s result; the small apparent gain does not yet justify global loss of file isolation.","next_action_hint":"Try isolate:false with 75% workers to see whether reuse also prevents the language-server teardown race while retaining higher throughput."}} +{"run":19,"commit":"3263f08","metric":802,"metrics":{"packages_coverage_seconds":613,"examples_seconds":116,"startup_seconds":67,"ci_run_id":32489173159},"status":"keep","description":"Combine three CI workers with cross-file worker reuse","timestamp":1787321250384,"segment":0,"confidence":9.64516129032258,"asi":{"hypothesis":"Worker reuse at 75% could preserve higher throughput and avoid the prior per-file teardown rejection.","result":"The job passed in 802s, 45s below baseline and 13s below the best 50%-reuse sample. Coverage took 613s. No language-server or database error occurred.","risk":"Global isolate:false remains a semantic tradeoff and this timing is slower than successful isolate:true 75% samples.","next_action_hint":"Repeat for stability; discard isolate:false if it does not demonstrate a clear repeatable advantage over simpler 75% isolation."}} From 5d856e21e65487356d55c3362839446e67124f8b Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 14:24:19 +0000 Subject: [PATCH 22/80] autoresearch: measure Test CI 20260821T142418Z Signed-off-by: Steven McClankerton --- .auto/ideas.md | 1 + .auto/log.jsonl | 1 + .auto/prompt.md | 4 ++++ scripts/coverage-config.test.mjs | 1 + vitest.config.ts | 5 ++--- 5 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.auto/ideas.md b/.auto/ideas.md index e09ad388bd89..df5457fa5c6f 100644 --- a/.auto/ideas.md +++ b/.auto/ideas.md @@ -5,3 +5,4 @@ - Evaluate splitting package coverage and example tests into concurrent processes inside the same job only if CPU/memory contention does not erase the wall-clock gain; preserve a single job and all checks. - Explore safe coverage sharding plus Istanbul JSON merge if one Vitest coordinator cannot keep the runner busy, but do not weaken per-package coverage ownership or thresholds. - Determine whether the cloudflare-worker Postgres startup can overlap dependency linking/build setup without changing readiness or teardown guarantees. +- Selectively disable file isolation only in demonstrably stateless projects (high-file-count candidates: cli, sql-orm-client, framework-components, sql-contract-ts, migration). Global `isolate: false` passed twice but violates Supabase's explicit per-file isolation requirement and is too broad to ship. diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 775dc3127ceb..cd0621f04849 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -18,3 +18,4 @@ {"run":17,"commit":"dea3311","metric":815,"metrics":{"packages_coverage_seconds":641,"examples_seconds":109,"startup_seconds":60,"ci_run_id":32486508106},"status":"keep","description":"Reuse CI Vitest workers across files at the safe 50% concurrency cap","timestamp":1787319442925,"segment":0,"confidence":7.119047619047619,"asi":{"hypothesis":"Disabling per-file worker isolation should eliminate most of the 998 worker spawns Vitest reported while retaining the stable two-worker resource cap.","result":"The complete job passed in 815s, 32s below baseline; package coverage improved from 668s to 641s. All tests, type tests, coverage thresholds/reporting, examples, and clean-tree checks passed.","risk":"Worker reuse can expose shared module/process state between files, so repeated runs are required even though this run was clean.","next_action_hint":"Repeat isolate:false at 50% to test stability and timing."}} {"run":18,"commit":"7817d7a","metric":836,"metrics":{"packages_coverage_seconds":661,"examples_seconds":112,"startup_seconds":57,"ci_run_id":32487830559},"status":"discard","description":"Repeat 50%-worker reuse configuration","timestamp":1787320362900,"segment":0,"confidence":8.191780821917808,"asi":{"hypothesis":"A second isolate:false run at 50% would confirm the spawn-overhead improvement without shared-state failures.","result":"The run passed in 836s. Two samples are 815s and 836s (median 825.5s), only 21.5s below the 847s baseline and within broader hosted-runner variability.","rollback_reason":"The repeated point did not improve the retained 815s result; the small apparent gain does not yet justify global loss of file isolation.","next_action_hint":"Try isolate:false with 75% workers to see whether reuse also prevents the language-server teardown race while retaining higher throughput."}} {"run":19,"commit":"3263f08","metric":802,"metrics":{"packages_coverage_seconds":613,"examples_seconds":116,"startup_seconds":67,"ci_run_id":32489173159},"status":"keep","description":"Combine three CI workers with cross-file worker reuse","timestamp":1787321250384,"segment":0,"confidence":9.64516129032258,"asi":{"hypothesis":"Worker reuse at 75% could preserve higher throughput and avoid the prior per-file teardown rejection.","result":"The job passed in 802s, 45s below baseline and 13s below the best 50%-reuse sample. Coverage took 613s. No language-server or database error occurred.","risk":"Global isolate:false remains a semantic tradeoff and this timing is slower than successful isolate:true 75% samples.","next_action_hint":"Repeat for stability; discard isolate:false if it does not demonstrate a clear repeatable advantage over simpler 75% isolation."}} +{"run":20,"commit":"31196fa","metric":781,"metrics":{"packages_coverage_seconds":603,"examples_seconds":114,"startup_seconds":59,"ci_run_id":32490455956},"status":"keep","description":"Repeat three-worker CI with cross-file worker reuse","timestamp":1787322107513,"segment":0,"confidence":9.2,"asi":{"hypothesis":"A second 75%-reuse run would show whether the configuration is stable and repeatably faster than baseline.","result":"The job passed in 781s. Two samples are 802s and 781s (median 791.5s), with package coverage at 613s and 603s. No unhandled rejection or database loss occurred.","tradeoff":"The median is ~55s below baseline but only modestly different from 75% with isolation, while shared process state is a broader semantic change.","next_action_hint":"Investigate targeted isolate:false only for packages without stateful/database tests, or retain isolation and optimize worker spawning another way."}} diff --git a/.auto/prompt.md b/.auto/prompt.md index b3e8d124f198..dffb316e62c0 100644 --- a/.auto/prompt.md +++ b/.auto/prompt.md @@ -44,3 +44,7 @@ Reduce the wall-clock duration of the `Test` job in `.github/workflows/ci.yml` o - Before this session, package unit tests and package coverage were combined into one coverage-enabled Vitest pass, eliminating duplicate execution. That landed in PR #30082 and is the current baseline. - The root config currently caps CI workers at 50% to avoid oversubscribing PGlite-heavy suites and the Postgres service. Worker-count experiments are promising but must prove stability. - Historical predecessor-PR `Test` job durations were 723s, 799s, 828s, and 854s. Package coverage dominated at 549s, 608s, 612s, and 650s; example tests took 110–143s. +- Increasing serial package coverage from 50% to 75% workers produced successful 773s and 787s jobs, but a repeat ended with a language-server teardown rejection after every test passed. This was not a database failure, but the configuration needs caution. +- Running package coverage and examples concurrently looked fast but was rejected after a real `ECONNRESET`/non-queryable Postgres failure. Do not retry workload overlap. +- Full worker reuse (`isolate: false`) passed at 50% and 75%, but it is too broad: Supabase's config explicitly requires per-file isolation to prevent pg mocks leaking into integration files. Consider only targeted reuse in proven stateless projects. +- Hosted-runner timing has large outliers. Compare repeated medians and stability rather than trusting minima; a 548s overlap run was followed by 739s and 735s unchanged runs. diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 138f155d9fff..ea0df983a0b4 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -269,6 +269,7 @@ describe('coverage config', () => { assert.match(rootVitestConfig, /from ['"]\.\/scripts\/coverage-config['"]/); assert.doesNotMatch(rootVitestConfig, /from ['"]\.\/scripts\/coverage-config\.[^'"]+['"]/); assert.match(rootVitestConfig, /provider:\s*['"]v8['"]/); + assert.match(rootVitestConfig, /reporter:\s*\[['"]json['"]\]/); assert.match(rootVitestConfig, /reportOnFailure:\s*true/); }); diff --git a/vitest.config.ts b/vitest.config.ts index 69aac52c1c55..80e437429dd2 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,8 +11,7 @@ export default defineConfig({ // (cli, sql runtime, postgres/supabase extensions, postgres adapter + // driver) don't all peak at once. Uncapped, several CPU-hungry PGlite // forks plus the Postgres service can oversubscribe the runner. - maxWorkers: process.env['CI'] ? '75%' : undefined, - isolate: process.env['CI'] ? false : undefined, + maxWorkers: process.env['CI'] ? '50%' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. @@ -25,7 +24,7 @@ export default defineConfig({ coverage: { provider: 'v8', reportsDirectory: resolve(import.meta.dirname, 'coverage'), - reporter: ['text', 'json'], + reporter: ['json'], reportOnFailure: true, ...coveragePolicy, }, From 1d7d53c764b14606b3a79d1e8de8bb11ea6d6aa5 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 14:38:01 +0000 Subject: [PATCH 23/80] autoresearch: measure Test CI 20260821T143801Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index cd0621f04849..2c340a966186 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -19,3 +19,4 @@ {"run":18,"commit":"7817d7a","metric":836,"metrics":{"packages_coverage_seconds":661,"examples_seconds":112,"startup_seconds":57,"ci_run_id":32487830559},"status":"discard","description":"Repeat 50%-worker reuse configuration","timestamp":1787320362900,"segment":0,"confidence":8.191780821917808,"asi":{"hypothesis":"A second isolate:false run at 50% would confirm the spawn-overhead improvement without shared-state failures.","result":"The run passed in 836s. Two samples are 815s and 836s (median 825.5s), only 21.5s below the 847s baseline and within broader hosted-runner variability.","rollback_reason":"The repeated point did not improve the retained 815s result; the small apparent gain does not yet justify global loss of file isolation.","next_action_hint":"Try isolate:false with 75% workers to see whether reuse also prevents the language-server teardown race while retaining higher throughput."}} {"run":19,"commit":"3263f08","metric":802,"metrics":{"packages_coverage_seconds":613,"examples_seconds":116,"startup_seconds":67,"ci_run_id":32489173159},"status":"keep","description":"Combine three CI workers with cross-file worker reuse","timestamp":1787321250384,"segment":0,"confidence":9.64516129032258,"asi":{"hypothesis":"Worker reuse at 75% could preserve higher throughput and avoid the prior per-file teardown rejection.","result":"The job passed in 802s, 45s below baseline and 13s below the best 50%-reuse sample. Coverage took 613s. No language-server or database error occurred.","risk":"Global isolate:false remains a semantic tradeoff and this timing is slower than successful isolate:true 75% samples.","next_action_hint":"Repeat for stability; discard isolate:false if it does not demonstrate a clear repeatable advantage over simpler 75% isolation."}} {"run":20,"commit":"31196fa","metric":781,"metrics":{"packages_coverage_seconds":603,"examples_seconds":114,"startup_seconds":59,"ci_run_id":32490455956},"status":"keep","description":"Repeat three-worker CI with cross-file worker reuse","timestamp":1787322107513,"segment":0,"confidence":9.2,"asi":{"hypothesis":"A second 75%-reuse run would show whether the configuration is stable and repeatably faster than baseline.","result":"The job passed in 781s. Two samples are 802s and 781s (median 791.5s), with package coverage at 613s and 603s. No unhandled rejection or database loss occurred.","tradeoff":"The median is ~55s below baseline but only modestly different from 75% with isolation, while shared process state is a broader semantic change.","next_action_hint":"Investigate targeted isolate:false only for packages without stateful/database tests, or retain isolation and optimize worker spawning another way."}} +{"run":21,"commit":"5d856e2","metric":742,"metrics":{"packages_coverage_seconds":552,"examples_seconds":113,"startup_seconds":71,"ci_run_id":32491971312},"status":"keep","description":"Drop redundant text coverage reporter at the safe 50% worker cap","timestamp":1787323078051,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"The custom JSON coverage gate makes Vitest's huge text table redundant; emitting only JSON should reduce report generation/log transport while restoring safe isolation and worker count.","result":"The complete job passed in 742s, 105s below baseline; package coverage took 552s. The magnitude exceeds expected reporter savings and likely includes favorable runner variance.","correctness":"Coverage JSON was generated, the custom per-package report passed, examples passed, and file isolation plus the original 50% resource cap were restored.","next_action_hint":"Repeat unchanged to estimate the true reporter-only gain."}} From 37f1ad0bba837466f1d41a56591d2ce10f652ad0 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 14:55:14 +0000 Subject: [PATCH 24/80] autoresearch: measure Test CI 20260821T145514Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 2c340a966186..15616ea23863 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -20,3 +20,4 @@ {"run":19,"commit":"3263f08","metric":802,"metrics":{"packages_coverage_seconds":613,"examples_seconds":116,"startup_seconds":67,"ci_run_id":32489173159},"status":"keep","description":"Combine three CI workers with cross-file worker reuse","timestamp":1787321250384,"segment":0,"confidence":9.64516129032258,"asi":{"hypothesis":"Worker reuse at 75% could preserve higher throughput and avoid the prior per-file teardown rejection.","result":"The job passed in 802s, 45s below baseline and 13s below the best 50%-reuse sample. Coverage took 613s. No language-server or database error occurred.","risk":"Global isolate:false remains a semantic tradeoff and this timing is slower than successful isolate:true 75% samples.","next_action_hint":"Repeat for stability; discard isolate:false if it does not demonstrate a clear repeatable advantage over simpler 75% isolation."}} {"run":20,"commit":"31196fa","metric":781,"metrics":{"packages_coverage_seconds":603,"examples_seconds":114,"startup_seconds":59,"ci_run_id":32490455956},"status":"keep","description":"Repeat three-worker CI with cross-file worker reuse","timestamp":1787322107513,"segment":0,"confidence":9.2,"asi":{"hypothesis":"A second 75%-reuse run would show whether the configuration is stable and repeatably faster than baseline.","result":"The job passed in 781s. Two samples are 802s and 781s (median 791.5s), with package coverage at 613s and 603s. No unhandled rejection or database loss occurred.","tradeoff":"The median is ~55s below baseline but only modestly different from 75% with isolation, while shared process state is a broader semantic change.","next_action_hint":"Investigate targeted isolate:false only for packages without stateful/database tests, or retain isolation and optimize worker spawning another way."}} {"run":21,"commit":"5d856e2","metric":742,"metrics":{"packages_coverage_seconds":552,"examples_seconds":113,"startup_seconds":71,"ci_run_id":32491971312},"status":"keep","description":"Drop redundant text coverage reporter at the safe 50% worker cap","timestamp":1787323078051,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"The custom JSON coverage gate makes Vitest's huge text table redundant; emitting only JSON should reduce report generation/log transport while restoring safe isolation and worker count.","result":"The complete job passed in 742s, 105s below baseline; package coverage took 552s. The magnitude exceeds expected reporter savings and likely includes favorable runner variance.","correctness":"Coverage JSON was generated, the custom per-package report passed, examples passed, and file isolation plus the original 50% resource cap were restored.","next_action_hint":"Repeat unchanged to estimate the true reporter-only gain."}} +{"run":22,"commit":"1d7d53c","metric":945,"metrics":{"packages_coverage_seconds":751,"examples_seconds":127,"startup_seconds":61,"ci_run_id":32493202956},"status":"discard","description":"Repeat JSON-only coverage reporter configuration","timestamp":1787324087263,"segment":0,"confidence":9.2,"asi":{"hypothesis":"A repeat would show a small consistent gain from avoiding text coverage rendering/logging.","result":"The unchanged run passed but took 945s, with coverage at 751s. Together with 742s, this confirms extreme hosted-runner variance and does not quantify a meaningful reporter-only speedup.","rollback_reason":"The repeated primary metric is 98s worse than baseline; no code delta exists in this measurement commit.","next_action_hint":"Retain JSON-only only if simplification is independently worthwhile; pursue structural savings with less sensitivity to runner speed."}} diff --git a/vitest.config.ts b/vitest.config.ts index 80e437429dd2..bcea59f8cc06 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -12,6 +12,7 @@ export default defineConfig({ // driver) don't all peak at once. Uncapped, several CPU-hungry PGlite // forks plus the Postgres service can oversubscribe the runner. maxWorkers: process.env['CI'] ? '50%' : undefined, + pool: process.env['CI'] ? 'vmForks' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 20443e364c411421fe6c703143460140c43386b5 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 15:11:47 +0000 Subject: [PATCH 25/80] autoresearch: measure Test CI 20260821T151146Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 15616ea23863..355aa6297df0 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -21,3 +21,4 @@ {"run":20,"commit":"31196fa","metric":781,"metrics":{"packages_coverage_seconds":603,"examples_seconds":114,"startup_seconds":59,"ci_run_id":32490455956},"status":"keep","description":"Repeat three-worker CI with cross-file worker reuse","timestamp":1787322107513,"segment":0,"confidence":9.2,"asi":{"hypothesis":"A second 75%-reuse run would show whether the configuration is stable and repeatably faster than baseline.","result":"The job passed in 781s. Two samples are 802s and 781s (median 791.5s), with package coverage at 613s and 603s. No unhandled rejection or database loss occurred.","tradeoff":"The median is ~55s below baseline but only modestly different from 75% with isolation, while shared process state is a broader semantic change.","next_action_hint":"Investigate targeted isolate:false only for packages without stateful/database tests, or retain isolation and optimize worker spawning another way."}} {"run":21,"commit":"5d856e2","metric":742,"metrics":{"packages_coverage_seconds":552,"examples_seconds":113,"startup_seconds":71,"ci_run_id":32491971312},"status":"keep","description":"Drop redundant text coverage reporter at the safe 50% worker cap","timestamp":1787323078051,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"The custom JSON coverage gate makes Vitest's huge text table redundant; emitting only JSON should reduce report generation/log transport while restoring safe isolation and worker count.","result":"The complete job passed in 742s, 105s below baseline; package coverage took 552s. The magnitude exceeds expected reporter savings and likely includes favorable runner variance.","correctness":"Coverage JSON was generated, the custom per-package report passed, examples passed, and file isolation plus the original 50% resource cap were restored.","next_action_hint":"Repeat unchanged to estimate the true reporter-only gain."}} {"run":22,"commit":"1d7d53c","metric":945,"metrics":{"packages_coverage_seconds":751,"examples_seconds":127,"startup_seconds":61,"ci_run_id":32493202956},"status":"discard","description":"Repeat JSON-only coverage reporter configuration","timestamp":1787324087263,"segment":0,"confidence":9.2,"asi":{"hypothesis":"A repeat would show a small consistent gain from avoiding text coverage rendering/logging.","result":"The unchanged run passed but took 945s, with coverage at 751s. Together with 742s, this confirms extreme hosted-runner variance and does not quantify a meaningful reporter-only speedup.","rollback_reason":"The repeated primary metric is 98s worse than baseline; no code delta exists in this measurement commit.","next_action_hint":"Retain JSON-only only if simplification is independently worthwhile; pursue structural savings with less sensitivity to runner speed."}} +{"run":23,"commit":"37f1ad0","metric":864,"metrics":{"packages_coverage_seconds":635,"examples_seconds":144,"startup_seconds":78,"ci_run_id":32494774140},"status":"discard","description":"Use VM-isolated fork workers at the safe 50% concurrency cap","timestamp":1787325084988,"segment":0,"confidence":8.08108108108108,"asi":{"hypothesis":"vmForks could reuse child processes while creating an isolated VM context per file, avoiding global shared state and most fork startup cost.","result":"All tests passed, but the complete job took 864s. Coverage was 635s; startup and examples were also slower than typical. No clear primary benefit appeared.","rollback_reason":"Primary metric regressed 17s versus baseline and 122s versus the best JSON-only safe run.","next_action_hint":"Try vmThreads only if cross-realm/native compatibility is acceptable; otherwise remove VM pooling and focus on CI step structure or coverage output."}} diff --git a/vitest.config.ts b/vitest.config.ts index bcea59f8cc06..a86dc310ea24 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -12,7 +12,7 @@ export default defineConfig({ // driver) don't all peak at once. Uncapped, several CPU-hungry PGlite // forks plus the Postgres service can oversubscribe the runner. maxWorkers: process.env['CI'] ? '50%' : undefined, - pool: process.env['CI'] ? 'vmForks' : undefined, + pool: process.env['CI'] ? 'vmThreads' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From f867c64b0fb24d4d098c2fab88a8cc7bf75d1d86 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 15:26:54 +0000 Subject: [PATCH 26/80] autoresearch: measure Test CI 20260821T152654Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 355aa6297df0..21bb90977e79 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -22,3 +22,4 @@ {"run":21,"commit":"5d856e2","metric":742,"metrics":{"packages_coverage_seconds":552,"examples_seconds":113,"startup_seconds":71,"ci_run_id":32491971312},"status":"keep","description":"Drop redundant text coverage reporter at the safe 50% worker cap","timestamp":1787323078051,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"The custom JSON coverage gate makes Vitest's huge text table redundant; emitting only JSON should reduce report generation/log transport while restoring safe isolation and worker count.","result":"The complete job passed in 742s, 105s below baseline; package coverage took 552s. The magnitude exceeds expected reporter savings and likely includes favorable runner variance.","correctness":"Coverage JSON was generated, the custom per-package report passed, examples passed, and file isolation plus the original 50% resource cap were restored.","next_action_hint":"Repeat unchanged to estimate the true reporter-only gain."}} {"run":22,"commit":"1d7d53c","metric":945,"metrics":{"packages_coverage_seconds":751,"examples_seconds":127,"startup_seconds":61,"ci_run_id":32493202956},"status":"discard","description":"Repeat JSON-only coverage reporter configuration","timestamp":1787324087263,"segment":0,"confidence":9.2,"asi":{"hypothesis":"A repeat would show a small consistent gain from avoiding text coverage rendering/logging.","result":"The unchanged run passed but took 945s, with coverage at 751s. Together with 742s, this confirms extreme hosted-runner variance and does not quantify a meaningful reporter-only speedup.","rollback_reason":"The repeated primary metric is 98s worse than baseline; no code delta exists in this measurement commit.","next_action_hint":"Retain JSON-only only if simplification is independently worthwhile; pursue structural savings with less sensitivity to runner speed."}} {"run":23,"commit":"37f1ad0","metric":864,"metrics":{"packages_coverage_seconds":635,"examples_seconds":144,"startup_seconds":78,"ci_run_id":32494774140},"status":"discard","description":"Use VM-isolated fork workers at the safe 50% concurrency cap","timestamp":1787325084988,"segment":0,"confidence":8.08108108108108,"asi":{"hypothesis":"vmForks could reuse child processes while creating an isolated VM context per file, avoiding global shared state and most fork startup cost.","result":"All tests passed, but the complete job took 864s. Coverage was 635s; startup and examples were also slower than typical. No clear primary benefit appeared.","rollback_reason":"Primary metric regressed 17s versus baseline and 122s versus the best JSON-only safe run.","next_action_hint":"Try vmThreads only if cross-realm/native compatibility is acceptable; otherwise remove VM pooling and focus on CI step structure or coverage output."}} +{"run":24,"commit":"20443e3","metric":819,"metrics":{"packages_coverage_seconds":641,"examples_seconds":113,"startup_seconds":58,"ci_run_id":32496300021},"status":"discard","description":"Use VM-isolated thread workers at the safe 50% concurrency cap","timestamp":1787326005771,"segment":0,"confidence":7.569620253164557,"asi":{"hypothesis":"vmThreads should preserve per-file isolation while avoiding per-file process spawn overhead more efficiently than vmForks.","result":"The complete job passed in 819s, 45s faster than vmForks and 28s below baseline. Coverage took 641s. No cross-realm or native/WASM failure appeared.","rollback_reason":"The single result did not beat the retained safe JSON-only minimum and is close to normal runner variation.","next_action_hint":"Repeat unchanged to test vmThreads stability and estimate its median."}} From 8bd6d7117eaeecf31d34166f64c97cc078f6bb1f Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 15:41:46 +0000 Subject: [PATCH 27/80] autoresearch: measure Test CI 20260821T154146Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 21bb90977e79..7425f68383e7 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -23,3 +23,4 @@ {"run":22,"commit":"1d7d53c","metric":945,"metrics":{"packages_coverage_seconds":751,"examples_seconds":127,"startup_seconds":61,"ci_run_id":32493202956},"status":"discard","description":"Repeat JSON-only coverage reporter configuration","timestamp":1787324087263,"segment":0,"confidence":9.2,"asi":{"hypothesis":"A repeat would show a small consistent gain from avoiding text coverage rendering/logging.","result":"The unchanged run passed but took 945s, with coverage at 751s. Together with 742s, this confirms extreme hosted-runner variance and does not quantify a meaningful reporter-only speedup.","rollback_reason":"The repeated primary metric is 98s worse than baseline; no code delta exists in this measurement commit.","next_action_hint":"Retain JSON-only only if simplification is independently worthwhile; pursue structural savings with less sensitivity to runner speed."}} {"run":23,"commit":"37f1ad0","metric":864,"metrics":{"packages_coverage_seconds":635,"examples_seconds":144,"startup_seconds":78,"ci_run_id":32494774140},"status":"discard","description":"Use VM-isolated fork workers at the safe 50% concurrency cap","timestamp":1787325084988,"segment":0,"confidence":8.08108108108108,"asi":{"hypothesis":"vmForks could reuse child processes while creating an isolated VM context per file, avoiding global shared state and most fork startup cost.","result":"All tests passed, but the complete job took 864s. Coverage was 635s; startup and examples were also slower than typical. No clear primary benefit appeared.","rollback_reason":"Primary metric regressed 17s versus baseline and 122s versus the best JSON-only safe run.","next_action_hint":"Try vmThreads only if cross-realm/native compatibility is acceptable; otherwise remove VM pooling and focus on CI step structure or coverage output."}} {"run":24,"commit":"20443e3","metric":819,"metrics":{"packages_coverage_seconds":641,"examples_seconds":113,"startup_seconds":58,"ci_run_id":32496300021},"status":"discard","description":"Use VM-isolated thread workers at the safe 50% concurrency cap","timestamp":1787326005771,"segment":0,"confidence":7.569620253164557,"asi":{"hypothesis":"vmThreads should preserve per-file isolation while avoiding per-file process spawn overhead more efficiently than vmForks.","result":"The complete job passed in 819s, 45s faster than vmForks and 28s below baseline. Coverage took 641s. No cross-realm or native/WASM failure appeared.","rollback_reason":"The single result did not beat the retained safe JSON-only minimum and is close to normal runner variation.","next_action_hint":"Repeat unchanged to test vmThreads stability and estimate its median."}} +{"run":25,"commit":"f867c64","metric":814,"metrics":{"packages_coverage_seconds":640,"examples_seconds":109,"startup_seconds":60,"ci_run_id":32497656915},"status":"keep","description":"Repeat VM-isolated thread workers at 50% CI concurrency","timestamp":1787326899628,"segment":0,"confidence":7.666666666666667,"asi":{"hypothesis":"A second vmThreads run would confirm stable isolated worker reuse.","result":"The job passed again in 814s, nearly identical to 819s; coverage was 640s versus 641s. The 816.5s median is 30.5s below baseline and materially steadier than default-fork hosted samples.","correctness":"vmThreads creates a fresh VM context per file, so file-level module/global isolation remains. Supabase explicitly overrides the root pool to forks for its WASM/mock requirements.","risk":"VM realms can change instanceof identity and memory behavior; two complete passes found no incompatibility, but a third pass is prudent.","next_action_hint":"Run a third vmThreads sample, then consider vmMemoryLimit or 75% vmThreads if stability remains clean."}} From 16f966cafd8316127629e8f0e5e9d28068763dbc Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 15:56:36 +0000 Subject: [PATCH 28/80] autoresearch: measure Test CI 20260821T155636Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 7425f68383e7..bc375a05758a 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -24,3 +24,4 @@ {"run":23,"commit":"37f1ad0","metric":864,"metrics":{"packages_coverage_seconds":635,"examples_seconds":144,"startup_seconds":78,"ci_run_id":32494774140},"status":"discard","description":"Use VM-isolated fork workers at the safe 50% concurrency cap","timestamp":1787325084988,"segment":0,"confidence":8.08108108108108,"asi":{"hypothesis":"vmForks could reuse child processes while creating an isolated VM context per file, avoiding global shared state and most fork startup cost.","result":"All tests passed, but the complete job took 864s. Coverage was 635s; startup and examples were also slower than typical. No clear primary benefit appeared.","rollback_reason":"Primary metric regressed 17s versus baseline and 122s versus the best JSON-only safe run.","next_action_hint":"Try vmThreads only if cross-realm/native compatibility is acceptable; otherwise remove VM pooling and focus on CI step structure or coverage output."}} {"run":24,"commit":"20443e3","metric":819,"metrics":{"packages_coverage_seconds":641,"examples_seconds":113,"startup_seconds":58,"ci_run_id":32496300021},"status":"discard","description":"Use VM-isolated thread workers at the safe 50% concurrency cap","timestamp":1787326005771,"segment":0,"confidence":7.569620253164557,"asi":{"hypothesis":"vmThreads should preserve per-file isolation while avoiding per-file process spawn overhead more efficiently than vmForks.","result":"The complete job passed in 819s, 45s faster than vmForks and 28s below baseline. Coverage took 641s. No cross-realm or native/WASM failure appeared.","rollback_reason":"The single result did not beat the retained safe JSON-only minimum and is close to normal runner variation.","next_action_hint":"Repeat unchanged to test vmThreads stability and estimate its median."}} {"run":25,"commit":"f867c64","metric":814,"metrics":{"packages_coverage_seconds":640,"examples_seconds":109,"startup_seconds":60,"ci_run_id":32497656915},"status":"keep","description":"Repeat VM-isolated thread workers at 50% CI concurrency","timestamp":1787326899628,"segment":0,"confidence":7.666666666666667,"asi":{"hypothesis":"A second vmThreads run would confirm stable isolated worker reuse.","result":"The job passed again in 814s, nearly identical to 819s; coverage was 640s versus 641s. The 816.5s median is 30.5s below baseline and materially steadier than default-fork hosted samples.","correctness":"vmThreads creates a fresh VM context per file, so file-level module/global isolation remains. Supabase explicitly overrides the root pool to forks for its WASM/mock requirements.","risk":"VM realms can change instanceof identity and memory behavior; two complete passes found no incompatibility, but a third pass is prudent.","next_action_hint":"Run a third vmThreads sample, then consider vmMemoryLimit or 75% vmThreads if stability remains clean."}} +{"run":26,"commit":"8bd6d71","metric":811,"metrics":{"packages_coverage_seconds":636,"examples_seconds":112,"startup_seconds":58,"ci_run_id":32499005740},"status":"keep","description":"Third VM-thread isolation stability run","timestamp":1787327787502,"segment":0,"confidence":7.666666666666667,"asi":{"hypothesis":"A third vmThreads pass would rule out immediate cross-realm, memory, and flake concerns.","result":"The third run passed in 811s with coverage at 636s. Three samples are 819/814/811 (median 814s, 8s range), the most stable candidate distribution in the session.","correctness":"All 1,164 files, 15,364 tests, type tests, JSON coverage collection, custom thresholds, examples, and clean-tree checks passed for a third time.","next_action_hint":"Test vmThreads at 75% workers; the VM pool may avoid process oversubscription while using the third core."}} diff --git a/vitest.config.ts b/vitest.config.ts index a86dc310ea24..198543189a4d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,7 +11,7 @@ export default defineConfig({ // (cli, sql runtime, postgres/supabase extensions, postgres adapter + // driver) don't all peak at once. Uncapped, several CPU-hungry PGlite // forks plus the Postgres service can oversubscribe the runner. - maxWorkers: process.env['CI'] ? '50%' : undefined, + maxWorkers: process.env['CI'] ? '75%' : undefined, pool: process.env['CI'] ? 'vmThreads' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender From 54687b856e496d7d20dc722c84d1c3c4860f7b9e Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 16:10:34 +0000 Subject: [PATCH 29/80] autoresearch: measure Test CI 20260821T161034Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index bc375a05758a..f6b6cec14fc5 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -25,3 +25,4 @@ {"run":24,"commit":"20443e3","metric":819,"metrics":{"packages_coverage_seconds":641,"examples_seconds":113,"startup_seconds":58,"ci_run_id":32496300021},"status":"discard","description":"Use VM-isolated thread workers at the safe 50% concurrency cap","timestamp":1787326005771,"segment":0,"confidence":7.569620253164557,"asi":{"hypothesis":"vmThreads should preserve per-file isolation while avoiding per-file process spawn overhead more efficiently than vmForks.","result":"The complete job passed in 819s, 45s faster than vmForks and 28s below baseline. Coverage took 641s. No cross-realm or native/WASM failure appeared.","rollback_reason":"The single result did not beat the retained safe JSON-only minimum and is close to normal runner variation.","next_action_hint":"Repeat unchanged to test vmThreads stability and estimate its median."}} {"run":25,"commit":"f867c64","metric":814,"metrics":{"packages_coverage_seconds":640,"examples_seconds":109,"startup_seconds":60,"ci_run_id":32497656915},"status":"keep","description":"Repeat VM-isolated thread workers at 50% CI concurrency","timestamp":1787326899628,"segment":0,"confidence":7.666666666666667,"asi":{"hypothesis":"A second vmThreads run would confirm stable isolated worker reuse.","result":"The job passed again in 814s, nearly identical to 819s; coverage was 640s versus 641s. The 816.5s median is 30.5s below baseline and materially steadier than default-fork hosted samples.","correctness":"vmThreads creates a fresh VM context per file, so file-level module/global isolation remains. Supabase explicitly overrides the root pool to forks for its WASM/mock requirements.","risk":"VM realms can change instanceof identity and memory behavior; two complete passes found no incompatibility, but a third pass is prudent.","next_action_hint":"Run a third vmThreads sample, then consider vmMemoryLimit or 75% vmThreads if stability remains clean."}} {"run":26,"commit":"8bd6d71","metric":811,"metrics":{"packages_coverage_seconds":636,"examples_seconds":112,"startup_seconds":58,"ci_run_id":32499005740},"status":"keep","description":"Third VM-thread isolation stability run","timestamp":1787327787502,"segment":0,"confidence":7.666666666666667,"asi":{"hypothesis":"A third vmThreads pass would rule out immediate cross-realm, memory, and flake concerns.","result":"The third run passed in 811s with coverage at 636s. Three samples are 819/814/811 (median 814s, 8s range), the most stable candidate distribution in the session.","correctness":"All 1,164 files, 15,364 tests, type tests, JSON coverage collection, custom thresholds, examples, and clean-tree checks passed for a third time.","next_action_hint":"Test vmThreads at 75% workers; the VM pool may avoid process oversubscription while using the third core."}} +{"run":27,"commit":"16f966c","metric":753,"metrics":{"packages_coverage_seconds":581,"examples_seconds":109,"startup_seconds":58,"ci_run_id":32500335101},"status":"keep","description":"Run VM-isolated CI tests with three workers","timestamp":1787328630035,"segment":0,"confidence":7.868421052631579,"asi":{"hypothesis":"A third VM-thread worker can improve throughput without the fork/process pressure that motivated the original 50% cap.","result":"The full job passed in 753s, 58s faster than the best 50%-vmThreads sample and 94s below baseline. Coverage took 581s; examples and startup were unchanged.","risk":"Higher concurrency previously exposed a language-server teardown race under fork isolation, so vmThreads 75% needs repeated stability proof.","next_action_hint":"Repeat 75%-vmThreads at least twice; reject on any unhandled rejection, database loss, or memory failure."}} From 160299b8e86f1bfd15c2181375f4d569230321ec Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 16:24:41 +0000 Subject: [PATCH 30/80] autoresearch: measure Test CI 20260821T162441Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index f6b6cec14fc5..9b0ace48b30b 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -26,3 +26,4 @@ {"run":25,"commit":"f867c64","metric":814,"metrics":{"packages_coverage_seconds":640,"examples_seconds":109,"startup_seconds":60,"ci_run_id":32497656915},"status":"keep","description":"Repeat VM-isolated thread workers at 50% CI concurrency","timestamp":1787326899628,"segment":0,"confidence":7.666666666666667,"asi":{"hypothesis":"A second vmThreads run would confirm stable isolated worker reuse.","result":"The job passed again in 814s, nearly identical to 819s; coverage was 640s versus 641s. The 816.5s median is 30.5s below baseline and materially steadier than default-fork hosted samples.","correctness":"vmThreads creates a fresh VM context per file, so file-level module/global isolation remains. Supabase explicitly overrides the root pool to forks for its WASM/mock requirements.","risk":"VM realms can change instanceof identity and memory behavior; two complete passes found no incompatibility, but a third pass is prudent.","next_action_hint":"Run a third vmThreads sample, then consider vmMemoryLimit or 75% vmThreads if stability remains clean."}} {"run":26,"commit":"8bd6d71","metric":811,"metrics":{"packages_coverage_seconds":636,"examples_seconds":112,"startup_seconds":58,"ci_run_id":32499005740},"status":"keep","description":"Third VM-thread isolation stability run","timestamp":1787327787502,"segment":0,"confidence":7.666666666666667,"asi":{"hypothesis":"A third vmThreads pass would rule out immediate cross-realm, memory, and flake concerns.","result":"The third run passed in 811s with coverage at 636s. Three samples are 819/814/811 (median 814s, 8s range), the most stable candidate distribution in the session.","correctness":"All 1,164 files, 15,364 tests, type tests, JSON coverage collection, custom thresholds, examples, and clean-tree checks passed for a third time.","next_action_hint":"Test vmThreads at 75% workers; the VM pool may avoid process oversubscription while using the third core."}} {"run":27,"commit":"16f966c","metric":753,"metrics":{"packages_coverage_seconds":581,"examples_seconds":109,"startup_seconds":58,"ci_run_id":32500335101},"status":"keep","description":"Run VM-isolated CI tests with three workers","timestamp":1787328630035,"segment":0,"confidence":7.868421052631579,"asi":{"hypothesis":"A third VM-thread worker can improve throughput without the fork/process pressure that motivated the original 50% cap.","result":"The full job passed in 753s, 58s faster than the best 50%-vmThreads sample and 94s below baseline. Coverage took 581s; examples and startup were unchanged.","risk":"Higher concurrency previously exposed a language-server teardown race under fork isolation, so vmThreads 75% needs repeated stability proof.","next_action_hint":"Repeat 75%-vmThreads at least twice; reject on any unhandled rejection, database loss, or memory failure."}} +{"run":28,"commit":"54687b8","metric":767,"metrics":{"packages_coverage_seconds":588,"examples_seconds":109,"startup_seconds":64,"ci_run_id":32501576368},"status":"discard","description":"Second three-worker VM-thread stability run","timestamp":1787329477381,"segment":0,"confidence":7.973333333333334,"asi":{"hypothesis":"The 75%-vmThreads configuration would repeat without the fork-pool teardown/resource failures.","result":"The second run passed in 767s; coverage took 588s. Two samples are 753s and 767s, both substantially below the 814s median at 50% vmThreads.","rollback_reason":"The repeated point is 14s slower than the retained 753s minimum; no code delta exists.","next_action_hint":"Collect a third stability sample before selecting 75%-vmThreads."}} From 8fb6db386c86c04de965bb6047575cf721edf1bf Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 16:39:13 +0000 Subject: [PATCH 31/80] autoresearch: measure Test CI 20260821T163913Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 9b0ace48b30b..b118077ed51c 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -27,3 +27,4 @@ {"run":26,"commit":"8bd6d71","metric":811,"metrics":{"packages_coverage_seconds":636,"examples_seconds":112,"startup_seconds":58,"ci_run_id":32499005740},"status":"keep","description":"Third VM-thread isolation stability run","timestamp":1787327787502,"segment":0,"confidence":7.666666666666667,"asi":{"hypothesis":"A third vmThreads pass would rule out immediate cross-realm, memory, and flake concerns.","result":"The third run passed in 811s with coverage at 636s. Three samples are 819/814/811 (median 814s, 8s range), the most stable candidate distribution in the session.","correctness":"All 1,164 files, 15,364 tests, type tests, JSON coverage collection, custom thresholds, examples, and clean-tree checks passed for a third time.","next_action_hint":"Test vmThreads at 75% workers; the VM pool may avoid process oversubscription while using the third core."}} {"run":27,"commit":"16f966c","metric":753,"metrics":{"packages_coverage_seconds":581,"examples_seconds":109,"startup_seconds":58,"ci_run_id":32500335101},"status":"keep","description":"Run VM-isolated CI tests with three workers","timestamp":1787328630035,"segment":0,"confidence":7.868421052631579,"asi":{"hypothesis":"A third VM-thread worker can improve throughput without the fork/process pressure that motivated the original 50% cap.","result":"The full job passed in 753s, 58s faster than the best 50%-vmThreads sample and 94s below baseline. Coverage took 581s; examples and startup were unchanged.","risk":"Higher concurrency previously exposed a language-server teardown race under fork isolation, so vmThreads 75% needs repeated stability proof.","next_action_hint":"Repeat 75%-vmThreads at least twice; reject on any unhandled rejection, database loss, or memory failure."}} {"run":28,"commit":"54687b8","metric":767,"metrics":{"packages_coverage_seconds":588,"examples_seconds":109,"startup_seconds":64,"ci_run_id":32501576368},"status":"discard","description":"Second three-worker VM-thread stability run","timestamp":1787329477381,"segment":0,"confidence":7.973333333333334,"asi":{"hypothesis":"The 75%-vmThreads configuration would repeat without the fork-pool teardown/resource failures.","result":"The second run passed in 767s; coverage took 588s. Two samples are 753s and 767s, both substantially below the 814s median at 50% vmThreads.","rollback_reason":"The repeated point is 14s slower than the retained 753s minimum; no code delta exists.","next_action_hint":"Collect a third stability sample before selecting 75%-vmThreads."}} +{"run":29,"commit":"160299b","metric":789,"metrics":{"packages_coverage_seconds":554,"examples_seconds":152,"startup_seconds":77,"ci_run_id":32502789397},"status":"discard","description":"Third three-worker VM-thread stability run","timestamp":1787330345499,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"A third 75%-vmThreads run would complete cleanly and establish a robust distribution.","result":"The third run passed in 789s. Coverage was fastest at 554s, but unrelated startup/examples noise raised total time. Three samples are 753/767/789 (median 767s), all successful.","rollback_reason":"The 789s point did not improve the retained minimum; no code delta exists.","next_action_hint":"75%-vmThreads now has three clean samples. Test 100% vmThreads as a boundary, then choose 75% unless four workers improve repeatably."}} diff --git a/vitest.config.ts b/vitest.config.ts index 198543189a4d..0eacb54cf6c0 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,7 +11,7 @@ export default defineConfig({ // (cli, sql runtime, postgres/supabase extensions, postgres adapter + // driver) don't all peak at once. Uncapped, several CPU-hungry PGlite // forks plus the Postgres service can oversubscribe the runner. - maxWorkers: process.env['CI'] ? '75%' : undefined, + maxWorkers: process.env['CI'] ? '100%' : undefined, pool: process.env['CI'] ? 'vmThreads' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender From f76df2b4ccd8f9fe4141cbe2d071d214b6882ab5 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 16:53:01 +0000 Subject: [PATCH 32/80] autoresearch: measure Test CI 20260821T165301Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .auto/measure.sh | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index b118077ed51c..cedd9ac5db80 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -28,3 +28,4 @@ {"run":27,"commit":"16f966c","metric":753,"metrics":{"packages_coverage_seconds":581,"examples_seconds":109,"startup_seconds":58,"ci_run_id":32500335101},"status":"keep","description":"Run VM-isolated CI tests with three workers","timestamp":1787328630035,"segment":0,"confidence":7.868421052631579,"asi":{"hypothesis":"A third VM-thread worker can improve throughput without the fork/process pressure that motivated the original 50% cap.","result":"The full job passed in 753s, 58s faster than the best 50%-vmThreads sample and 94s below baseline. Coverage took 581s; examples and startup were unchanged.","risk":"Higher concurrency previously exposed a language-server teardown race under fork isolation, so vmThreads 75% needs repeated stability proof.","next_action_hint":"Repeat 75%-vmThreads at least twice; reject on any unhandled rejection, database loss, or memory failure."}} {"run":28,"commit":"54687b8","metric":767,"metrics":{"packages_coverage_seconds":588,"examples_seconds":109,"startup_seconds":64,"ci_run_id":32501576368},"status":"discard","description":"Second three-worker VM-thread stability run","timestamp":1787329477381,"segment":0,"confidence":7.973333333333334,"asi":{"hypothesis":"The 75%-vmThreads configuration would repeat without the fork-pool teardown/resource failures.","result":"The second run passed in 767s; coverage took 588s. Two samples are 753s and 767s, both substantially below the 814s median at 50% vmThreads.","rollback_reason":"The repeated point is 14s slower than the retained 753s minimum; no code delta exists.","next_action_hint":"Collect a third stability sample before selecting 75%-vmThreads."}} {"run":29,"commit":"160299b","metric":789,"metrics":{"packages_coverage_seconds":554,"examples_seconds":152,"startup_seconds":77,"ci_run_id":32502789397},"status":"discard","description":"Third three-worker VM-thread stability run","timestamp":1787330345499,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"A third 75%-vmThreads run would complete cleanly and establish a robust distribution.","result":"The third run passed in 789s. Coverage was fastest at 554s, but unrelated startup/examples noise raised total time. Three samples are 753/767/789 (median 767s), all successful.","rollback_reason":"The 789s point did not improve the retained minimum; no code delta exists.","next_action_hint":"75%-vmThreads now has three clean samples. Test 100% vmThreads as a boundary, then choose 75% unless four workers improve repeatably."}} +{"run":30,"commit":"8fb6db3","metric":0,"metrics":{"packages_coverage_seconds":568,"examples_seconds":111,"startup_seconds":58,"ci_run_id":32504039496},"status":"crash","description":"First four-worker VM-thread run; measurement API transient","timestamp":1787331170166,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"Four VM-thread workers might improve coverage without fork-process pressure.","result":"The hosted Test job itself succeeded in 741s (coverage 568s, examples 111s), but measure.sh exited on a transient GitHub Jobs API HTTP 502 before recording metrics.","error_details":"Manual recovery from run 32504039496 confirmed Test success from 16:40:08Z to 16:52:29Z. This is a harness failure, not a candidate failure.","rollback_reason":"Log as crash because run_experiment exited nonzero and emitted no structured primary metric; do not treat the recovered single point as retained evidence.","next_action_hint":"Make measure.sh retry transient gh API failures, then rerun 100%-vmThreads."}} diff --git a/.auto/measure.sh b/.auto/measure.sh index 6197983b3c81..7fbf08e813b9 100755 --- a/.auto/measure.sh +++ b/.auto/measure.sh @@ -41,7 +41,10 @@ jobs_file="$(mktemp)" job_log="$(mktemp)" trap 'rm -f "$jobs_file" "$job_log"' EXIT for _ in $(seq 1 180); do - gh run view "$run_id" --repo "$repo" --json jobs > "$jobs_file" + if ! gh run view "$run_id" --repo "$repo" --json jobs > "$jobs_file"; then + sleep 10 + continue + fi status="$(node -e ' const fs = require("node:fs"); const jobs = JSON.parse(fs.readFileSync(process.argv[1], "utf8")).jobs; @@ -57,7 +60,11 @@ job_id="$(node -e ' const jobs = JSON.parse(fs.readFileSync(process.argv[1], "utf8")).jobs; process.stdout.write(String(jobs.find(({ name }) => name === "Test")?.databaseId ?? "")); ' "$jobs_file")" -gh run view "$run_id" --repo "$repo" --job "$job_id" --log > "$job_log" +for _ in $(seq 1 6); do + gh run view "$run_id" --repo "$repo" --job "$job_id" --log > "$job_log" && break + sleep 10 +done +[[ -s "$job_log" ]] || { echo "Could not download Test job log" >&2; exit 1; } phase_metric() { grep -o "CI_PHASE $1=[0-9]*" "$job_log" | tail -1 | cut -d= -f2 || true } From 768e53b3c7cd902e1dbdcf3b292d16e3171af06b Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 17:06:57 +0000 Subject: [PATCH 33/80] autoresearch: measure Test CI 20260821T170657Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index cedd9ac5db80..940096b1a8b0 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -29,3 +29,4 @@ {"run":28,"commit":"54687b8","metric":767,"metrics":{"packages_coverage_seconds":588,"examples_seconds":109,"startup_seconds":64,"ci_run_id":32501576368},"status":"discard","description":"Second three-worker VM-thread stability run","timestamp":1787329477381,"segment":0,"confidence":7.973333333333334,"asi":{"hypothesis":"The 75%-vmThreads configuration would repeat without the fork-pool teardown/resource failures.","result":"The second run passed in 767s; coverage took 588s. Two samples are 753s and 767s, both substantially below the 814s median at 50% vmThreads.","rollback_reason":"The repeated point is 14s slower than the retained 753s minimum; no code delta exists.","next_action_hint":"Collect a third stability sample before selecting 75%-vmThreads."}} {"run":29,"commit":"160299b","metric":789,"metrics":{"packages_coverage_seconds":554,"examples_seconds":152,"startup_seconds":77,"ci_run_id":32502789397},"status":"discard","description":"Third three-worker VM-thread stability run","timestamp":1787330345499,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"A third 75%-vmThreads run would complete cleanly and establish a robust distribution.","result":"The third run passed in 789s. Coverage was fastest at 554s, but unrelated startup/examples noise raised total time. Three samples are 753/767/789 (median 767s), all successful.","rollback_reason":"The 789s point did not improve the retained minimum; no code delta exists.","next_action_hint":"75%-vmThreads now has three clean samples. Test 100% vmThreads as a boundary, then choose 75% unless four workers improve repeatably."}} {"run":30,"commit":"8fb6db3","metric":0,"metrics":{"packages_coverage_seconds":568,"examples_seconds":111,"startup_seconds":58,"ci_run_id":32504039496},"status":"crash","description":"First four-worker VM-thread run; measurement API transient","timestamp":1787331170166,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"Four VM-thread workers might improve coverage without fork-process pressure.","result":"The hosted Test job itself succeeded in 741s (coverage 568s, examples 111s), but measure.sh exited on a transient GitHub Jobs API HTTP 502 before recording metrics.","error_details":"Manual recovery from run 32504039496 confirmed Test success from 16:40:08Z to 16:52:29Z. This is a harness failure, not a candidate failure.","rollback_reason":"Log as crash because run_experiment exited nonzero and emitted no structured primary metric; do not treat the recovered single point as retained evidence.","next_action_hint":"Make measure.sh retry transient gh API failures, then rerun 100%-vmThreads."}} +{"run":31,"commit":"f76df2b","metric":761,"metrics":{"packages_coverage_seconds":583,"examples_seconds":114,"startup_seconds":59,"ci_run_id":32505247674},"status":"discard","description":"Repeat four-worker VM-thread configuration with API retries","timestamp":1787332012002,"segment":0,"confidence":8.305555555555555,"asi":{"hypothesis":"A retried measurement would validate the manually recovered 741s four-worker VM result.","result":"The second Test job passed in 761s; coverage took 583s. Together with the recovered 741s run, 100%-vmThreads is currently ~16s faster in median than 75%-vmThreads.","rollback_reason":"The structured 761s point did not improve the retained 742s minimum; no code delta exists in this measurement commit.","next_action_hint":"Collect a third 100%-vmThreads stability sample before choosing between 100% and 75%."}} From 27c3dbdd9d3e1c0f798358c7583b6e765d15764b Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 17:21:27 +0000 Subject: [PATCH 34/80] autoresearch: measure Test CI 20260821T172127Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + scripts/coverage-config.test.mjs | 1 - vitest.config.ts | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 940096b1a8b0..2f34945ee3cc 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -30,3 +30,4 @@ {"run":29,"commit":"160299b","metric":789,"metrics":{"packages_coverage_seconds":554,"examples_seconds":152,"startup_seconds":77,"ci_run_id":32502789397},"status":"discard","description":"Third three-worker VM-thread stability run","timestamp":1787330345499,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"A third 75%-vmThreads run would complete cleanly and establish a robust distribution.","result":"The third run passed in 789s. Coverage was fastest at 554s, but unrelated startup/examples noise raised total time. Three samples are 753/767/789 (median 767s), all successful.","rollback_reason":"The 789s point did not improve the retained minimum; no code delta exists.","next_action_hint":"75%-vmThreads now has three clean samples. Test 100% vmThreads as a boundary, then choose 75% unless four workers improve repeatably."}} {"run":30,"commit":"8fb6db3","metric":0,"metrics":{"packages_coverage_seconds":568,"examples_seconds":111,"startup_seconds":58,"ci_run_id":32504039496},"status":"crash","description":"First four-worker VM-thread run; measurement API transient","timestamp":1787331170166,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"Four VM-thread workers might improve coverage without fork-process pressure.","result":"The hosted Test job itself succeeded in 741s (coverage 568s, examples 111s), but measure.sh exited on a transient GitHub Jobs API HTTP 502 before recording metrics.","error_details":"Manual recovery from run 32504039496 confirmed Test success from 16:40:08Z to 16:52:29Z. This is a harness failure, not a candidate failure.","rollback_reason":"Log as crash because run_experiment exited nonzero and emitted no structured primary metric; do not treat the recovered single point as retained evidence.","next_action_hint":"Make measure.sh retry transient gh API failures, then rerun 100%-vmThreads."}} {"run":31,"commit":"f76df2b","metric":761,"metrics":{"packages_coverage_seconds":583,"examples_seconds":114,"startup_seconds":59,"ci_run_id":32505247674},"status":"discard","description":"Repeat four-worker VM-thread configuration with API retries","timestamp":1787332012002,"segment":0,"confidence":8.305555555555555,"asi":{"hypothesis":"A retried measurement would validate the manually recovered 741s four-worker VM result.","result":"The second Test job passed in 761s; coverage took 583s. Together with the recovered 741s run, 100%-vmThreads is currently ~16s faster in median than 75%-vmThreads.","rollback_reason":"The structured 761s point did not improve the retained 742s minimum; no code delta exists in this measurement commit.","next_action_hint":"Collect a third 100%-vmThreads stability sample before choosing between 100% and 75%."}} +{"run":32,"commit":"768e53b","metric":752,"metrics":{"packages_coverage_seconds":569,"examples_seconds":112,"startup_seconds":64,"ci_run_id":32506470362},"status":"keep","description":"Third four-worker VM-thread stability run","timestamp":1787332843370,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"A third four-worker VM run would establish stable throughput without database or teardown failures.","result":"The third run passed in 752s. Three complete jobs are 741/761/752 (median 752s, 20s range), versus 753/767/789 (median 767s) at 75% and 819/814/811 (median 814s) at 50%.","correctness":"No Postgres, PGlite, language-server, cross-realm, or memory failure occurred across all three 100%-vmThreads jobs.","decision":"Select 100%-vmThreads: it preserves per-file VM isolation, honors Supabase's explicit forks/maxWorkers override, and has the best stable candidate median.","next_action_hint":"Test whether JSON-only coverage output materially contributes by restoring the text reporter under the selected VM configuration for an A/B sample."}} diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index ea0df983a0b4..138f155d9fff 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -269,7 +269,6 @@ describe('coverage config', () => { assert.match(rootVitestConfig, /from ['"]\.\/scripts\/coverage-config['"]/); assert.doesNotMatch(rootVitestConfig, /from ['"]\.\/scripts\/coverage-config\.[^'"]+['"]/); assert.match(rootVitestConfig, /provider:\s*['"]v8['"]/); - assert.match(rootVitestConfig, /reporter:\s*\[['"]json['"]\]/); assert.match(rootVitestConfig, /reportOnFailure:\s*true/); }); diff --git a/vitest.config.ts b/vitest.config.ts index 0eacb54cf6c0..77c1bd3f9cd9 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -25,7 +25,7 @@ export default defineConfig({ coverage: { provider: 'v8', reportsDirectory: resolve(import.meta.dirname, 'coverage'), - reporter: ['json'], + reporter: ['text', 'json'], reportOnFailure: true, ...coveragePolicy, }, From c3bf5d2749b9dfe4178a6f6144b5c29b8355063d Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 17:36:40 +0000 Subject: [PATCH 35/80] autoresearch: measure Test CI 20260821T173640Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .github/workflows/ci.yml | 6 +++--- scripts/coverage-config.test.mjs | 8 ++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 2f34945ee3cc..492f3c2d9e3d 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -31,3 +31,4 @@ {"run":30,"commit":"8fb6db3","metric":0,"metrics":{"packages_coverage_seconds":568,"examples_seconds":111,"startup_seconds":58,"ci_run_id":32504039496},"status":"crash","description":"First four-worker VM-thread run; measurement API transient","timestamp":1787331170166,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"Four VM-thread workers might improve coverage without fork-process pressure.","result":"The hosted Test job itself succeeded in 741s (coverage 568s, examples 111s), but measure.sh exited on a transient GitHub Jobs API HTTP 502 before recording metrics.","error_details":"Manual recovery from run 32504039496 confirmed Test success from 16:40:08Z to 16:52:29Z. This is a harness failure, not a candidate failure.","rollback_reason":"Log as crash because run_experiment exited nonzero and emitted no structured primary metric; do not treat the recovered single point as retained evidence.","next_action_hint":"Make measure.sh retry transient gh API failures, then rerun 100%-vmThreads."}} {"run":31,"commit":"f76df2b","metric":761,"metrics":{"packages_coverage_seconds":583,"examples_seconds":114,"startup_seconds":59,"ci_run_id":32505247674},"status":"discard","description":"Repeat four-worker VM-thread configuration with API retries","timestamp":1787332012002,"segment":0,"confidence":8.305555555555555,"asi":{"hypothesis":"A retried measurement would validate the manually recovered 741s four-worker VM result.","result":"The second Test job passed in 761s; coverage took 583s. Together with the recovered 741s run, 100%-vmThreads is currently ~16s faster in median than 75%-vmThreads.","rollback_reason":"The structured 761s point did not improve the retained 742s minimum; no code delta exists in this measurement commit.","next_action_hint":"Collect a third 100%-vmThreads stability sample before choosing between 100% and 75%."}} {"run":32,"commit":"768e53b","metric":752,"metrics":{"packages_coverage_seconds":569,"examples_seconds":112,"startup_seconds":64,"ci_run_id":32506470362},"status":"keep","description":"Third four-worker VM-thread stability run","timestamp":1787332843370,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"A third four-worker VM run would establish stable throughput without database or teardown failures.","result":"The third run passed in 752s. Three complete jobs are 741/761/752 (median 752s, 20s range), versus 753/767/789 (median 767s) at 75% and 819/814/811 (median 814s) at 50%.","correctness":"No Postgres, PGlite, language-server, cross-realm, or memory failure occurred across all three 100%-vmThreads jobs.","decision":"Select 100%-vmThreads: it preserves per-file VM isolation, honors Supabase's explicit forks/maxWorkers override, and has the best stable candidate median.","next_action_hint":"Test whether JSON-only coverage output materially contributes by restoring the text reporter under the selected VM configuration for an A/B sample."}} +{"run":33,"commit":"27c3dbd","metric":772,"metrics":{"packages_coverage_seconds":591,"examples_seconds":116,"startup_seconds":59,"ci_run_id":32507749529},"status":"discard","description":"Restore text coverage diagnostics under selected VM-thread configuration","timestamp":1787333745699,"segment":0,"confidence":9.34375,"asi":{"hypothesis":"A/B with the text reporter restored would show whether JSON-only output contributes to the speedup.","result":"The job passed in 772s. Vitest reported 587.58s while the API coverage step was 591s, a 3.42s tail; JSON-only had a 565.39s Vitest duration and 569s API step, a 3.61s tail. Text rendering adds no measurable post-suite cost.","rollback_reason":"The 772s primary point is slower than the selected JSON-only samples, but phase-internal evidence attributes that to runner execution variance, not the text reporter.","decision":"Restore the text reporter to preserve existing diagnostics and keep the final product diff focused on worker pool/concurrency only.","next_action_hint":"Now optimize startup or test execution structurally; do not revisit coverage reporter output."}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac0e6125ad0e..0e4773f381b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,15 +201,15 @@ jobs: - name: Link bins if: needs.changes.outputs.inert != 'true' run: pnpm install --frozen-lockfile - - name: Start cloudflare-worker Postgres (5433, pg_stat_statements) - if: needs.changes.outputs.inert != 'true' - run: pnpm --filter prisma-8-cloudflare-worker db:up - name: Test packages with coverage if: needs.changes.outputs.inert != 'true' run: pnpm coverage:packages - name: Report package coverage if: ${{ !cancelled() && needs.changes.outputs.inert != 'true' }} run: pnpm coverage:report + - name: Start cloudflare-worker Postgres (5433, pg_stat_statements) + if: ${{ !cancelled() && needs.changes.outputs.inert != 'true' }} + run: pnpm --filter prisma-8-cloudflare-worker db:up - name: Test examples if: ${{ !cancelled() && needs.changes.outputs.inert != 'true' }} run: pnpm test:examples diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 138f155d9fff..5b26551d05f8 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -288,6 +288,14 @@ describe('coverage config', () => { testJob, /- name: Test examples\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm test:examples/, ); + assert.ok( + testJob.indexOf('- name: Test packages with coverage') < + testJob.indexOf('- name: Start cloudflare-worker Postgres'), + ); + assert.ok( + testJob.indexOf('- name: Start cloudflare-worker Postgres') < + testJob.indexOf('- name: Test examples'), + ); assert.doesNotMatch(workflow, /\n {2}coverage:\n/); assert.equal(workflow.match(/run: pnpm coverage:packages/g)?.length, 1); assert.equal(workflow.match(/run: pnpm test:examples/g)?.length, 1); From 97a56427947d55c3ffb1198e628b3cf8f829fe1d Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 17:51:38 +0000 Subject: [PATCH 36/80] autoresearch: measure Test CI 20260821T175138Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 492f3c2d9e3d..32cc33c18564 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -32,3 +32,4 @@ {"run":31,"commit":"f76df2b","metric":761,"metrics":{"packages_coverage_seconds":583,"examples_seconds":114,"startup_seconds":59,"ci_run_id":32505247674},"status":"discard","description":"Repeat four-worker VM-thread configuration with API retries","timestamp":1787332012002,"segment":0,"confidence":8.305555555555555,"asi":{"hypothesis":"A retried measurement would validate the manually recovered 741s four-worker VM result.","result":"The second Test job passed in 761s; coverage took 583s. Together with the recovered 741s run, 100%-vmThreads is currently ~16s faster in median than 75%-vmThreads.","rollback_reason":"The structured 761s point did not improve the retained 742s minimum; no code delta exists in this measurement commit.","next_action_hint":"Collect a third 100%-vmThreads stability sample before choosing between 100% and 75%."}} {"run":32,"commit":"768e53b","metric":752,"metrics":{"packages_coverage_seconds":569,"examples_seconds":112,"startup_seconds":64,"ci_run_id":32506470362},"status":"keep","description":"Third four-worker VM-thread stability run","timestamp":1787332843370,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"A third four-worker VM run would establish stable throughput without database or teardown failures.","result":"The third run passed in 752s. Three complete jobs are 741/761/752 (median 752s, 20s range), versus 753/767/789 (median 767s) at 75% and 819/814/811 (median 814s) at 50%.","correctness":"No Postgres, PGlite, language-server, cross-realm, or memory failure occurred across all three 100%-vmThreads jobs.","decision":"Select 100%-vmThreads: it preserves per-file VM isolation, honors Supabase's explicit forks/maxWorkers override, and has the best stable candidate median.","next_action_hint":"Test whether JSON-only coverage output materially contributes by restoring the text reporter under the selected VM configuration for an A/B sample."}} {"run":33,"commit":"27c3dbd","metric":772,"metrics":{"packages_coverage_seconds":591,"examples_seconds":116,"startup_seconds":59,"ci_run_id":32507749529},"status":"discard","description":"Restore text coverage diagnostics under selected VM-thread configuration","timestamp":1787333745699,"segment":0,"confidence":9.34375,"asi":{"hypothesis":"A/B with the text reporter restored would show whether JSON-only output contributes to the speedup.","result":"The job passed in 772s. Vitest reported 587.58s while the API coverage step was 591s, a 3.42s tail; JSON-only had a 565.39s Vitest duration and 569s API step, a 3.61s tail. Text rendering adds no measurable post-suite cost.","rollback_reason":"The 772s primary point is slower than the selected JSON-only samples, but phase-internal evidence attributes that to runner execution variance, not the text reporter.","decision":"Restore the text reporter to preserve existing diagnostics and keep the final product diff focused on worker pool/concurrency only.","next_action_hint":"Now optimize startup or test execution structurally; do not revisit coverage reporter output."}} +{"run":34,"commit":"c3bf5d2","metric":812,"metrics":{"packages_coverage_seconds":574,"examples_seconds":156,"startup_seconds":65,"ci_run_id":32509068946},"status":"discard","description":"Start Cloudflare example Postgres after package coverage","timestamp":1787334692005,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"Deferring the example-only 5433 Postgres container would keep it from competing with the 9-minute package coverage phase.","result":"The job passed in 812s. Coverage took 574s, essentially the same as the selected 100%-vmThreads median (569s); Cloudflare startup took 11s after coverage. Example time was an unrelated 156s outlier.","rollback_reason":"No primary improvement in the first sample; total time regressed due example/startup runner noise.","next_action_hint":"Repeat once because coverage stayed near the selected median and the resource-lifetime reduction is logically sound."}} From f4a6266588a676cc3f5183909f6093fe30dfe87a Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 20:31:30 +0000 Subject: [PATCH 37/80] autoresearch: measure Test CI 20260821T203130Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .../language-server/test/server.test.ts | 40 +++++++++++++++---- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 32cc33c18564..d147c03e05a8 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -33,3 +33,4 @@ {"run":32,"commit":"768e53b","metric":752,"metrics":{"packages_coverage_seconds":569,"examples_seconds":112,"startup_seconds":64,"ci_run_id":32506470362},"status":"keep","description":"Third four-worker VM-thread stability run","timestamp":1787332843370,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"A third four-worker VM run would establish stable throughput without database or teardown failures.","result":"The third run passed in 752s. Three complete jobs are 741/761/752 (median 752s, 20s range), versus 753/767/789 (median 767s) at 75% and 819/814/811 (median 814s) at 50%.","correctness":"No Postgres, PGlite, language-server, cross-realm, or memory failure occurred across all three 100%-vmThreads jobs.","decision":"Select 100%-vmThreads: it preserves per-file VM isolation, honors Supabase's explicit forks/maxWorkers override, and has the best stable candidate median.","next_action_hint":"Test whether JSON-only coverage output materially contributes by restoring the text reporter under the selected VM configuration for an A/B sample."}} {"run":33,"commit":"27c3dbd","metric":772,"metrics":{"packages_coverage_seconds":591,"examples_seconds":116,"startup_seconds":59,"ci_run_id":32507749529},"status":"discard","description":"Restore text coverage diagnostics under selected VM-thread configuration","timestamp":1787333745699,"segment":0,"confidence":9.34375,"asi":{"hypothesis":"A/B with the text reporter restored would show whether JSON-only output contributes to the speedup.","result":"The job passed in 772s. Vitest reported 587.58s while the API coverage step was 591s, a 3.42s tail; JSON-only had a 565.39s Vitest duration and 569s API step, a 3.61s tail. Text rendering adds no measurable post-suite cost.","rollback_reason":"The 772s primary point is slower than the selected JSON-only samples, but phase-internal evidence attributes that to runner execution variance, not the text reporter.","decision":"Restore the text reporter to preserve existing diagnostics and keep the final product diff focused on worker pool/concurrency only.","next_action_hint":"Now optimize startup or test execution structurally; do not revisit coverage reporter output."}} {"run":34,"commit":"c3bf5d2","metric":812,"metrics":{"packages_coverage_seconds":574,"examples_seconds":156,"startup_seconds":65,"ci_run_id":32509068946},"status":"discard","description":"Start Cloudflare example Postgres after package coverage","timestamp":1787334692005,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"Deferring the example-only 5433 Postgres container would keep it from competing with the 9-minute package coverage phase.","result":"The job passed in 812s. Coverage took 574s, essentially the same as the selected 100%-vmThreads median (569s); Cloudflare startup took 11s after coverage. Example time was an unrelated 156s outlier.","rollback_reason":"No primary improvement in the first sample; total time regressed due example/startup runner noise.","next_action_hint":"Repeat once because coverage stayed near the selected median and the resource-lifetime reduction is logically sound."}} +{"run":35,"commit":"97a5642","metric":0,"metrics":{"packages_coverage_seconds":577,"examples_seconds":113,"startup_seconds":65,"ci_run_id":32510354529},"status":"crash","description":"Repeat deferred Cloudflare startup under four VM workers","timestamp":1787343003909,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"A second deferred-container sample would isolate whether removing idle Postgres competition helps coverage.","result":"All 1,165 files and 15,375 tests passed, but the coverage step failed on the known language-server teardown race: an unhandled `Connection is disposed` rejection after the final pull-diagnostics test. Coverage execution was 576.75s.","error_details":"This is the second occurrence across different pool configurations, so it is a benchmark-independent test harness teardown defect rather than evidence against deferred Cloudflare startup.","rollback_reason":"Any CI failure is unacceptable; no performance result can be kept from this run.","next_action_hint":"Add a graceful harness shutdown path that awaits an LSP ShutdownRequest before disposal, while retaining immediate dispose for tests that deliberately settle work after teardown."}} diff --git a/packages/1-framework/3-tooling/language-server/test/server.test.ts b/packages/1-framework/3-tooling/language-server/test/server.test.ts index 7ae4e1abd057..c523b7a79e4a 100644 --- a/packages/1-framework/3-tooling/language-server/test/server.test.ts +++ b/packages/1-framework/3-tooling/language-server/test/server.test.ts @@ -44,6 +44,7 @@ import { type RegistrationParams, RegistrationRequest, type SemanticTokens, + ShutdownRequest, StreamMessageReader, StreamMessageWriter, type TextEdit, @@ -207,6 +208,7 @@ interface Harness { readonly notifyConfigChanged: (uri?: string) => void; readonly getDocumentAst: (uri: string) => DocumentArtifacts | undefined; readonly getProjectSymbolTable: (uri: string) => SymbolTable | undefined; + readonly shutdown: () => Promise; dispose: () => void; } @@ -418,6 +420,13 @@ function startHarness( }, getDocumentAst: (uri) => server.getDocumentAst(uri), getProjectSymbolTable: (uri) => server.getProjectSymbolTable(uri), + shutdown: async () => { + await client.sendRequest(ShutdownRequest.type); + server.dispose(); + client.dispose(); + clientToServer.end(); + serverToClient.end(); + }, dispose: () => { // Dispose the server first, so its connection is gone before the // transport dies. Sends made after that point — an in-flight `publish` @@ -560,14 +569,7 @@ function deferredSettleable(): { let harness: Harness | undefined; afterEach(async () => { - // The harness disposes the server before the client (see `dispose` above), - // so the server's `disposed` guard is raised before the transport dies and - // an in-flight `publish` can never log through a dead connection. This tick - // is a separate concern: it lets any in-flight JSON-RPC request/response - // write flush before the streams are torn down, so vscode-jsonrpc's own - // internal error logging doesn't reject a notification mid-transmission. - await new Promise((resolve) => setTimeout(resolve, 0)); - harness?.dispose(); + await harness?.shutdown(); harness = undefined; configResolutionMock.resolveConfigInputs.mockReset(); configLoaderMock.findNearestConfigPathForFile.mockReset(); @@ -2046,6 +2048,28 @@ describe('language server preserved artifacts', { timeout: timeouts.databaseOper }); describe('language server disposal', { timeout: timeouts.databaseOperation }, () => { + it('gracefully shuts down after a pull-diagnostics request', async () => { + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown): void => { + unhandled.push(reason); + }; + process.on('unhandledRejection', onUnhandled); + try { + harness = startHarness(resolveToSchema, pullDiagnosticsCapabilities); + await harness.initialize(); + openDocument(harness, schemaUri, duplicateModelSource); + await requestPullDiagnostics(harness, schemaUri); + + await harness.shutdown(); + harness = undefined; + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(unhandled).toEqual([]); + } finally { + process.off('unhandledRejection', onUnhandled); + } + }); + async function assertNoUnhandledRejection( settle: (load: { readonly resolve: (value: ConfigResolution) => void; From 64d2307e57a5763ba19a39f397e76f704153e3bd Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 21:04:03 +0000 Subject: [PATCH 38/80] autoresearch: measure Test CI 20260821T210403Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index d147c03e05a8..aa4397bbabdc 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -34,3 +34,4 @@ {"run":33,"commit":"27c3dbd","metric":772,"metrics":{"packages_coverage_seconds":591,"examples_seconds":116,"startup_seconds":59,"ci_run_id":32507749529},"status":"discard","description":"Restore text coverage diagnostics under selected VM-thread configuration","timestamp":1787333745699,"segment":0,"confidence":9.34375,"asi":{"hypothesis":"A/B with the text reporter restored would show whether JSON-only output contributes to the speedup.","result":"The job passed in 772s. Vitest reported 587.58s while the API coverage step was 591s, a 3.42s tail; JSON-only had a 565.39s Vitest duration and 569s API step, a 3.61s tail. Text rendering adds no measurable post-suite cost.","rollback_reason":"The 772s primary point is slower than the selected JSON-only samples, but phase-internal evidence attributes that to runner execution variance, not the text reporter.","decision":"Restore the text reporter to preserve existing diagnostics and keep the final product diff focused on worker pool/concurrency only.","next_action_hint":"Now optimize startup or test execution structurally; do not revisit coverage reporter output."}} {"run":34,"commit":"c3bf5d2","metric":812,"metrics":{"packages_coverage_seconds":574,"examples_seconds":156,"startup_seconds":65,"ci_run_id":32509068946},"status":"discard","description":"Start Cloudflare example Postgres after package coverage","timestamp":1787334692005,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"Deferring the example-only 5433 Postgres container would keep it from competing with the 9-minute package coverage phase.","result":"The job passed in 812s. Coverage took 574s, essentially the same as the selected 100%-vmThreads median (569s); Cloudflare startup took 11s after coverage. Example time was an unrelated 156s outlier.","rollback_reason":"No primary improvement in the first sample; total time regressed due example/startup runner noise.","next_action_hint":"Repeat once because coverage stayed near the selected median and the resource-lifetime reduction is logically sound."}} {"run":35,"commit":"97a5642","metric":0,"metrics":{"packages_coverage_seconds":577,"examples_seconds":113,"startup_seconds":65,"ci_run_id":32510354529},"status":"crash","description":"Repeat deferred Cloudflare startup under four VM workers","timestamp":1787343003909,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"A second deferred-container sample would isolate whether removing idle Postgres competition helps coverage.","result":"All 1,165 files and 15,375 tests passed, but the coverage step failed on the known language-server teardown race: an unhandled `Connection is disposed` rejection after the final pull-diagnostics test. Coverage execution was 576.75s.","error_details":"This is the second occurrence across different pool configurations, so it is a benchmark-independent test harness teardown defect rather than evidence against deferred Cloudflare startup.","rollback_reason":"Any CI failure is unacceptable; no performance result can be kept from this run.","next_action_hint":"Add a graceful harness shutdown path that awaits an LSP ShutdownRequest before disposal, while retaining immediate dispose for tests that deliberately settle work after teardown."}} +{"run":36,"commit":"f4a6266","metric":792,"metrics":{"packages_coverage_seconds":597,"examples_seconds":116,"startup_seconds":64,"ci_run_id":32524070966},"status":"keep","description":"Drain language-server JSON-RPC before test harness disposal","timestamp":1787346235078,"segment":0,"confidence":8.666666666666666,"asi":{"hypothesis":"Awaiting the standard LSP ShutdownRequest before routine harness disposal will drain queued JSON-RPC writes and eliminate the intermittent post-test `Connection is disposed` rejection.","result":"The complete CI Test job passed in 792s after the fix; the language-server package also passed locally with 261 tests. No unhandled rejection occurred.","correctness":"Immediate `dispose()` remains available for tests that deliberately settle a load after teardown; normal afterEach uses the new graceful `shutdown()`. A regression test covers shutdown after pull diagnostics.","scope_note":"This stabilizes the test harness rather than weakening Vitest unhandled-error handling or retrying failed tests.","next_action_hint":"Repeat the exact configuration to verify the teardown race stays absent; then decide whether deferred Cloudflare startup has value."}} From aef253deab60d683a8bf0e72a9bf67e49275a63f Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 21:32:53 +0000 Subject: [PATCH 39/80] autoresearch: measure Test CI 20260821T213250Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index aa4397bbabdc..3d862a442b1f 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -35,3 +35,4 @@ {"run":34,"commit":"c3bf5d2","metric":812,"metrics":{"packages_coverage_seconds":574,"examples_seconds":156,"startup_seconds":65,"ci_run_id":32509068946},"status":"discard","description":"Start Cloudflare example Postgres after package coverage","timestamp":1787334692005,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"Deferring the example-only 5433 Postgres container would keep it from competing with the 9-minute package coverage phase.","result":"The job passed in 812s. Coverage took 574s, essentially the same as the selected 100%-vmThreads median (569s); Cloudflare startup took 11s after coverage. Example time was an unrelated 156s outlier.","rollback_reason":"No primary improvement in the first sample; total time regressed due example/startup runner noise.","next_action_hint":"Repeat once because coverage stayed near the selected median and the resource-lifetime reduction is logically sound."}} {"run":35,"commit":"97a5642","metric":0,"metrics":{"packages_coverage_seconds":577,"examples_seconds":113,"startup_seconds":65,"ci_run_id":32510354529},"status":"crash","description":"Repeat deferred Cloudflare startup under four VM workers","timestamp":1787343003909,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"A second deferred-container sample would isolate whether removing idle Postgres competition helps coverage.","result":"All 1,165 files and 15,375 tests passed, but the coverage step failed on the known language-server teardown race: an unhandled `Connection is disposed` rejection after the final pull-diagnostics test. Coverage execution was 576.75s.","error_details":"This is the second occurrence across different pool configurations, so it is a benchmark-independent test harness teardown defect rather than evidence against deferred Cloudflare startup.","rollback_reason":"Any CI failure is unacceptable; no performance result can be kept from this run.","next_action_hint":"Add a graceful harness shutdown path that awaits an LSP ShutdownRequest before disposal, while retaining immediate dispose for tests that deliberately settle work after teardown."}} {"run":36,"commit":"f4a6266","metric":792,"metrics":{"packages_coverage_seconds":597,"examples_seconds":116,"startup_seconds":64,"ci_run_id":32524070966},"status":"keep","description":"Drain language-server JSON-RPC before test harness disposal","timestamp":1787346235078,"segment":0,"confidence":8.666666666666666,"asi":{"hypothesis":"Awaiting the standard LSP ShutdownRequest before routine harness disposal will drain queued JSON-RPC writes and eliminate the intermittent post-test `Connection is disposed` rejection.","result":"The complete CI Test job passed in 792s after the fix; the language-server package also passed locally with 261 tests. No unhandled rejection occurred.","correctness":"Immediate `dispose()` remains available for tests that deliberately settle a load after teardown; normal afterEach uses the new graceful `shutdown()`. A regression test covers shutdown after pull diagnostics.","scope_note":"This stabilizes the test harness rather than weakening Vitest unhandled-error handling or retrying failed tests.","next_action_hint":"Repeat the exact configuration to verify the teardown race stays absent; then decide whether deferred Cloudflare startup has value."}} +{"run":37,"commit":"64d2307","metric":761,"metrics":{"packages_coverage_seconds":582,"examples_seconds":116,"startup_seconds":49,"ci_run_id":32526728495},"status":"keep","description":"Repeat graceful LSP teardown with four VM workers","timestamp":1787347957699,"segment":0,"confidence":9.64516129032258,"asi":{"hypothesis":"The teardown fix and selected VM-thread configuration would pass repeatedly.","result":"The second post-fix job passed in 761s, with coverage at 582s. Both post-fix runs completed without the formerly intermittent language-server unhandled rejection.","stability":"Across the broader 100%-vmThreads set, five successful timing samples now cluster around a 761s upper median; the single teardown failure has a targeted fix and two subsequent clean runs.","next_action_hint":"Run a third post-fix sample for confidence, then A/B deferred Cloudflare startup by moving it back while retaining VM threads and graceful teardown."}} From 720976a176eaa2e77c12e17d340f9b253e9961c8 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 21:48:30 +0000 Subject: [PATCH 40/80] autoresearch: measure Test CI 20260821T214830Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 3d862a442b1f..6457e90ad49b 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -36,3 +36,4 @@ {"run":35,"commit":"97a5642","metric":0,"metrics":{"packages_coverage_seconds":577,"examples_seconds":113,"startup_seconds":65,"ci_run_id":32510354529},"status":"crash","description":"Repeat deferred Cloudflare startup under four VM workers","timestamp":1787343003909,"segment":0,"confidence":8.794117647058824,"asi":{"hypothesis":"A second deferred-container sample would isolate whether removing idle Postgres competition helps coverage.","result":"All 1,165 files and 15,375 tests passed, but the coverage step failed on the known language-server teardown race: an unhandled `Connection is disposed` rejection after the final pull-diagnostics test. Coverage execution was 576.75s.","error_details":"This is the second occurrence across different pool configurations, so it is a benchmark-independent test harness teardown defect rather than evidence against deferred Cloudflare startup.","rollback_reason":"Any CI failure is unacceptable; no performance result can be kept from this run.","next_action_hint":"Add a graceful harness shutdown path that awaits an LSP ShutdownRequest before disposal, while retaining immediate dispose for tests that deliberately settle work after teardown."}} {"run":36,"commit":"f4a6266","metric":792,"metrics":{"packages_coverage_seconds":597,"examples_seconds":116,"startup_seconds":64,"ci_run_id":32524070966},"status":"keep","description":"Drain language-server JSON-RPC before test harness disposal","timestamp":1787346235078,"segment":0,"confidence":8.666666666666666,"asi":{"hypothesis":"Awaiting the standard LSP ShutdownRequest before routine harness disposal will drain queued JSON-RPC writes and eliminate the intermittent post-test `Connection is disposed` rejection.","result":"The complete CI Test job passed in 792s after the fix; the language-server package also passed locally with 261 tests. No unhandled rejection occurred.","correctness":"Immediate `dispose()` remains available for tests that deliberately settle a load after teardown; normal afterEach uses the new graceful `shutdown()`. A regression test covers shutdown after pull diagnostics.","scope_note":"This stabilizes the test harness rather than weakening Vitest unhandled-error handling or retrying failed tests.","next_action_hint":"Repeat the exact configuration to verify the teardown race stays absent; then decide whether deferred Cloudflare startup has value."}} {"run":37,"commit":"64d2307","metric":761,"metrics":{"packages_coverage_seconds":582,"examples_seconds":116,"startup_seconds":49,"ci_run_id":32526728495},"status":"keep","description":"Repeat graceful LSP teardown with four VM workers","timestamp":1787347957699,"segment":0,"confidence":9.64516129032258,"asi":{"hypothesis":"The teardown fix and selected VM-thread configuration would pass repeatedly.","result":"The second post-fix job passed in 761s, with coverage at 582s. Both post-fix runs completed without the formerly intermittent language-server unhandled rejection.","stability":"Across the broader 100%-vmThreads set, five successful timing samples now cluster around a 761s upper median; the single teardown failure has a targeted fix and two subsequent clean runs.","next_action_hint":"Run a third post-fix sample for confidence, then A/B deferred Cloudflare startup by moving it back while retaining VM threads and graceful teardown."}} +{"run":38,"commit":"aef253d","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Third post-fix stability sample blocked by SSH transport","timestamp":1787348901289,"segment":0,"confidence":9.64516129032258,"asi":{"hypothesis":"A third post-fix hosted run would provide final teardown stability confidence.","result":"No CI run started. The measurement commit succeeded locally, but `git push` failed immediately because github.com closed the SSH connection.","error_details":"`fatal: the remote end hung up upon initial contact`; this is external transport failure, not candidate behavior.","rollback_reason":"No hosted benchmark exists for this iteration.","next_action_hint":"Retry the unchanged configuration; the next measure commit will include and push the local unpushed commit."}} From bc77d5181e46720a86a8ba9b6ddf1f3fd32e601d Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 22:08:24 +0000 Subject: [PATCH 41/80] autoresearch: measure Test CI 20260821T220824Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .github/workflows/ci.yml | 6 +++--- scripts/coverage-config.test.mjs | 8 -------- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 6457e90ad49b..d2544dd907d9 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -37,3 +37,4 @@ {"run":36,"commit":"f4a6266","metric":792,"metrics":{"packages_coverage_seconds":597,"examples_seconds":116,"startup_seconds":64,"ci_run_id":32524070966},"status":"keep","description":"Drain language-server JSON-RPC before test harness disposal","timestamp":1787346235078,"segment":0,"confidence":8.666666666666666,"asi":{"hypothesis":"Awaiting the standard LSP ShutdownRequest before routine harness disposal will drain queued JSON-RPC writes and eliminate the intermittent post-test `Connection is disposed` rejection.","result":"The complete CI Test job passed in 792s after the fix; the language-server package also passed locally with 261 tests. No unhandled rejection occurred.","correctness":"Immediate `dispose()` remains available for tests that deliberately settle a load after teardown; normal afterEach uses the new graceful `shutdown()`. A regression test covers shutdown after pull diagnostics.","scope_note":"This stabilizes the test harness rather than weakening Vitest unhandled-error handling or retrying failed tests.","next_action_hint":"Repeat the exact configuration to verify the teardown race stays absent; then decide whether deferred Cloudflare startup has value."}} {"run":37,"commit":"64d2307","metric":761,"metrics":{"packages_coverage_seconds":582,"examples_seconds":116,"startup_seconds":49,"ci_run_id":32526728495},"status":"keep","description":"Repeat graceful LSP teardown with four VM workers","timestamp":1787347957699,"segment":0,"confidence":9.64516129032258,"asi":{"hypothesis":"The teardown fix and selected VM-thread configuration would pass repeatedly.","result":"The second post-fix job passed in 761s, with coverage at 582s. Both post-fix runs completed without the formerly intermittent language-server unhandled rejection.","stability":"Across the broader 100%-vmThreads set, five successful timing samples now cluster around a 761s upper median; the single teardown failure has a targeted fix and two subsequent clean runs.","next_action_hint":"Run a third post-fix sample for confidence, then A/B deferred Cloudflare startup by moving it back while retaining VM threads and graceful teardown."}} {"run":38,"commit":"aef253d","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Third post-fix stability sample blocked by SSH transport","timestamp":1787348901289,"segment":0,"confidence":9.64516129032258,"asi":{"hypothesis":"A third post-fix hosted run would provide final teardown stability confidence.","result":"No CI run started. The measurement commit succeeded locally, but `git push` failed immediately because github.com closed the SSH connection.","error_details":"`fatal: the remote end hung up upon initial contact`; this is external transport failure, not candidate behavior.","rollback_reason":"No hosted benchmark exists for this iteration.","next_action_hint":"Retry the unchanged configuration; the next measure commit will include and push the local unpushed commit."}} +{"run":39,"commit":"720976a","metric":772,"metrics":{"packages_coverage_seconds":580,"examples_seconds":116,"startup_seconds":60,"ci_run_id":32530231871},"status":"discard","description":"Third post-fix graceful-teardown stability run","timestamp":1787349852614,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"The third actual post-fix job would remain free of language-server teardown failures.","result":"The job passed in 772s with coverage at 580s. Three post-fix jobs (792/761/772) all passed, establishing the graceful shutdown fix as stable under four VM workers.","rollback_reason":"The 772s point is 11s slower than the retained post-fix minimum; no code delta exists.","next_action_hint":"A/B Cloudflare startup position now that pool and teardown are stable."}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e4773f381b0..ac0e6125ad0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,15 +201,15 @@ jobs: - name: Link bins if: needs.changes.outputs.inert != 'true' run: pnpm install --frozen-lockfile + - name: Start cloudflare-worker Postgres (5433, pg_stat_statements) + if: needs.changes.outputs.inert != 'true' + run: pnpm --filter prisma-8-cloudflare-worker db:up - name: Test packages with coverage if: needs.changes.outputs.inert != 'true' run: pnpm coverage:packages - name: Report package coverage if: ${{ !cancelled() && needs.changes.outputs.inert != 'true' }} run: pnpm coverage:report - - name: Start cloudflare-worker Postgres (5433, pg_stat_statements) - if: ${{ !cancelled() && needs.changes.outputs.inert != 'true' }} - run: pnpm --filter prisma-8-cloudflare-worker db:up - name: Test examples if: ${{ !cancelled() && needs.changes.outputs.inert != 'true' }} run: pnpm test:examples diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 5b26551d05f8..138f155d9fff 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -288,14 +288,6 @@ describe('coverage config', () => { testJob, /- name: Test examples\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm test:examples/, ); - assert.ok( - testJob.indexOf('- name: Test packages with coverage') < - testJob.indexOf('- name: Start cloudflare-worker Postgres'), - ); - assert.ok( - testJob.indexOf('- name: Start cloudflare-worker Postgres') < - testJob.indexOf('- name: Test examples'), - ); assert.doesNotMatch(workflow, /\n {2}coverage:\n/); assert.equal(workflow.match(/run: pnpm coverage:packages/g)?.length, 1); assert.equal(workflow.match(/run: pnpm test:examples/g)?.length, 1); From 90a9c779b1526a0c173e198e70587e8238ddd104 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Fri, 21 Aug 2026 22:24:28 +0000 Subject: [PATCH 42/80] autoresearch: measure Test CI 20260821T222427Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .auto/measure.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index d2544dd907d9..1bbcf77e711b 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -38,3 +38,4 @@ {"run":37,"commit":"64d2307","metric":761,"metrics":{"packages_coverage_seconds":582,"examples_seconds":116,"startup_seconds":49,"ci_run_id":32526728495},"status":"keep","description":"Repeat graceful LSP teardown with four VM workers","timestamp":1787347957699,"segment":0,"confidence":9.64516129032258,"asi":{"hypothesis":"The teardown fix and selected VM-thread configuration would pass repeatedly.","result":"The second post-fix job passed in 761s, with coverage at 582s. Both post-fix runs completed without the formerly intermittent language-server unhandled rejection.","stability":"Across the broader 100%-vmThreads set, five successful timing samples now cluster around a 761s upper median; the single teardown failure has a targeted fix and two subsequent clean runs.","next_action_hint":"Run a third post-fix sample for confidence, then A/B deferred Cloudflare startup by moving it back while retaining VM threads and graceful teardown."}} {"run":38,"commit":"aef253d","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Third post-fix stability sample blocked by SSH transport","timestamp":1787348901289,"segment":0,"confidence":9.64516129032258,"asi":{"hypothesis":"A third post-fix hosted run would provide final teardown stability confidence.","result":"No CI run started. The measurement commit succeeded locally, but `git push` failed immediately because github.com closed the SSH connection.","error_details":"`fatal: the remote end hung up upon initial contact`; this is external transport failure, not candidate behavior.","rollback_reason":"No hosted benchmark exists for this iteration.","next_action_hint":"Retry the unchanged configuration; the next measure commit will include and push the local unpushed commit."}} {"run":39,"commit":"720976a","metric":772,"metrics":{"packages_coverage_seconds":580,"examples_seconds":116,"startup_seconds":60,"ci_run_id":32530231871},"status":"discard","description":"Third post-fix graceful-teardown stability run","timestamp":1787349852614,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"The third actual post-fix job would remain free of language-server teardown failures.","result":"The job passed in 772s with coverage at 580s. Three post-fix jobs (792/761/772) all passed, establishing the graceful shutdown fix as stable under four VM workers.","rollback_reason":"The 772s point is 11s slower than the retained post-fix minimum; no code delta exists.","next_action_hint":"A/B Cloudflare startup position now that pool and teardown are stable."}} +{"run":40,"commit":"bc77d51","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Cloudflare startup A/B blocked by GitHub API timeout","timestamp":1787351040831,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Restoring Cloudflare startup to its original position would test whether deferral changes performance.","result":"The branch pushed and CI likely triggered, but measure.sh failed while listing workflows due a GitHub API TLS handshake timeout before discovering the run.","error_details":"`couldn't fetch workflows ... net/http: TLS handshake timeout`","rollback_reason":"No benchmark result was acquired in the experiment cycle.","next_action_hint":"Retry run discovery API failures in measure.sh and recover or rerun the exact SHA."}} diff --git a/.auto/measure.sh b/.auto/measure.sh index 7fbf08e813b9..7c0f49d59ad0 100755 --- a/.auto/measure.sh +++ b/.auto/measure.sh @@ -27,7 +27,7 @@ echo "Benchmarking PR #$pr_number at $sha" >&2 run_id="" for _ in $(seq 1 60); do - run_id="$(gh run list --repo "$repo" --workflow "$workflow" --event pull_request --commit "$sha" --limit 1 --json databaseId --jq '.[0].databaseId // empty')" + run_id="$(gh run list --repo "$repo" --workflow "$workflow" --event pull_request --commit "$sha" --limit 1 --json databaseId --jq '.[0].databaseId // empty' 2>/dev/null || true)" [[ -n "$run_id" ]] && break sleep 5 done From 08dfbbbb4e97900dcffa20a3125041cf4759d5ab Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 00:00:37 +0000 Subject: [PATCH 43/80] autoresearch: measure Test CI 20260822T000037Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .auto/measure.sh | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 1bbcf77e711b..5f563f63434f 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -39,3 +39,4 @@ {"run":38,"commit":"aef253d","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Third post-fix stability sample blocked by SSH transport","timestamp":1787348901289,"segment":0,"confidence":9.64516129032258,"asi":{"hypothesis":"A third post-fix hosted run would provide final teardown stability confidence.","result":"No CI run started. The measurement commit succeeded locally, but `git push` failed immediately because github.com closed the SSH connection.","error_details":"`fatal: the remote end hung up upon initial contact`; this is external transport failure, not candidate behavior.","rollback_reason":"No hosted benchmark exists for this iteration.","next_action_hint":"Retry the unchanged configuration; the next measure commit will include and push the local unpushed commit."}} {"run":39,"commit":"720976a","metric":772,"metrics":{"packages_coverage_seconds":580,"examples_seconds":116,"startup_seconds":60,"ci_run_id":32530231871},"status":"discard","description":"Third post-fix graceful-teardown stability run","timestamp":1787349852614,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"The third actual post-fix job would remain free of language-server teardown failures.","result":"The job passed in 772s with coverage at 580s. Three post-fix jobs (792/761/772) all passed, establishing the graceful shutdown fix as stable under four VM workers.","rollback_reason":"The 772s point is 11s slower than the retained post-fix minimum; no code delta exists.","next_action_hint":"A/B Cloudflare startup position now that pool and teardown are stable."}} {"run":40,"commit":"bc77d51","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Cloudflare startup A/B blocked by GitHub API timeout","timestamp":1787351040831,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Restoring Cloudflare startup to its original position would test whether deferral changes performance.","result":"The branch pushed and CI likely triggered, but measure.sh failed while listing workflows due a GitHub API TLS handshake timeout before discovering the run.","error_details":"`couldn't fetch workflows ... net/http: TLS handshake timeout`","rollback_reason":"No benchmark result was acquired in the experiment cycle.","next_action_hint":"Retry run discovery API failures in measure.sh and recover or rerun the exact SHA."}} +{"run":41,"commit":"90a9c77","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Cloudflare startup A/B retry blocked by SSH disconnect","timestamp":1787354953888,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Retry the original Cloudflare startup ordering benchmark after hardening API discovery.","result":"No hosted run started because the SSH push connection was closed by GitHub after a long network stall.","error_details":"`fatal: Could not read from remote repository` after 2015s wall time.","rollback_reason":"External transport failure produced no metric.","next_action_hint":"Retry push/measurement when GitHub SSH recovers; consider adding bounded push retries/timeouts to the harness."}} diff --git a/.auto/measure.sh b/.auto/measure.sh index 7c0f49d59ad0..544979d5b7a0 100755 --- a/.auto/measure.sh +++ b/.auto/measure.sh @@ -16,7 +16,15 @@ fi git add -A git commit --signoff --allow-empty -m "autoresearch: measure Test CI $(date -u +%Y%m%dT%H%M%SZ)" >/dev/null sha="$(git rev-parse HEAD)" -git push --set-upstream origin "$branch" >/dev/null +pushed=false +for _ in $(seq 1 3); do + if timeout 120 git push --set-upstream origin "$branch" >/dev/null; then + pushed=true + break + fi + sleep 10 +done +[[ "$pushed" == true ]] || { echo "Could not push experiment after three attempts" >&2; exit 1; } pr_number="$(gh pr list --repo "$repo" --state open --head "$branch" --json number --jq '.[0].number // empty')" if [[ -z "$pr_number" ]]; then From 937737963b44b76b1f76986c70cee8905046eb45 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 01:20:02 +0000 Subject: [PATCH 44/80] autoresearch: measure Test CI 20260822T012001Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 5f563f63434f..e21f0c230e77 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -40,3 +40,4 @@ {"run":39,"commit":"720976a","metric":772,"metrics":{"packages_coverage_seconds":580,"examples_seconds":116,"startup_seconds":60,"ci_run_id":32530231871},"status":"discard","description":"Third post-fix graceful-teardown stability run","timestamp":1787349852614,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"The third actual post-fix job would remain free of language-server teardown failures.","result":"The job passed in 772s with coverage at 580s. Three post-fix jobs (792/761/772) all passed, establishing the graceful shutdown fix as stable under four VM workers.","rollback_reason":"The 772s point is 11s slower than the retained post-fix minimum; no code delta exists.","next_action_hint":"A/B Cloudflare startup position now that pool and teardown are stable."}} {"run":40,"commit":"bc77d51","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Cloudflare startup A/B blocked by GitHub API timeout","timestamp":1787351040831,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Restoring Cloudflare startup to its original position would test whether deferral changes performance.","result":"The branch pushed and CI likely triggered, but measure.sh failed while listing workflows due a GitHub API TLS handshake timeout before discovering the run.","error_details":"`couldn't fetch workflows ... net/http: TLS handshake timeout`","rollback_reason":"No benchmark result was acquired in the experiment cycle.","next_action_hint":"Retry run discovery API failures in measure.sh and recover or rerun the exact SHA."}} {"run":41,"commit":"90a9c77","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Cloudflare startup A/B retry blocked by SSH disconnect","timestamp":1787354953888,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Retry the original Cloudflare startup ordering benchmark after hardening API discovery.","result":"No hosted run started because the SSH push connection was closed by GitHub after a long network stall.","error_details":"`fatal: Could not read from remote repository` after 2015s wall time.","rollback_reason":"External transport failure produced no metric.","next_action_hint":"Retry push/measurement when GitHub SSH recovers; consider adding bounded push retries/timeouts to the harness."}} +{"run":42,"commit":"08dfbbb","metric":776,"metrics":{"packages_coverage_seconds":590,"examples_seconds":115,"startup_seconds":66,"ci_run_id":32538912290},"status":"discard","description":"Restore original Cloudflare Postgres startup ordering","timestamp":1787357778143,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Starting Cloudflare Postgres before coverage would A/B whether deferral affected the critical path.","result":"The job passed in 776s with coverage at 590s. This is effectively tied with deferred-start post-fix samples (median 772s); moving an idle container did not produce a measurable benefit.","rollback_reason":"The point did not improve the retained 761s result.","decision":"Keep the original workflow ordering to minimize the final product diff; the optimization is entirely in Vitest pooling/concurrency, plus the teardown stabilization it required.","next_action_hint":"Update session notes, then test another independent Vitest setting or prepare the selected minimal diff."}} diff --git a/vitest.config.ts b/vitest.config.ts index 77c1bd3f9cd9..4ffe16e2f484 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -7,11 +7,8 @@ const coveragePolicy = composeCoverageConfig(import.meta.dirname); export default defineConfig({ test: { projects: ['packages/**/vitest.config.ts'], - // Cap fork concurrency on CI so the PGlite-WASM-heavy package suites - // (cli, sql runtime, postgres/supabase extensions, postgres adapter + - // driver) don't all peak at once. Uncapped, several CPU-hungry PGlite - // forks plus the Postgres service can oversubscribe the runner. - maxWorkers: process.env['CI'] ? '100%' : undefined, + // Reuse CI worker threads while keeping a fresh VM context per test file. + // Stateful projects can override this default, as the Supabase suite does. pool: process.env['CI'] ? 'vmThreads' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender From 8d9d788c3d5c4273b0f206e198a78b7e0baa49b4 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 02:23:39 +0000 Subject: [PATCH 45/80] autoresearch: measure Test CI 20260822T022338Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index e21f0c230e77..abfcbc39b8cd 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -41,3 +41,4 @@ {"run":40,"commit":"bc77d51","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Cloudflare startup A/B blocked by GitHub API timeout","timestamp":1787351040831,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Restoring Cloudflare startup to its original position would test whether deferral changes performance.","result":"The branch pushed and CI likely triggered, but measure.sh failed while listing workflows due a GitHub API TLS handshake timeout before discovering the run.","error_details":"`couldn't fetch workflows ... net/http: TLS handshake timeout`","rollback_reason":"No benchmark result was acquired in the experiment cycle.","next_action_hint":"Retry run discovery API failures in measure.sh and recover or rerun the exact SHA."}} {"run":41,"commit":"90a9c77","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Cloudflare startup A/B retry blocked by SSH disconnect","timestamp":1787354953888,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Retry the original Cloudflare startup ordering benchmark after hardening API discovery.","result":"No hosted run started because the SSH push connection was closed by GitHub after a long network stall.","error_details":"`fatal: Could not read from remote repository` after 2015s wall time.","rollback_reason":"External transport failure produced no metric.","next_action_hint":"Retry push/measurement when GitHub SSH recovers; consider adding bounded push retries/timeouts to the harness."}} {"run":42,"commit":"08dfbbb","metric":776,"metrics":{"packages_coverage_seconds":590,"examples_seconds":115,"startup_seconds":66,"ci_run_id":32538912290},"status":"discard","description":"Restore original Cloudflare Postgres startup ordering","timestamp":1787357778143,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Starting Cloudflare Postgres before coverage would A/B whether deferral affected the critical path.","result":"The job passed in 776s with coverage at 590s. This is effectively tied with deferred-start post-fix samples (median 772s); moving an idle container did not produce a measurable benefit.","rollback_reason":"The point did not improve the retained 761s result.","decision":"Keep the original workflow ordering to minimize the final product diff; the optimization is entirely in Vitest pooling/concurrency, plus the teardown stabilization it required.","next_action_hint":"Update session notes, then test another independent Vitest setting or prepare the selected minimal diff."}} +{"run":43,"commit":"9377379","metric":763,"metrics":{"packages_coverage_seconds":590,"examples_seconds":112,"startup_seconds":57,"ci_run_id":32543138090},"status":"keep","description":"Use vmThreads default concurrency without a redundant 100% cap","timestamp":1787362543034,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Vitest's default worker count equals 100% of available CPUs, so removing the explicit cap should preserve performance with a smaller, clearer config change.","result":"The job passed in 763s, effectively equal to the 761s retained run. Coverage took 590s, within the selected pool's normal range.","simplicity":"The final config now changes only the CI pool. The comment accurately explains VM per-file isolation and package-level overrides instead of claiming 100% is a cap.","next_action_hint":"Try VM pool memory recycling only if logs show growth; otherwise the minimal one-line pool change is the strongest final candidate."}} diff --git a/vitest.config.ts b/vitest.config.ts index 4ffe16e2f484..31159ecc5cdc 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -7,8 +7,9 @@ const coveragePolicy = composeCoverageConfig(import.meta.dirname); export default defineConfig({ test: { projects: ['packages/**/vitest.config.ts'], - // Reuse CI worker threads while keeping a fresh VM context per test file. + // Reuse all CI runner cores while keeping a fresh VM context per test file. // Stateful projects can override this default, as the Supabase suite does. + maxWorkers: process.env['CI'] ? '100%' : undefined, pool: process.env['CI'] ? 'vmThreads' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender From 6bac16a6df909a79d4cae4d707b54bd926e73727 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 04:00:28 +0000 Subject: [PATCH 46/80] autoresearch: measure Test CI 20260822T040028Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .auto/prompt.md | 3 +++ vitest.config.ts | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index abfcbc39b8cd..489320bf67cb 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -42,3 +42,4 @@ {"run":41,"commit":"90a9c77","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Cloudflare startup A/B retry blocked by SSH disconnect","timestamp":1787354953888,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Retry the original Cloudflare startup ordering benchmark after hardening API discovery.","result":"No hosted run started because the SSH push connection was closed by GitHub after a long network stall.","error_details":"`fatal: Could not read from remote repository` after 2015s wall time.","rollback_reason":"External transport failure produced no metric.","next_action_hint":"Retry push/measurement when GitHub SSH recovers; consider adding bounded push retries/timeouts to the harness."}} {"run":42,"commit":"08dfbbb","metric":776,"metrics":{"packages_coverage_seconds":590,"examples_seconds":115,"startup_seconds":66,"ci_run_id":32538912290},"status":"discard","description":"Restore original Cloudflare Postgres startup ordering","timestamp":1787357778143,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Starting Cloudflare Postgres before coverage would A/B whether deferral affected the critical path.","result":"The job passed in 776s with coverage at 590s. This is effectively tied with deferred-start post-fix samples (median 772s); moving an idle container did not produce a measurable benefit.","rollback_reason":"The point did not improve the retained 761s result.","decision":"Keep the original workflow ordering to minimize the final product diff; the optimization is entirely in Vitest pooling/concurrency, plus the teardown stabilization it required.","next_action_hint":"Update session notes, then test another independent Vitest setting or prepare the selected minimal diff."}} {"run":43,"commit":"9377379","metric":763,"metrics":{"packages_coverage_seconds":590,"examples_seconds":112,"startup_seconds":57,"ci_run_id":32543138090},"status":"keep","description":"Use vmThreads default concurrency without a redundant 100% cap","timestamp":1787362543034,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Vitest's default worker count equals 100% of available CPUs, so removing the explicit cap should preserve performance with a smaller, clearer config change.","result":"The job passed in 763s, effectively equal to the 761s retained run. Coverage took 590s, within the selected pool's normal range.","simplicity":"The final config now changes only the CI pool. The comment accurately explains VM per-file isolation and package-level overrides instead of claiming 100% is a cap.","next_action_hint":"Try VM pool memory recycling only if logs show growth; otherwise the minimal one-line pool change is the strongest final candidate."}} +{"run":44,"commit":"8d9d788","metric":704,"metrics":{"packages_coverage_seconds":478,"examples_seconds":155,"startup_seconds":66,"ci_run_id":32546815915},"status":"keep","description":"Restore all-core VM-thread concurrency after teardown stabilization","timestamp":1787367293787,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Explicit 100% worker concurrency should outperform Vitest's default CPU-minus-one policy now that VM isolation and graceful teardown are stable.","result":"The job passed in 704s, the fastest safe serial-workload result. Package coverage fell to 478s; examples were a slower 155s, so the primary improvement came entirely from package execution.","variance_note":"478s coverage is unusually favorable relative to prior 100%-vmThreads samples (568-597s), so use the established median evidence—not this minimum alone—to justify all-core concurrency.","decision":"Retain explicit 100%: three pre-fix all-core VM jobs had a 752s median, and the post-fix all-core runs remain clean; CPU-minus-one was 763s in its sample.","next_action_hint":"Update the experiment prompt with the selected result, then inspect the final diff and run repository correctness checks."}} diff --git a/.auto/prompt.md b/.auto/prompt.md index dffb316e62c0..31f4ffe24212 100644 --- a/.auto/prompt.md +++ b/.auto/prompt.md @@ -48,3 +48,6 @@ Reduce the wall-clock duration of the `Test` job in `.github/workflows/ci.yml` o - Running package coverage and examples concurrently looked fast but was rejected after a real `ECONNRESET`/non-queryable Postgres failure. Do not retry workload overlap. - Full worker reuse (`isolate: false`) passed at 50% and 75%, but it is too broad: Supabase's config explicitly requires per-file isolation to prevent pg mocks leaking into integration files. Consider only targeted reuse in proven stateless projects. - Hosted-runner timing has large outliers. Compare repeated medians and stability rather than trusting minima; a 548s overlap run was followed by 739s and 735s unchanged runs. +- Selected candidate: CI uses `pool: 'vmThreads'` with `maxWorkers: '100%'`. VM contexts preserve file-level isolation while reusable threads avoid fork churn; Supabase retains its explicit `pool: 'forks'` and one-worker override. Three initial all-core VM runs passed at 741s, 761s, and 752s (median 752s), versus a 767s median at 75% and 814s at 50%. +- The language-server test harness had an existing intermittent teardown rejection across multiple pools. Normal teardown now awaits the standard LSP ShutdownRequest before disposal, while immediate disposal tests keep their old path. Three post-fix all-core jobs passed, followed by a 704s favorable-runner result. +- JSON-only coverage output showed no measurable phase-tail saving, so the text reporter was restored. Deferring Cloudflare Postgres startup did not improve coverage, so the original workflow order was restored. The intended product diff is limited to `vitest.config.ts` and language-server harness tests. diff --git a/vitest.config.ts b/vitest.config.ts index 31159ecc5cdc..293eb477bc8e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,6 +11,11 @@ export default defineConfig({ // Stateful projects can override this default, as the Supabase suite does. maxWorkers: process.env['CI'] ? '100%' : undefined, pool: process.env['CI'] ? 'vmThreads' : undefined, + deps: { + optimizer: { + ssr: { enabled: process.env['CI'] === 'true' }, + }, + }, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From bbadb42ac0ff02e358b086242666fc0cab989a67 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 05:05:55 +0000 Subject: [PATCH 47/80] autoresearch: measure Test CI 20260822T050554Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 489320bf67cb..0af1ef879452 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -43,3 +43,4 @@ {"run":42,"commit":"08dfbbb","metric":776,"metrics":{"packages_coverage_seconds":590,"examples_seconds":115,"startup_seconds":66,"ci_run_id":32538912290},"status":"discard","description":"Restore original Cloudflare Postgres startup ordering","timestamp":1787357778143,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Starting Cloudflare Postgres before coverage would A/B whether deferral affected the critical path.","result":"The job passed in 776s with coverage at 590s. This is effectively tied with deferred-start post-fix samples (median 772s); moving an idle container did not produce a measurable benefit.","rollback_reason":"The point did not improve the retained 761s result.","decision":"Keep the original workflow ordering to minimize the final product diff; the optimization is entirely in Vitest pooling/concurrency, plus the teardown stabilization it required.","next_action_hint":"Update session notes, then test another independent Vitest setting or prepare the selected minimal diff."}} {"run":43,"commit":"9377379","metric":763,"metrics":{"packages_coverage_seconds":590,"examples_seconds":112,"startup_seconds":57,"ci_run_id":32543138090},"status":"keep","description":"Use vmThreads default concurrency without a redundant 100% cap","timestamp":1787362543034,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Vitest's default worker count equals 100% of available CPUs, so removing the explicit cap should preserve performance with a smaller, clearer config change.","result":"The job passed in 763s, effectively equal to the 761s retained run. Coverage took 590s, within the selected pool's normal range.","simplicity":"The final config now changes only the CI pool. The comment accurately explains VM per-file isolation and package-level overrides instead of claiming 100% is a cap.","next_action_hint":"Try VM pool memory recycling only if logs show growth; otherwise the minimal one-line pool change is the strongest final candidate."}} {"run":44,"commit":"8d9d788","metric":704,"metrics":{"packages_coverage_seconds":478,"examples_seconds":155,"startup_seconds":66,"ci_run_id":32546815915},"status":"keep","description":"Restore all-core VM-thread concurrency after teardown stabilization","timestamp":1787367293787,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Explicit 100% worker concurrency should outperform Vitest's default CPU-minus-one policy now that VM isolation and graceful teardown are stable.","result":"The job passed in 704s, the fastest safe serial-workload result. Package coverage fell to 478s; examples were a slower 155s, so the primary improvement came entirely from package execution.","variance_note":"478s coverage is unusually favorable relative to prior 100%-vmThreads samples (568-597s), so use the established median evidence—not this minimum alone—to justify all-core concurrency.","decision":"Retain explicit 100%: three pre-fix all-core VM jobs had a 752s median, and the post-fix all-core runs remain clean; CPU-minus-one was 763s in its sample.","next_action_hint":"Update the experiment prompt with the selected result, then inspect the final diff and run repository correctness checks."}} +{"run":45,"commit":"6bac16a","metric":797,"metrics":{"packages_coverage_seconds":600,"examples_seconds":117,"startup_seconds":72,"ci_run_id":32550544989},"status":"discard","description":"Prebundle SSR dependencies for package tests","timestamp":1787372294147,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Vite SSR dependency optimization could reduce the 42-43% import share by prebundling repeatedly imported dependencies.","result":"The job passed but took 797s; coverage was 600s, slower than the selected VM-thread distribution. Cold prebundle work did not pay back in one CI run.","rollback_reason":"Primary metric regressed 93s versus the latest selected run and ~45s versus the all-core VM median.","next_action_hint":"Remove dependency optimization. Consider import-graph changes only with per-module diagnostics, not global cold prebundling."}} diff --git a/vitest.config.ts b/vitest.config.ts index 293eb477bc8e..058bdb7994a5 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,9 +11,10 @@ export default defineConfig({ // Stateful projects can override this default, as the Supabase suite does. maxWorkers: process.env['CI'] ? '100%' : undefined, pool: process.env['CI'] ? 'vmThreads' : undefined, - deps: { - optimizer: { - ssr: { enabled: process.env['CI'] === 'true' }, + experimental: { + importDurations: { + print: process.env['CI'] ? true : false, + limit: 20, }, }, // Hard-suppress telemetry across every package test suite. The CLI's From 8f085fb98f151043f0cf668828afdcd91592a1e7 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 05:51:53 +0000 Subject: [PATCH 48/80] autoresearch: measure Test CI 20260822T055153Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 10 ++-------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 0af1ef879452..6fb3cf5d576a 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -44,3 +44,4 @@ {"run":43,"commit":"9377379","metric":763,"metrics":{"packages_coverage_seconds":590,"examples_seconds":112,"startup_seconds":57,"ci_run_id":32543138090},"status":"keep","description":"Use vmThreads default concurrency without a redundant 100% cap","timestamp":1787362543034,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Vitest's default worker count equals 100% of available CPUs, so removing the explicit cap should preserve performance with a smaller, clearer config change.","result":"The job passed in 763s, effectively equal to the 761s retained run. Coverage took 590s, within the selected pool's normal range.","simplicity":"The final config now changes only the CI pool. The comment accurately explains VM per-file isolation and package-level overrides instead of claiming 100% is a cap.","next_action_hint":"Try VM pool memory recycling only if logs show growth; otherwise the minimal one-line pool change is the strongest final candidate."}} {"run":44,"commit":"8d9d788","metric":704,"metrics":{"packages_coverage_seconds":478,"examples_seconds":155,"startup_seconds":66,"ci_run_id":32546815915},"status":"keep","description":"Restore all-core VM-thread concurrency after teardown stabilization","timestamp":1787367293787,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Explicit 100% worker concurrency should outperform Vitest's default CPU-minus-one policy now that VM isolation and graceful teardown are stable.","result":"The job passed in 704s, the fastest safe serial-workload result. Package coverage fell to 478s; examples were a slower 155s, so the primary improvement came entirely from package execution.","variance_note":"478s coverage is unusually favorable relative to prior 100%-vmThreads samples (568-597s), so use the established median evidence—not this minimum alone—to justify all-core concurrency.","decision":"Retain explicit 100%: three pre-fix all-core VM jobs had a 752s median, and the post-fix all-core runs remain clean; CPU-minus-one was 763s in its sample.","next_action_hint":"Update the experiment prompt with the selected result, then inspect the final diff and run repository correctness checks."}} {"run":45,"commit":"6bac16a","metric":797,"metrics":{"packages_coverage_seconds":600,"examples_seconds":117,"startup_seconds":72,"ci_run_id":32550544989},"status":"discard","description":"Prebundle SSR dependencies for package tests","timestamp":1787372294147,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Vite SSR dependency optimization could reduce the 42-43% import share by prebundling repeatedly imported dependencies.","result":"The job passed but took 797s; coverage was 600s, slower than the selected VM-thread distribution. Cold prebundle work did not pay back in one CI run.","rollback_reason":"Primary metric regressed 93s versus the latest selected run and ~45s versus the all-core VM median.","next_action_hint":"Remove dependency optimization. Consider import-graph changes only with per-module diagnostics, not global cold prebundling."}} +{"run":46,"commit":"bbadb42","metric":728,"metrics":{"packages_coverage_seconds":552,"examples_seconds":110,"startup_seconds":62,"ci_run_id":32553470674},"status":"discard","description":"Collect Vitest import-duration diagnostics","timestamp":1787377552098,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Import-duration instrumentation would identify a small set of module-graph hotspots behind the 42% import share.","result":"The job passed in 728s, but no per-import breakdown appeared for the root multi-project package run, so the instrumentation yielded no actionable hotspot data. The fast timing is consistent with runner variance.","rollback_reason":"Instrumentation adds config/output complexity without producing its intended diagnostic and does not beat the 704s selected minimum.","next_action_hint":"Remove importDurations. Avoid broad import optimization without actionable module-level evidence."}} diff --git a/vitest.config.ts b/vitest.config.ts index 058bdb7994a5..ec96e48dfdf0 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -7,16 +7,10 @@ const coveragePolicy = composeCoverageConfig(import.meta.dirname); export default defineConfig({ test: { projects: ['packages/**/vitest.config.ts'], - // Reuse all CI runner cores while keeping a fresh VM context per test file. + // Reuse CI worker threads while keeping a fresh VM context per test file. // Stateful projects can override this default, as the Supabase suite does. - maxWorkers: process.env['CI'] ? '100%' : undefined, + maxWorkers: process.env['CI'] ? 5 : undefined, pool: process.env['CI'] ? 'vmThreads' : undefined, - experimental: { - importDurations: { - print: process.env['CI'] ? true : false, - limit: 20, - }, - }, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 912d02d645fe0e3331780b0fa4ce39aabc67bf13 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 06:40:03 +0000 Subject: [PATCH 49/80] autoresearch: measure Test CI 20260822T064003Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 6fb3cf5d576a..37462ccd5411 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -45,3 +45,4 @@ {"run":44,"commit":"8d9d788","metric":704,"metrics":{"packages_coverage_seconds":478,"examples_seconds":155,"startup_seconds":66,"ci_run_id":32546815915},"status":"keep","description":"Restore all-core VM-thread concurrency after teardown stabilization","timestamp":1787367293787,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Explicit 100% worker concurrency should outperform Vitest's default CPU-minus-one policy now that VM isolation and graceful teardown are stable.","result":"The job passed in 704s, the fastest safe serial-workload result. Package coverage fell to 478s; examples were a slower 155s, so the primary improvement came entirely from package execution.","variance_note":"478s coverage is unusually favorable relative to prior 100%-vmThreads samples (568-597s), so use the established median evidence—not this minimum alone—to justify all-core concurrency.","decision":"Retain explicit 100%: three pre-fix all-core VM jobs had a 752s median, and the post-fix all-core runs remain clean; CPU-minus-one was 763s in its sample.","next_action_hint":"Update the experiment prompt with the selected result, then inspect the final diff and run repository correctness checks."}} {"run":45,"commit":"6bac16a","metric":797,"metrics":{"packages_coverage_seconds":600,"examples_seconds":117,"startup_seconds":72,"ci_run_id":32550544989},"status":"discard","description":"Prebundle SSR dependencies for package tests","timestamp":1787372294147,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Vite SSR dependency optimization could reduce the 42-43% import share by prebundling repeatedly imported dependencies.","result":"The job passed but took 797s; coverage was 600s, slower than the selected VM-thread distribution. Cold prebundle work did not pay back in one CI run.","rollback_reason":"Primary metric regressed 93s versus the latest selected run and ~45s versus the all-core VM median.","next_action_hint":"Remove dependency optimization. Consider import-graph changes only with per-module diagnostics, not global cold prebundling."}} {"run":46,"commit":"bbadb42","metric":728,"metrics":{"packages_coverage_seconds":552,"examples_seconds":110,"startup_seconds":62,"ci_run_id":32553470674},"status":"discard","description":"Collect Vitest import-duration diagnostics","timestamp":1787377552098,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Import-duration instrumentation would identify a small set of module-graph hotspots behind the 42% import share.","result":"The job passed in 728s, but no per-import breakdown appeared for the root multi-project package run, so the instrumentation yielded no actionable hotspot data. The fast timing is consistent with runner variance.","rollback_reason":"Instrumentation adds config/output complexity without producing its intended diagnostic and does not beat the 704s selected minimum.","next_action_hint":"Remove importDurations. Avoid broad import optimization without actionable module-level evidence."}} +{"run":47,"commit":"8f085fb","metric":616,"metrics":{"packages_coverage_seconds":445,"examples_seconds":105,"startup_seconds":62,"ci_run_id":32555459998},"status":"keep","description":"Run five VM-isolated workers on the four-core CI runner","timestamp":1787378922265,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"One oversubscribed VM worker can hide database/import waits in a workload split roughly evenly between test execution and imports.","result":"The job passed in 616s, 88s faster than the prior safe minimum. Coverage took 445s and examples 105s; no database, memory, or teardown failure occurred.","risk":"A fixed worker count of five assumes current runner capacity and oversubscription may reintroduce resource flakes. The first point may also be a favorable runner outlier.","next_action_hint":"Repeat maxWorkers=5 at least twice before retaining; reject on any resource instability."}} From 13d01601e33f1b331fe11d0cd4961ca9674412c7 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 07:08:40 +0000 Subject: [PATCH 50/80] autoresearch: measure Test CI 20260822T070840Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 37462ccd5411..65218a8a95c6 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -46,3 +46,4 @@ {"run":45,"commit":"6bac16a","metric":797,"metrics":{"packages_coverage_seconds":600,"examples_seconds":117,"startup_seconds":72,"ci_run_id":32550544989},"status":"discard","description":"Prebundle SSR dependencies for package tests","timestamp":1787372294147,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Vite SSR dependency optimization could reduce the 42-43% import share by prebundling repeatedly imported dependencies.","result":"The job passed but took 797s; coverage was 600s, slower than the selected VM-thread distribution. Cold prebundle work did not pay back in one CI run.","rollback_reason":"Primary metric regressed 93s versus the latest selected run and ~45s versus the all-core VM median.","next_action_hint":"Remove dependency optimization. Consider import-graph changes only with per-module diagnostics, not global cold prebundling."}} {"run":46,"commit":"bbadb42","metric":728,"metrics":{"packages_coverage_seconds":552,"examples_seconds":110,"startup_seconds":62,"ci_run_id":32553470674},"status":"discard","description":"Collect Vitest import-duration diagnostics","timestamp":1787377552098,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Import-duration instrumentation would identify a small set of module-graph hotspots behind the 42% import share.","result":"The job passed in 728s, but no per-import breakdown appeared for the root multi-project package run, so the instrumentation yielded no actionable hotspot data. The fast timing is consistent with runner variance.","rollback_reason":"Instrumentation adds config/output complexity without producing its intended diagnostic and does not beat the 704s selected minimum.","next_action_hint":"Remove importDurations. Avoid broad import optimization without actionable module-level evidence."}} {"run":47,"commit":"8f085fb","metric":616,"metrics":{"packages_coverage_seconds":445,"examples_seconds":105,"startup_seconds":62,"ci_run_id":32555459998},"status":"keep","description":"Run five VM-isolated workers on the four-core CI runner","timestamp":1787378922265,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"One oversubscribed VM worker can hide database/import waits in a workload split roughly evenly between test execution and imports.","result":"The job passed in 616s, 88s faster than the prior safe minimum. Coverage took 445s and examples 105s; no database, memory, or teardown failure occurred.","risk":"A fixed worker count of five assumes current runner capacity and oversubscription may reintroduce resource flakes. The first point may also be a favorable runner outlier.","next_action_hint":"Repeat maxWorkers=5 at least twice before retaining; reject on any resource instability."}} +{"run":48,"commit":"912d02d","metric":732,"metrics":{"packages_coverage_seconds":557,"examples_seconds":111,"startup_seconds":58,"ci_run_id":32557576559},"status":"discard","description":"Second five-worker VM isolation run","timestamp":1787382506588,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Five workers would repeat the large all-core-plus-one throughput gain.","result":"The second run passed in 732s with coverage at 557s. It did not reproduce the 616s outlier, but remains about 20s below the four-worker median.","rollback_reason":"The 732s point regressed from the retained 616s minimum; no code delta exists.","next_action_hint":"Collect a third five-worker sample to compare medians and resource stability against four workers."}} From d1c5cf62851d69d9766b6527461a62f816acad94 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 08:19:21 +0000 Subject: [PATCH 51/80] autoresearch: measure Test CI 20260822T081920Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 65218a8a95c6..6777febf04a1 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -47,3 +47,4 @@ {"run":46,"commit":"bbadb42","metric":728,"metrics":{"packages_coverage_seconds":552,"examples_seconds":110,"startup_seconds":62,"ci_run_id":32553470674},"status":"discard","description":"Collect Vitest import-duration diagnostics","timestamp":1787377552098,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Import-duration instrumentation would identify a small set of module-graph hotspots behind the 42% import share.","result":"The job passed in 728s, but no per-import breakdown appeared for the root multi-project package run, so the instrumentation yielded no actionable hotspot data. The fast timing is consistent with runner variance.","rollback_reason":"Instrumentation adds config/output complexity without producing its intended diagnostic and does not beat the 704s selected minimum.","next_action_hint":"Remove importDurations. Avoid broad import optimization without actionable module-level evidence."}} {"run":47,"commit":"8f085fb","metric":616,"metrics":{"packages_coverage_seconds":445,"examples_seconds":105,"startup_seconds":62,"ci_run_id":32555459998},"status":"keep","description":"Run five VM-isolated workers on the four-core CI runner","timestamp":1787378922265,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"One oversubscribed VM worker can hide database/import waits in a workload split roughly evenly between test execution and imports.","result":"The job passed in 616s, 88s faster than the prior safe minimum. Coverage took 445s and examples 105s; no database, memory, or teardown failure occurred.","risk":"A fixed worker count of five assumes current runner capacity and oversubscription may reintroduce resource flakes. The first point may also be a favorable runner outlier.","next_action_hint":"Repeat maxWorkers=5 at least twice before retaining; reject on any resource instability."}} {"run":48,"commit":"912d02d","metric":732,"metrics":{"packages_coverage_seconds":557,"examples_seconds":111,"startup_seconds":58,"ci_run_id":32557576559},"status":"discard","description":"Second five-worker VM isolation run","timestamp":1787382506588,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Five workers would repeat the large all-core-plus-one throughput gain.","result":"The second run passed in 732s with coverage at 557s. It did not reproduce the 616s outlier, but remains about 20s below the four-worker median.","rollback_reason":"The 732s point regressed from the retained 616s minimum; no code delta exists.","next_action_hint":"Collect a third five-worker sample to compare medians and resource stability against four workers."}} +{"run":49,"commit":"13d0160","metric":0,"metrics":{"packages_coverage_seconds":584,"examples_seconds":112,"startup_seconds":68,"ci_run_id":32558829664},"status":"crash","description":"Third five-worker VM isolation stability run","timestamp":1787384097665,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"A third five-worker run would establish oversubscription stability.","result":"The run failed: adapter-postgres `rls-migration-plan.integration.test.ts` timed out at its intentional 200ms limit despite being offline. 1,164 other files passed.","error_details":"The package phase ran about 584s before failure. Five concurrent workers can starve a fast test's event loop enough to violate tight correctness timeouts.","rollback_reason":"Oversubscribing beyond runner cores introduces unacceptable timing flakes; reject maxWorkers=5 despite its faster successful runs.","next_action_hint":"Restore maxWorkers='100%' (four cores). Do not raise test timeouts to accommodate an optimization-induced resource flake."}} diff --git a/vitest.config.ts b/vitest.config.ts index ec96e48dfdf0..cbba40e2537b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -6,10 +6,10 @@ const coveragePolicy = composeCoverageConfig(import.meta.dirname); export default defineConfig({ test: { - projects: ['packages/**/vitest.config.ts'], - // Reuse CI worker threads while keeping a fresh VM context per test file. + projects: ['packages/3-extensions/supabase/vitest.config.ts', 'packages/**/vitest.config.ts'], + // Reuse all CI runner cores while keeping a fresh VM context per test file. // Stateful projects can override this default, as the Supabase suite does. - maxWorkers: process.env['CI'] ? 5 : undefined, + maxWorkers: process.env['CI'] ? '100%' : undefined, pool: process.env['CI'] ? 'vmThreads' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender From 5991d512992726c9a62758e1fb9be84741a593a4 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 10:21:49 +0000 Subject: [PATCH 52/80] autoresearch: measure Test CI 20260822T102148Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + packages/3-extensions/supabase/vitest.config.ts | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 6777febf04a1..211e2d323306 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -48,3 +48,4 @@ {"run":47,"commit":"8f085fb","metric":616,"metrics":{"packages_coverage_seconds":445,"examples_seconds":105,"startup_seconds":62,"ci_run_id":32555459998},"status":"keep","description":"Run five VM-isolated workers on the four-core CI runner","timestamp":1787378922265,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"One oversubscribed VM worker can hide database/import waits in a workload split roughly evenly between test execution and imports.","result":"The job passed in 616s, 88s faster than the prior safe minimum. Coverage took 445s and examples 105s; no database, memory, or teardown failure occurred.","risk":"A fixed worker count of five assumes current runner capacity and oversubscription may reintroduce resource flakes. The first point may also be a favorable runner outlier.","next_action_hint":"Repeat maxWorkers=5 at least twice before retaining; reject on any resource instability."}} {"run":48,"commit":"912d02d","metric":732,"metrics":{"packages_coverage_seconds":557,"examples_seconds":111,"startup_seconds":58,"ci_run_id":32557576559},"status":"discard","description":"Second five-worker VM isolation run","timestamp":1787382506588,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Five workers would repeat the large all-core-plus-one throughput gain.","result":"The second run passed in 732s with coverage at 557s. It did not reproduce the 616s outlier, but remains about 20s below the four-worker median.","rollback_reason":"The 732s point regressed from the retained 616s minimum; no code delta exists.","next_action_hint":"Collect a third five-worker sample to compare medians and resource stability against four workers."}} {"run":49,"commit":"13d0160","metric":0,"metrics":{"packages_coverage_seconds":584,"examples_seconds":112,"startup_seconds":68,"ci_run_id":32558829664},"status":"crash","description":"Third five-worker VM isolation stability run","timestamp":1787384097665,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"A third five-worker run would establish oversubscription stability.","result":"The run failed: adapter-postgres `rls-migration-plan.integration.test.ts` timed out at its intentional 200ms limit despite being offline. 1,164 other files passed.","error_details":"The package phase ran about 584s before failure. Five concurrent workers can starve a fast test's event loop enough to violate tight correctness timeouts.","rollback_reason":"Oversubscribing beyond runner cores introduces unacceptable timing flakes; reject maxWorkers=5 despite its faster successful runs.","next_action_hint":"Restore maxWorkers='100%' (four cores). Do not raise test timeouts to accommodate an optimization-induced resource flake."}} +{"run":50,"commit":"d1c5cf6","metric":609,"metrics":{"packages_coverage_seconds":416,"examples_seconds":117,"startup_seconds":70,"ci_run_id":32561989841},"status":"keep","description":"List the serialized Supabase project first in Vitest projects","timestamp":1787389682193,"segment":0,"confidence":9.492063492063492,"asi":{"hypothesis":"Putting the one-worker Supabase suite first might overlap its serialized work with the rest of the project graph and eliminate its ~60s tail.","result":"The job passed in 609s with coverage at 416s, but log timestamps show Supabase still began near the end (08:27:48) and finished at 08:28:33 before the 08:28:37 summary. The configured project order did not alter scheduling.","variance_note":"The fast result is favorable runner variance, not evidence for the ordering change; adapter-postgres began six minutes before Supabase exactly as before.","next_action_hint":"Repeat once to confirm, then remove the redundant explicit project path unless ordering changes in the second log."}} diff --git a/packages/3-extensions/supabase/vitest.config.ts b/packages/3-extensions/supabase/vitest.config.ts index 433fac02b51b..43365f51957e 100644 --- a/packages/3-extensions/supabase/vitest.config.ts +++ b/packages/3-extensions/supabase/vitest.config.ts @@ -17,7 +17,6 @@ export default defineConfig({ // hosted these tests. Per-file isolation stays on: supabase-facade // mocks the pg module, and a shared module registry would leak the // real pg into it (or the mock into the integration files). - pool: 'forks', maxWorkers: 1, testTimeout: timeouts.default, hookTimeout: timeouts.default, From 6573cf67b281c6ced4fc356f8b985fe8638f2ab7 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 12:23:58 +0000 Subject: [PATCH 53/80] autoresearch: measure Test CI 20260822T122357Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + packages/3-extensions/supabase/vitest.config.ts | 1 + vitest.config.ts | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 211e2d323306..f09c0a74975c 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -49,3 +49,4 @@ {"run":48,"commit":"912d02d","metric":732,"metrics":{"packages_coverage_seconds":557,"examples_seconds":111,"startup_seconds":58,"ci_run_id":32557576559},"status":"discard","description":"Second five-worker VM isolation run","timestamp":1787382506588,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Five workers would repeat the large all-core-plus-one throughput gain.","result":"The second run passed in 732s with coverage at 557s. It did not reproduce the 616s outlier, but remains about 20s below the four-worker median.","rollback_reason":"The 732s point regressed from the retained 616s minimum; no code delta exists.","next_action_hint":"Collect a third five-worker sample to compare medians and resource stability against four workers."}} {"run":49,"commit":"13d0160","metric":0,"metrics":{"packages_coverage_seconds":584,"examples_seconds":112,"startup_seconds":68,"ci_run_id":32558829664},"status":"crash","description":"Third five-worker VM isolation stability run","timestamp":1787384097665,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"A third five-worker run would establish oversubscription stability.","result":"The run failed: adapter-postgres `rls-migration-plan.integration.test.ts` timed out at its intentional 200ms limit despite being offline. 1,164 other files passed.","error_details":"The package phase ran about 584s before failure. Five concurrent workers can starve a fast test's event loop enough to violate tight correctness timeouts.","rollback_reason":"Oversubscribing beyond runner cores introduces unacceptable timing flakes; reject maxWorkers=5 despite its faster successful runs.","next_action_hint":"Restore maxWorkers='100%' (four cores). Do not raise test timeouts to accommodate an optimization-induced resource flake."}} {"run":50,"commit":"d1c5cf6","metric":609,"metrics":{"packages_coverage_seconds":416,"examples_seconds":117,"startup_seconds":70,"ci_run_id":32561989841},"status":"keep","description":"List the serialized Supabase project first in Vitest projects","timestamp":1787389682193,"segment":0,"confidence":9.492063492063492,"asi":{"hypothesis":"Putting the one-worker Supabase suite first might overlap its serialized work with the rest of the project graph and eliminate its ~60s tail.","result":"The job passed in 609s with coverage at 416s, but log timestamps show Supabase still began near the end (08:27:48) and finished at 08:28:33 before the 08:28:37 summary. The configured project order did not alter scheduling.","variance_note":"The fast result is favorable runner variance, not evidence for the ordering change; adapter-postgres began six minutes before Supabase exactly as before.","next_action_hint":"Repeat once to confirm, then remove the redundant explicit project path unless ordering changes in the second log."}} +{"run":51,"commit":"5991d51","metric":775,"metrics":{"packages_coverage_seconds":585,"examples_seconds":114,"startup_seconds":68,"ci_run_id":32567703419},"status":"discard","description":"Let Supabase inherit VM threads while prioritizing its project","timestamp":1787396361780,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Moving Supabase into the common VM pool plus listing it first might start its one-worker suite early and remove its serialized tail.","result":"The suite passed, proving vmThreads compatibility in this run, but Supabase still started last (10:41:08 versus adapter-postgres at 10:32:33) and coverage took 585s. Neither project order nor pool unification changed scheduling.","rollback_reason":"Primary metric regressed 166s from the favorable prior point and the changes remove a documented Supabase safety override without benefit.","next_action_hint":"Restore the single project glob and Supabase `pool: 'forks'`. The Supabase tail appears intrinsic to Vitest scheduling and is not worth custom sequencing complexity."}} diff --git a/packages/3-extensions/supabase/vitest.config.ts b/packages/3-extensions/supabase/vitest.config.ts index 43365f51957e..433fac02b51b 100644 --- a/packages/3-extensions/supabase/vitest.config.ts +++ b/packages/3-extensions/supabase/vitest.config.ts @@ -17,6 +17,7 @@ export default defineConfig({ // hosted these tests. Per-file isolation stays on: supabase-facade // mocks the pg module, and a shared module registry would leak the // real pg into it (or the mock into the integration files). + pool: 'forks', maxWorkers: 1, testTimeout: timeouts.default, hookTimeout: timeouts.default, diff --git a/vitest.config.ts b/vitest.config.ts index cbba40e2537b..31159ecc5cdc 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -6,7 +6,7 @@ const coveragePolicy = composeCoverageConfig(import.meta.dirname); export default defineConfig({ test: { - projects: ['packages/3-extensions/supabase/vitest.config.ts', 'packages/**/vitest.config.ts'], + projects: ['packages/**/vitest.config.ts'], // Reuse all CI runner cores while keeping a fresh VM context per test file. // Stateful projects can override this default, as the Supabase suite does. maxWorkers: process.env['CI'] ? '100%' : undefined, From a91676f75d44a96c423b4b3541edd07c5f7ecf3b Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 13:42:58 +0000 Subject: [PATCH 54/80] autoresearch: measure Test CI 20260822T134257Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index f09c0a74975c..938e9a7983f0 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -50,3 +50,4 @@ {"run":49,"commit":"13d0160","metric":0,"metrics":{"packages_coverage_seconds":584,"examples_seconds":112,"startup_seconds":68,"ci_run_id":32558829664},"status":"crash","description":"Third five-worker VM isolation stability run","timestamp":1787384097665,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"A third five-worker run would establish oversubscription stability.","result":"The run failed: adapter-postgres `rls-migration-plan.integration.test.ts` timed out at its intentional 200ms limit despite being offline. 1,164 other files passed.","error_details":"The package phase ran about 584s before failure. Five concurrent workers can starve a fast test's event loop enough to violate tight correctness timeouts.","rollback_reason":"Oversubscribing beyond runner cores introduces unacceptable timing flakes; reject maxWorkers=5 despite its faster successful runs.","next_action_hint":"Restore maxWorkers='100%' (four cores). Do not raise test timeouts to accommodate an optimization-induced resource flake."}} {"run":50,"commit":"d1c5cf6","metric":609,"metrics":{"packages_coverage_seconds":416,"examples_seconds":117,"startup_seconds":70,"ci_run_id":32561989841},"status":"keep","description":"List the serialized Supabase project first in Vitest projects","timestamp":1787389682193,"segment":0,"confidence":9.492063492063492,"asi":{"hypothesis":"Putting the one-worker Supabase suite first might overlap its serialized work with the rest of the project graph and eliminate its ~60s tail.","result":"The job passed in 609s with coverage at 416s, but log timestamps show Supabase still began near the end (08:27:48) and finished at 08:28:33 before the 08:28:37 summary. The configured project order did not alter scheduling.","variance_note":"The fast result is favorable runner variance, not evidence for the ordering change; adapter-postgres began six minutes before Supabase exactly as before.","next_action_hint":"Repeat once to confirm, then remove the redundant explicit project path unless ordering changes in the second log."}} {"run":51,"commit":"5991d51","metric":775,"metrics":{"packages_coverage_seconds":585,"examples_seconds":114,"startup_seconds":68,"ci_run_id":32567703419},"status":"discard","description":"Let Supabase inherit VM threads while prioritizing its project","timestamp":1787396361780,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Moving Supabase into the common VM pool plus listing it first might start its one-worker suite early and remove its serialized tail.","result":"The suite passed, proving vmThreads compatibility in this run, but Supabase still started last (10:41:08 versus adapter-postgres at 10:32:33) and coverage took 585s. Neither project order nor pool unification changed scheduling.","rollback_reason":"Primary metric regressed 166s from the favorable prior point and the changes remove a documented Supabase safety override without benefit.","next_action_hint":"Restore the single project glob and Supabase `pool: 'forks'`. The Supabase tail appears intrinsic to Vitest scheduling and is not worth custom sequencing complexity."}} +{"run":52,"commit":"6573cf6","metric":770,"metrics":{"packages_coverage_seconds":588,"examples_seconds":114,"startup_seconds":62,"ci_run_id":32573118090},"status":"keep","description":"Restore minimal selected pool configuration after scheduling experiments","timestamp":1787405080228,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Returning to the single project glob and Supabase's forks override would preserve the proven VM-thread speedup with the intended safety boundaries.","result":"The exact final candidate passed in 770s, 77s (9.1%) below baseline. Coverage took 588s; examples took 114s.","correctness":"Supabase again uses its documented single fork, project discovery is unchanged, all-core vmThreads remains the only root execution optimization, and graceful LSP teardown remains active.","decision":"This is the minimal final product shape. Discard further scheduler/oversubscription ideas unless new evidence appears."}} diff --git a/vitest.config.ts b/vitest.config.ts index 31159ecc5cdc..dd0834cf5ab3 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -10,7 +10,7 @@ export default defineConfig({ // Reuse all CI runner cores while keeping a fresh VM context per test file. // Stateful projects can override this default, as the Supabase suite does. maxWorkers: process.env['CI'] ? '100%' : undefined, - pool: process.env['CI'] ? 'vmThreads' : undefined, + pool: process.env['CI'] ? 'threads' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 9b9a36ac7c3ff4afb07019a565b2a4208170b40e Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 14:41:39 +0000 Subject: [PATCH 55/80] autoresearch: measure Test CI 20260822T144138Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 938e9a7983f0..160d53d09291 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -51,3 +51,4 @@ {"run":50,"commit":"d1c5cf6","metric":609,"metrics":{"packages_coverage_seconds":416,"examples_seconds":117,"startup_seconds":70,"ci_run_id":32561989841},"status":"keep","description":"List the serialized Supabase project first in Vitest projects","timestamp":1787389682193,"segment":0,"confidence":9.492063492063492,"asi":{"hypothesis":"Putting the one-worker Supabase suite first might overlap its serialized work with the rest of the project graph and eliminate its ~60s tail.","result":"The job passed in 609s with coverage at 416s, but log timestamps show Supabase still began near the end (08:27:48) and finished at 08:28:33 before the 08:28:37 summary. The configured project order did not alter scheduling.","variance_note":"The fast result is favorable runner variance, not evidence for the ordering change; adapter-postgres began six minutes before Supabase exactly as before.","next_action_hint":"Repeat once to confirm, then remove the redundant explicit project path unless ordering changes in the second log."}} {"run":51,"commit":"5991d51","metric":775,"metrics":{"packages_coverage_seconds":585,"examples_seconds":114,"startup_seconds":68,"ci_run_id":32567703419},"status":"discard","description":"Let Supabase inherit VM threads while prioritizing its project","timestamp":1787396361780,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Moving Supabase into the common VM pool plus listing it first might start its one-worker suite early and remove its serialized tail.","result":"The suite passed, proving vmThreads compatibility in this run, but Supabase still started last (10:41:08 versus adapter-postgres at 10:32:33) and coverage took 585s. Neither project order nor pool unification changed scheduling.","rollback_reason":"Primary metric regressed 166s from the favorable prior point and the changes remove a documented Supabase safety override without benefit.","next_action_hint":"Restore the single project glob and Supabase `pool: 'forks'`. The Supabase tail appears intrinsic to Vitest scheduling and is not worth custom sequencing complexity."}} {"run":52,"commit":"6573cf6","metric":770,"metrics":{"packages_coverage_seconds":588,"examples_seconds":114,"startup_seconds":62,"ci_run_id":32573118090},"status":"keep","description":"Restore minimal selected pool configuration after scheduling experiments","timestamp":1787405080228,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Returning to the single project glob and Supabase's forks override would preserve the proven VM-thread speedup with the intended safety boundaries.","result":"The exact final candidate passed in 770s, 77s (9.1%) below baseline. Coverage took 588s; examples took 114s.","correctness":"Supabase again uses its documented single fork, project discovery is unchanged, all-core vmThreads remains the only root execution optimization, and graceful LSP teardown remains active.","decision":"This is the minimal final product shape. Discard further scheduler/oversubscription ideas unless new evidence appears."}} +{"run":53,"commit":"a91676f","metric":764,"metrics":{"packages_coverage_seconds":583,"examples_seconds":113,"startup_seconds":62,"ci_run_id":32577334208},"status":"discard","description":"Use plain isolated worker threads on all CI cores","timestamp":1787408351854,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"Plain threads might match VM-thread throughput while avoiding cross-realm semantics, at the cost of creating a fresh thread worker per isolated file.","result":"The job passed in 764s, effectively tied with the exact vmThreads final run at 770s. Coverage was 583s.","rollback_reason":"One tied sample does not outweigh vmThreads' three-sample stable evidence or demonstrate an improvement.","next_action_hint":"Repeat plain threads once; prefer threads if it remains tied and stability is equal because semantics are closer to the original fork pool."}} From e82363bdd4893bb808afb8bc4f19bc6a820056ce Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 15:26:46 +0000 Subject: [PATCH 56/80] autoresearch: measure Test CI 20260822T152645Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .auto/measure.sh | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 160d53d09291..247c82e63162 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -52,3 +52,4 @@ {"run":51,"commit":"5991d51","metric":775,"metrics":{"packages_coverage_seconds":585,"examples_seconds":114,"startup_seconds":68,"ci_run_id":32567703419},"status":"discard","description":"Let Supabase inherit VM threads while prioritizing its project","timestamp":1787396361780,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Moving Supabase into the common VM pool plus listing it first might start its one-worker suite early and remove its serialized tail.","result":"The suite passed, proving vmThreads compatibility in this run, but Supabase still started last (10:41:08 versus adapter-postgres at 10:32:33) and coverage took 585s. Neither project order nor pool unification changed scheduling.","rollback_reason":"Primary metric regressed 166s from the favorable prior point and the changes remove a documented Supabase safety override without benefit.","next_action_hint":"Restore the single project glob and Supabase `pool: 'forks'`. The Supabase tail appears intrinsic to Vitest scheduling and is not worth custom sequencing complexity."}} {"run":52,"commit":"6573cf6","metric":770,"metrics":{"packages_coverage_seconds":588,"examples_seconds":114,"startup_seconds":62,"ci_run_id":32573118090},"status":"keep","description":"Restore minimal selected pool configuration after scheduling experiments","timestamp":1787405080228,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Returning to the single project glob and Supabase's forks override would preserve the proven VM-thread speedup with the intended safety boundaries.","result":"The exact final candidate passed in 770s, 77s (9.1%) below baseline. Coverage took 588s; examples took 114s.","correctness":"Supabase again uses its documented single fork, project discovery is unchanged, all-core vmThreads remains the only root execution optimization, and graceful LSP teardown remains active.","decision":"This is the minimal final product shape. Discard further scheduler/oversubscription ideas unless new evidence appears."}} {"run":53,"commit":"a91676f","metric":764,"metrics":{"packages_coverage_seconds":583,"examples_seconds":113,"startup_seconds":62,"ci_run_id":32577334208},"status":"discard","description":"Use plain isolated worker threads on all CI cores","timestamp":1787408351854,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"Plain threads might match VM-thread throughput while avoiding cross-realm semantics, at the cost of creating a fresh thread worker per isolated file.","result":"The job passed in 764s, effectively tied with the exact vmThreads final run at 770s. Coverage was 583s.","rollback_reason":"One tied sample does not outweigh vmThreads' three-sample stable evidence or demonstrate an improvement.","next_action_hint":"Repeat plain threads once; prefer threads if it remains tied and stability is equal because semantics are closer to the original fork pool."}} +{"run":54,"commit":"9b9a36a","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Plain-thread repeat blocked by GitHub API connectivity","timestamp":1787410632292,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"A second plain-thread run would compare stability and median against vmThreads.","result":"The commit pushed, but measure.sh failed before run discovery when api.github.com was unreachable.","error_details":"`error connecting to api.github.com`; no candidate result acquired in this cycle.","rollback_reason":"External measurement-harness failure.","next_action_hint":"Retry unchanged; harden PR lookup API retries if connectivity failures persist."}} diff --git a/.auto/measure.sh b/.auto/measure.sh index 544979d5b7a0..f2f426dc0e8c 100755 --- a/.auto/measure.sh +++ b/.auto/measure.sh @@ -26,7 +26,12 @@ for _ in $(seq 1 3); do done [[ "$pushed" == true ]] || { echo "Could not push experiment after three attempts" >&2; exit 1; } -pr_number="$(gh pr list --repo "$repo" --state open --head "$branch" --json number --jq '.[0].number // empty')" +pr_number="" +for _ in $(seq 1 6); do + pr_number="$(gh pr list --repo "$repo" --state open --head "$branch" --json number --jq '.[0].number // empty' 2>/dev/null || true)" + [[ -n "$pr_number" ]] && break + sleep 10 +done if [[ -z "$pr_number" ]]; then pr_url="$(gh pr create --repo "$repo" --base main --head "$branch" --draft --title "autoresearch: speed up Test CI job" --body $'Temporary draft PR for hosted-CI performance experiments.\n\nDo not review or merge. The final result will be prepared separately.')" pr_number="${pr_url##*/}" From 96ac20f48752944ca27721b0c952c38819e56e7a Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 15:42:48 +0000 Subject: [PATCH 57/80] autoresearch: measure Test CI 20260822T154247Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 247c82e63162..09e88990ef65 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -53,3 +53,4 @@ {"run":52,"commit":"6573cf6","metric":770,"metrics":{"packages_coverage_seconds":588,"examples_seconds":114,"startup_seconds":62,"ci_run_id":32573118090},"status":"keep","description":"Restore minimal selected pool configuration after scheduling experiments","timestamp":1787405080228,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Returning to the single project glob and Supabase's forks override would preserve the proven VM-thread speedup with the intended safety boundaries.","result":"The exact final candidate passed in 770s, 77s (9.1%) below baseline. Coverage took 588s; examples took 114s.","correctness":"Supabase again uses its documented single fork, project discovery is unchanged, all-core vmThreads remains the only root execution optimization, and graceful LSP teardown remains active.","decision":"This is the minimal final product shape. Discard further scheduler/oversubscription ideas unless new evidence appears."}} {"run":53,"commit":"a91676f","metric":764,"metrics":{"packages_coverage_seconds":583,"examples_seconds":113,"startup_seconds":62,"ci_run_id":32577334208},"status":"discard","description":"Use plain isolated worker threads on all CI cores","timestamp":1787408351854,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"Plain threads might match VM-thread throughput while avoiding cross-realm semantics, at the cost of creating a fresh thread worker per isolated file.","result":"The job passed in 764s, effectively tied with the exact vmThreads final run at 770s. Coverage was 583s.","rollback_reason":"One tied sample does not outweigh vmThreads' three-sample stable evidence or demonstrate an improvement.","next_action_hint":"Repeat plain threads once; prefer threads if it remains tied and stability is equal because semantics are closer to the original fork pool."}} {"run":54,"commit":"9b9a36a","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Plain-thread repeat blocked by GitHub API connectivity","timestamp":1787410632292,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"A second plain-thread run would compare stability and median against vmThreads.","result":"The commit pushed, but measure.sh failed before run discovery when api.github.com was unreachable.","error_details":"`error connecting to api.github.com`; no candidate result acquired in this cycle.","rollback_reason":"External measurement-harness failure.","next_action_hint":"Retry unchanged; harden PR lookup API retries if connectivity failures persist."}} +{"run":55,"commit":"e82363b","metric":667,"metrics":{"packages_coverage_seconds":477,"examples_seconds":107,"startup_seconds":76,"ci_run_id":32581732238},"status":"keep","description":"Second successful all-core plain-thread run","timestamp":1787413350618,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Plain threads would repeat safely and might avoid VM-context overhead while retaining per-file isolation.","result":"The job passed in 667s with coverage at 477s. Two successful samples are 764s and 667s (median 715.5s), competitive with or better than vmThreads while using conventional thread isolation.","variance_note":"The 477s coverage phase is favorable and close to the best vmThreads outlier, so a third sample is required.","next_action_hint":"Collect a third plain-thread sample; prefer it over vmThreads only if stability remains clean and the median stays lower."}} From 5cbc844fdd3e293817cf34346dd61adde78bf40d Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 16:59:31 +0000 Subject: [PATCH 58/80] autoresearch: measure Test CI 20260822T165931Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 09e88990ef65..ae39da9ffccf 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -54,3 +54,4 @@ {"run":53,"commit":"a91676f","metric":764,"metrics":{"packages_coverage_seconds":583,"examples_seconds":113,"startup_seconds":62,"ci_run_id":32577334208},"status":"discard","description":"Use plain isolated worker threads on all CI cores","timestamp":1787408351854,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"Plain threads might match VM-thread throughput while avoiding cross-realm semantics, at the cost of creating a fresh thread worker per isolated file.","result":"The job passed in 764s, effectively tied with the exact vmThreads final run at 770s. Coverage was 583s.","rollback_reason":"One tied sample does not outweigh vmThreads' three-sample stable evidence or demonstrate an improvement.","next_action_hint":"Repeat plain threads once; prefer threads if it remains tied and stability is equal because semantics are closer to the original fork pool."}} {"run":54,"commit":"9b9a36a","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Plain-thread repeat blocked by GitHub API connectivity","timestamp":1787410632292,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"A second plain-thread run would compare stability and median against vmThreads.","result":"The commit pushed, but measure.sh failed before run discovery when api.github.com was unreachable.","error_details":"`error connecting to api.github.com`; no candidate result acquired in this cycle.","rollback_reason":"External measurement-harness failure.","next_action_hint":"Retry unchanged; harden PR lookup API retries if connectivity failures persist."}} {"run":55,"commit":"e82363b","metric":667,"metrics":{"packages_coverage_seconds":477,"examples_seconds":107,"startup_seconds":76,"ci_run_id":32581732238},"status":"keep","description":"Second successful all-core plain-thread run","timestamp":1787413350618,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Plain threads would repeat safely and might avoid VM-context overhead while retaining per-file isolation.","result":"The job passed in 667s with coverage at 477s. Two successful samples are 764s and 667s (median 715.5s), competitive with or better than vmThreads while using conventional thread isolation.","variance_note":"The 477s coverage phase is favorable and close to the best vmThreads outlier, so a third sample is required.","next_action_hint":"Collect a third plain-thread sample; prefer it over vmThreads only if stability remains clean and the median stays lower."}} +{"run":56,"commit":"96ac20f","metric":797,"metrics":{"packages_coverage_seconds":606,"examples_seconds":117,"startup_seconds":68,"ci_run_id":32583320784},"status":"discard","description":"Third all-core plain-thread sample","timestamp":1787417021696,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"A third plain-thread sample would establish a competitive median.","result":"The run passed in 797s with coverage at 606s. Three samples are 667/764/797 (median 764s), versus the initial vmThreads median of 752s and a broader set of clean vm results.","rollback_reason":"Plain threads are ~12s slower by three-sample median and show wider variance, so they do not improve the selected VM pool.","next_action_hint":"Restore vmThreads and retain the exact minimal candidate."}} diff --git a/vitest.config.ts b/vitest.config.ts index dd0834cf5ab3..31159ecc5cdc 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -10,7 +10,7 @@ export default defineConfig({ // Reuse all CI runner cores while keeping a fresh VM context per test file. // Stateful projects can override this default, as the Supabase suite does. maxWorkers: process.env['CI'] ? '100%' : undefined, - pool: process.env['CI'] ? 'threads' : undefined, + pool: process.env['CI'] ? 'vmThreads' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 996061356191f8b68d042e9f9c271cb7a7c66c39 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 18:29:56 +0000 Subject: [PATCH 59/80] autoresearch: measure Test CI 20260822T182955Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index ae39da9ffccf..26048363cfc9 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -55,3 +55,4 @@ {"run":54,"commit":"9b9a36a","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":0},"status":"crash","description":"Plain-thread repeat blocked by GitHub API connectivity","timestamp":1787410632292,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"A second plain-thread run would compare stability and median against vmThreads.","result":"The commit pushed, but measure.sh failed before run discovery when api.github.com was unreachable.","error_details":"`error connecting to api.github.com`; no candidate result acquired in this cycle.","rollback_reason":"External measurement-harness failure.","next_action_hint":"Retry unchanged; harden PR lookup API retries if connectivity failures persist."}} {"run":55,"commit":"e82363b","metric":667,"metrics":{"packages_coverage_seconds":477,"examples_seconds":107,"startup_seconds":76,"ci_run_id":32581732238},"status":"keep","description":"Second successful all-core plain-thread run","timestamp":1787413350618,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Plain threads would repeat safely and might avoid VM-context overhead while retaining per-file isolation.","result":"The job passed in 667s with coverage at 477s. Two successful samples are 764s and 667s (median 715.5s), competitive with or better than vmThreads while using conventional thread isolation.","variance_note":"The 477s coverage phase is favorable and close to the best vmThreads outlier, so a third sample is required.","next_action_hint":"Collect a third plain-thread sample; prefer it over vmThreads only if stability remains clean and the median stays lower."}} {"run":56,"commit":"96ac20f","metric":797,"metrics":{"packages_coverage_seconds":606,"examples_seconds":117,"startup_seconds":68,"ci_run_id":32583320784},"status":"discard","description":"Third all-core plain-thread sample","timestamp":1787417021696,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"A third plain-thread sample would establish a competitive median.","result":"The run passed in 797s with coverage at 606s. Three samples are 667/764/797 (median 764s), versus the initial vmThreads median of 752s and a broader set of clean vm results.","rollback_reason":"Plain threads are ~12s slower by three-sample median and show wider variance, so they do not improve the selected VM pool.","next_action_hint":"Restore vmThreads and retain the exact minimal candidate."}} +{"run":57,"commit":"5cbc844","metric":782,"metrics":{"packages_coverage_seconds":593,"examples_seconds":117,"startup_seconds":66,"ci_run_id":32587178883},"status":"keep","description":"Restore selected all-core VM-thread pool after plain-thread A/B","timestamp":1787420686336,"segment":0,"confidence":10.872727272727273,"asi":{"hypothesis":"Returning to vmThreads would preserve the selected semantic isolation and performance profile.","result":"The exact final candidate passed again in 782s, 65s below baseline. Coverage took 593s. This adds another clean all-core VM run after the teardown fix.","decision":"Keep vmThreads: its controlled three-sample comparison beat plain threads by median, and the final diff remains limited to root pool/concurrency plus graceful test teardown."}} From 8f66a6af067ff12d8d8eef614cfd3f9f42f7d327 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 19:48:17 +0000 Subject: [PATCH 60/80] autoresearch: measure Test CI 20260822T194817Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .auto/prompt.md | 4 ++-- .github/workflows/ci.yml | 1 + scripts/coverage-config.test.mjs | 4 ++++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 26048363cfc9..076bc7e6270e 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -56,3 +56,4 @@ {"run":55,"commit":"e82363b","metric":667,"metrics":{"packages_coverage_seconds":477,"examples_seconds":107,"startup_seconds":76,"ci_run_id":32581732238},"status":"keep","description":"Second successful all-core plain-thread run","timestamp":1787413350618,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Plain threads would repeat safely and might avoid VM-context overhead while retaining per-file isolation.","result":"The job passed in 667s with coverage at 477s. Two successful samples are 764s and 667s (median 715.5s), competitive with or better than vmThreads while using conventional thread isolation.","variance_note":"The 477s coverage phase is favorable and close to the best vmThreads outlier, so a third sample is required.","next_action_hint":"Collect a third plain-thread sample; prefer it over vmThreads only if stability remains clean and the median stays lower."}} {"run":56,"commit":"96ac20f","metric":797,"metrics":{"packages_coverage_seconds":606,"examples_seconds":117,"startup_seconds":68,"ci_run_id":32583320784},"status":"discard","description":"Third all-core plain-thread sample","timestamp":1787417021696,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"A third plain-thread sample would establish a competitive median.","result":"The run passed in 797s with coverage at 606s. Three samples are 667/764/797 (median 764s), versus the initial vmThreads median of 752s and a broader set of clean vm results.","rollback_reason":"Plain threads are ~12s slower by three-sample median and show wider variance, so they do not improve the selected VM pool.","next_action_hint":"Restore vmThreads and retain the exact minimal candidate."}} {"run":57,"commit":"5cbc844","metric":782,"metrics":{"packages_coverage_seconds":593,"examples_seconds":117,"startup_seconds":66,"ci_run_id":32587178883},"status":"keep","description":"Restore selected all-core VM-thread pool after plain-thread A/B","timestamp":1787420686336,"segment":0,"confidence":10.872727272727273,"asi":{"hypothesis":"Returning to vmThreads would preserve the selected semantic isolation and performance profile.","result":"The exact final candidate passed again in 782s, 65s below baseline. Coverage took 593s. This adds another clean all-core VM run after the teardown fix.","decision":"Keep vmThreads: its controlled three-sample comparison beat plain threads by median, and the final diff remains limited to root pool/concurrency plus graceful test teardown."}} +{"run":58,"commit":"9960613","metric":749,"metrics":{"packages_coverage_seconds":565,"examples_seconds":115,"startup_seconds":63,"ci_run_id":32591693433},"status":"keep","description":"Additional exact-final VM-thread stability sample","timestamp":1787426269809,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Another unchanged final-candidate run would strengthen the hosted median and stability evidence.","result":"The job passed in 749s with coverage at 565s. The exact final shape continues to pass and stays materially below the 847s baseline.","distribution":"Excluding deliberately altered pools and unsafe overlap, all-core vmThreads has repeated successful results across favorable and ordinary runners, with no post-fix teardown failures."}} diff --git a/.auto/prompt.md b/.auto/prompt.md index 31f4ffe24212..f2978d60e82e 100644 --- a/.auto/prompt.md +++ b/.auto/prompt.md @@ -48,6 +48,6 @@ Reduce the wall-clock duration of the `Test` job in `.github/workflows/ci.yml` o - Running package coverage and examples concurrently looked fast but was rejected after a real `ECONNRESET`/non-queryable Postgres failure. Do not retry workload overlap. - Full worker reuse (`isolate: false`) passed at 50% and 75%, but it is too broad: Supabase's config explicitly requires per-file isolation to prevent pg mocks leaking into integration files. Consider only targeted reuse in proven stateless projects. - Hosted-runner timing has large outliers. Compare repeated medians and stability rather than trusting minima; a 548s overlap run was followed by 739s and 735s unchanged runs. -- Selected candidate: CI uses `pool: 'vmThreads'` with `maxWorkers: '100%'`. VM contexts preserve file-level isolation while reusable threads avoid fork churn; Supabase retains its explicit `pool: 'forks'` and one-worker override. Three initial all-core VM runs passed at 741s, 761s, and 752s (median 752s), versus a 767s median at 75% and 814s at 50%. +- Selected candidate: CI uses `pool: 'vmThreads'` with `maxWorkers: '100%'`. VM contexts preserve file-level isolation while reusable threads avoid fork churn; Supabase retains its explicit `pool: 'forks'` and one-worker override. Three initial all-core VM runs passed at 741s, 761s, and 752s (median 752s), versus a 767s median at 75% and 814s at 50%. Later exact-final runs passed at 704s, 770s, 782s, and 749s across varying runner load. - The language-server test harness had an existing intermittent teardown rejection across multiple pools. Normal teardown now awaits the standard LSP ShutdownRequest before disposal, while immediate disposal tests keep their old path. Three post-fix all-core jobs passed, followed by a 704s favorable-runner result. -- JSON-only coverage output showed no measurable phase-tail saving, so the text reporter was restored. Deferring Cloudflare Postgres startup did not improve coverage, so the original workflow order was restored. The intended product diff is limited to `vitest.config.ts` and language-server harness tests. +- JSON-only coverage output showed no measurable phase-tail saving, so the text reporter was restored. Deferring Cloudflare Postgres startup did not improve coverage, so the original workflow order was restored. Five workers produced fast points but failed a legitimate 200ms test on the third sample, so do not oversubscribe. Plain threads had a 764s three-sample median versus 752s for VM threads. The intended product diff is limited to `vitest.config.ts` and language-server harness tests. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac0e6125ad0e..0fc85b53eb4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,6 +167,7 @@ jobs: runs-on: ubuntu-latest env: TEST_TIMEOUT_MULTIPLIER: 2 + NODE_COMPILE_CACHE: ${{ runner.temp }}/node-compile-cache # Used by examples/prisma-8-cloudflare-worker's vitest-pool-workers # integration test. Mirrors the .env.example pattern; the container is # brought up by `pnpm db:up` below (docker-compose, not a service diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 138f155d9fff..98be392543cc 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -280,6 +280,10 @@ describe('coverage config', () => { assert.ok(testJob); assert.match(testJob, /^ {4}name: Test$/m); + assert.match( + testJob, + /^ {6}NODE_COMPILE_CACHE: \$\{\{ runner\.temp \}\}\/node-compile-cache$/m, + ); assert.match( testJob, /run: pnpm coverage:packages\n {6}- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, From 34bb5602145ec05f973de0218951fc00c482ef92 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 22:11:34 +0000 Subject: [PATCH 61/80] autoresearch: measure Test CI 20260822T221134Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .github/workflows/ci.yml | 2 +- scripts/coverage-config.test.mjs | 5 +---- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 076bc7e6270e..bbfca149c36a 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -57,3 +57,4 @@ {"run":56,"commit":"96ac20f","metric":797,"metrics":{"packages_coverage_seconds":606,"examples_seconds":117,"startup_seconds":68,"ci_run_id":32583320784},"status":"discard","description":"Third all-core plain-thread sample","timestamp":1787417021696,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"A third plain-thread sample would establish a competitive median.","result":"The run passed in 797s with coverage at 606s. Three samples are 667/764/797 (median 764s), versus the initial vmThreads median of 752s and a broader set of clean vm results.","rollback_reason":"Plain threads are ~12s slower by three-sample median and show wider variance, so they do not improve the selected VM pool.","next_action_hint":"Restore vmThreads and retain the exact minimal candidate."}} {"run":57,"commit":"5cbc844","metric":782,"metrics":{"packages_coverage_seconds":593,"examples_seconds":117,"startup_seconds":66,"ci_run_id":32587178883},"status":"keep","description":"Restore selected all-core VM-thread pool after plain-thread A/B","timestamp":1787420686336,"segment":0,"confidence":10.872727272727273,"asi":{"hypothesis":"Returning to vmThreads would preserve the selected semantic isolation and performance profile.","result":"The exact final candidate passed again in 782s, 65s below baseline. Coverage took 593s. This adds another clean all-core VM run after the teardown fix.","decision":"Keep vmThreads: its controlled three-sample comparison beat plain threads by median, and the final diff remains limited to root pool/concurrency plus graceful test teardown."}} {"run":58,"commit":"9960613","metric":749,"metrics":{"packages_coverage_seconds":565,"examples_seconds":115,"startup_seconds":63,"ci_run_id":32591693433},"status":"keep","description":"Additional exact-final VM-thread stability sample","timestamp":1787426269809,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Another unchanged final-candidate run would strengthen the hosted median and stability evidence.","result":"The job passed in 749s with coverage at 565s. The exact final shape continues to pass and stays materially below the 847s baseline.","distribution":"Excluding deliberately altered pools and unsafe overlap, all-core vmThreads has repeated successful results across favorable and ordinary runners, with no post-fix teardown failures."}} +{"run":59,"commit":"8f66a6a","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":32594808943},"status":"crash","description":"Enable Node compile cache for the Test job","timestamp":1787434379569,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"A job-local Node compile cache could reuse compiled module code across isolated workers and reduce import cost.","result":"The CI workflow was rejected before jobs were created. GitHub recorded a zero-job `CI (PR)` push failure and emitted no pull_request run.","error_details":"The candidate used `${{ runner.temp }}` in job-level env, where the `runner` context is unavailable during workflow validation.","rollback_reason":"Invalid workflow; no benchmark ran.","next_action_hint":"Use a fixed fresh-runner path such as `/tmp/prisma-next-node-compile-cache`, update the workflow contract test, and rerun."}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fc85b53eb4a..3cf28fdccda2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,7 +167,7 @@ jobs: runs-on: ubuntu-latest env: TEST_TIMEOUT_MULTIPLIER: 2 - NODE_COMPILE_CACHE: ${{ runner.temp }}/node-compile-cache + NODE_COMPILE_CACHE: /tmp/prisma-next-node-compile-cache # Used by examples/prisma-8-cloudflare-worker's vitest-pool-workers # integration test. Mirrors the .env.example pattern; the container is # brought up by `pnpm db:up` below (docker-compose, not a service diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 98be392543cc..56a764222313 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -280,10 +280,7 @@ describe('coverage config', () => { assert.ok(testJob); assert.match(testJob, /^ {4}name: Test$/m); - assert.match( - testJob, - /^ {6}NODE_COMPILE_CACHE: \$\{\{ runner\.temp \}\}\/node-compile-cache$/m, - ); + assert.match(testJob, /^ {6}NODE_COMPILE_CACHE: \/tmp\/prisma-next-node-compile-cache$/m); assert.match( testJob, /run: pnpm coverage:packages\n {6}- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, From a17e873b1a15f289a17c6825671bad62d9e406cf Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sat, 22 Aug 2026 23:30:56 +0000 Subject: [PATCH 62/80] autoresearch: measure Test CI 20260822T233055Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index bbfca149c36a..e32eaef16d17 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -58,3 +58,4 @@ {"run":57,"commit":"5cbc844","metric":782,"metrics":{"packages_coverage_seconds":593,"examples_seconds":117,"startup_seconds":66,"ci_run_id":32587178883},"status":"keep","description":"Restore selected all-core VM-thread pool after plain-thread A/B","timestamp":1787420686336,"segment":0,"confidence":10.872727272727273,"asi":{"hypothesis":"Returning to vmThreads would preserve the selected semantic isolation and performance profile.","result":"The exact final candidate passed again in 782s, 65s below baseline. Coverage took 593s. This adds another clean all-core VM run after the teardown fix.","decision":"Keep vmThreads: its controlled three-sample comparison beat plain threads by median, and the final diff remains limited to root pool/concurrency plus graceful test teardown."}} {"run":58,"commit":"9960613","metric":749,"metrics":{"packages_coverage_seconds":565,"examples_seconds":115,"startup_seconds":63,"ci_run_id":32591693433},"status":"keep","description":"Additional exact-final VM-thread stability sample","timestamp":1787426269809,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Another unchanged final-candidate run would strengthen the hosted median and stability evidence.","result":"The job passed in 749s with coverage at 565s. The exact final shape continues to pass and stays materially below the 847s baseline.","distribution":"Excluding deliberately altered pools and unsafe overlap, all-core vmThreads has repeated successful results across favorable and ordinary runners, with no post-fix teardown failures."}} {"run":59,"commit":"8f66a6a","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":32594808943},"status":"crash","description":"Enable Node compile cache for the Test job","timestamp":1787434379569,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"A job-local Node compile cache could reuse compiled module code across isolated workers and reduce import cost.","result":"The CI workflow was rejected before jobs were created. GitHub recorded a zero-job `CI (PR)` push failure and emitted no pull_request run.","error_details":"The candidate used `${{ runner.temp }}` in job-level env, where the `runner` context is unavailable during workflow validation.","rollback_reason":"Invalid workflow; no benchmark ran.","next_action_hint":"Use a fixed fresh-runner path such as `/tmp/prisma-next-node-compile-cache`, update the workflow contract test, and rerun."}} +{"run":60,"commit":"34bb560","metric":630,"metrics":{"packages_coverage_seconds":457,"examples_seconds":111,"startup_seconds":57,"ci_run_id":32601770137},"status":"keep","description":"Enable a job-local Node module compile cache","timestamp":1787440523970,"segment":0,"confidence":10.491228070175438,"asi":{"hypothesis":"NODE_COMPILE_CACHE can reuse V8-compiled module code across the many isolated VM contexts and subprocesses in the Test job.","result":"The full job passed in 630s; coverage took 457s and examples 111s. No clean-tree artifact exists because the cache is under the runner's fresh `/tmp`.","variance_note":"The phase is faster than typical but overlaps prior favorable-runner outliers, so one sample cannot establish cache benefit.","next_action_hint":"Repeat twice with the fixed cache path; compare a three-sample median to the no-cache VM distribution."}} From 76add2893af1acb2f5fe81ea90f08e36305d1787 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sun, 23 Aug 2026 00:02:36 +0000 Subject: [PATCH 63/80] autoresearch: measure Test CI 20260823T000236Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index e32eaef16d17..e371b192a04a 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -59,3 +59,4 @@ {"run":58,"commit":"9960613","metric":749,"metrics":{"packages_coverage_seconds":565,"examples_seconds":115,"startup_seconds":63,"ci_run_id":32591693433},"status":"keep","description":"Additional exact-final VM-thread stability sample","timestamp":1787426269809,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Another unchanged final-candidate run would strengthen the hosted median and stability evidence.","result":"The job passed in 749s with coverage at 565s. The exact final shape continues to pass and stays materially below the 847s baseline.","distribution":"Excluding deliberately altered pools and unsafe overlap, all-core vmThreads has repeated successful results across favorable and ordinary runners, with no post-fix teardown failures."}} {"run":59,"commit":"8f66a6a","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":32594808943},"status":"crash","description":"Enable Node compile cache for the Test job","timestamp":1787434379569,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"A job-local Node compile cache could reuse compiled module code across isolated workers and reduce import cost.","result":"The CI workflow was rejected before jobs were created. GitHub recorded a zero-job `CI (PR)` push failure and emitted no pull_request run.","error_details":"The candidate used `${{ runner.temp }}` in job-level env, where the `runner` context is unavailable during workflow validation.","rollback_reason":"Invalid workflow; no benchmark ran.","next_action_hint":"Use a fixed fresh-runner path such as `/tmp/prisma-next-node-compile-cache`, update the workflow contract test, and rerun."}} {"run":60,"commit":"34bb560","metric":630,"metrics":{"packages_coverage_seconds":457,"examples_seconds":111,"startup_seconds":57,"ci_run_id":32601770137},"status":"keep","description":"Enable a job-local Node module compile cache","timestamp":1787440523970,"segment":0,"confidence":10.491228070175438,"asi":{"hypothesis":"NODE_COMPILE_CACHE can reuse V8-compiled module code across the many isolated VM contexts and subprocesses in the Test job.","result":"The full job passed in 630s; coverage took 457s and examples 111s. No clean-tree artifact exists because the cache is under the runner's fresh `/tmp`.","variance_note":"The phase is faster than typical but overlaps prior favorable-runner outliers, so one sample cannot establish cache benefit.","next_action_hint":"Repeat twice with the fixed cache path; compare a three-sample median to the no-cache VM distribution."}} +{"run":61,"commit":"a17e873","metric":774,"metrics":{"packages_coverage_seconds":527,"examples_seconds":162,"startup_seconds":79,"ci_run_id":32605440291},"status":"discard","description":"Second Node compile-cache sample","timestamp":1787442412690,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"The compile cache would repeat its package-phase improvement.","result":"The job passed in 774s. Coverage was 527s—70s slower than the first cache sample but still below the no-cache VM median—while startup and examples were unrelated slow outliers.","rollback_reason":"The complete-job primary metric regressed from 630s and did not improve the retained exact-final runs.","next_action_hint":"Collect a third cache sample; judge primarily by coverage-phase median while respecting the complete job as the keep/discard metric."}} From a550c0c5f6c4491bf3a10550a30cc84768b479e4 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sun, 23 Aug 2026 00:34:18 +0000 Subject: [PATCH 64/80] autoresearch: measure Test CI 20260823T003418Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .auto/prompt.md | 4 +++- vitest.config.ts | 7 +++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index e371b192a04a..9ea7ebb9dc60 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -60,3 +60,4 @@ {"run":59,"commit":"8f66a6a","metric":0,"metrics":{"packages_coverage_seconds":0,"examples_seconds":0,"startup_seconds":0,"ci_run_id":32594808943},"status":"crash","description":"Enable Node compile cache for the Test job","timestamp":1787434379569,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"A job-local Node compile cache could reuse compiled module code across isolated workers and reduce import cost.","result":"The CI workflow was rejected before jobs were created. GitHub recorded a zero-job `CI (PR)` push failure and emitted no pull_request run.","error_details":"The candidate used `${{ runner.temp }}` in job-level env, where the `runner` context is unavailable during workflow validation.","rollback_reason":"Invalid workflow; no benchmark ran.","next_action_hint":"Use a fixed fresh-runner path such as `/tmp/prisma-next-node-compile-cache`, update the workflow contract test, and rerun."}} {"run":60,"commit":"34bb560","metric":630,"metrics":{"packages_coverage_seconds":457,"examples_seconds":111,"startup_seconds":57,"ci_run_id":32601770137},"status":"keep","description":"Enable a job-local Node module compile cache","timestamp":1787440523970,"segment":0,"confidence":10.491228070175438,"asi":{"hypothesis":"NODE_COMPILE_CACHE can reuse V8-compiled module code across the many isolated VM contexts and subprocesses in the Test job.","result":"The full job passed in 630s; coverage took 457s and examples 111s. No clean-tree artifact exists because the cache is under the runner's fresh `/tmp`.","variance_note":"The phase is faster than typical but overlaps prior favorable-runner outliers, so one sample cannot establish cache benefit.","next_action_hint":"Repeat twice with the fixed cache path; compare a three-sample median to the no-cache VM distribution."}} {"run":61,"commit":"a17e873","metric":774,"metrics":{"packages_coverage_seconds":527,"examples_seconds":162,"startup_seconds":79,"ci_run_id":32605440291},"status":"discard","description":"Second Node compile-cache sample","timestamp":1787442412690,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"The compile cache would repeat its package-phase improvement.","result":"The job passed in 774s. Coverage was 527s—70s slower than the first cache sample but still below the no-cache VM median—while startup and examples were unrelated slow outliers.","rollback_reason":"The complete-job primary metric regressed from 630s and did not improve the retained exact-final runs.","next_action_hint":"Collect a third cache sample; judge primarily by coverage-phase median while respecting the complete job as the keep/discard metric."}} +{"run":62,"commit":"76add28","metric":734,"metrics":{"packages_coverage_seconds":553,"examples_seconds":113,"startup_seconds":62,"ci_run_id":32606843761},"status":"keep","description":"Third Node compile-cache stability sample","timestamp":1787444297420,"segment":0,"confidence":10.491228070175438,"asi":{"hypothesis":"A third sample would establish whether the job-local compile cache has a repeatable benefit.","result":"The third job passed in 734s with coverage at 553s. Cache samples are 630/774/734 (median 734s); coverage phases are 457/527/553 (median 527s).","comparison":"The broad no-cache all-core VM distribution centers around ~765s complete and ~580s coverage, so cache medians indicate roughly 30s complete-job and 50s package-phase improvement despite runner noise.","correctness":"The cache is fresh per GitHub-hosted job, source-keyed by Node, outside the checkout, and all tests/coverage/clean-tree checks passed three times.","decision":"Retain NODE_COMPILE_CACHE alongside vmThreads."}} diff --git a/.auto/prompt.md b/.auto/prompt.md index f2978d60e82e..039685dd3042 100644 --- a/.auto/prompt.md +++ b/.auto/prompt.md @@ -50,4 +50,6 @@ Reduce the wall-clock duration of the `Test` job in `.github/workflows/ci.yml` o - Hosted-runner timing has large outliers. Compare repeated medians and stability rather than trusting minima; a 548s overlap run was followed by 739s and 735s unchanged runs. - Selected candidate: CI uses `pool: 'vmThreads'` with `maxWorkers: '100%'`. VM contexts preserve file-level isolation while reusable threads avoid fork churn; Supabase retains its explicit `pool: 'forks'` and one-worker override. Three initial all-core VM runs passed at 741s, 761s, and 752s (median 752s), versus a 767s median at 75% and 814s at 50%. Later exact-final runs passed at 704s, 770s, 782s, and 749s across varying runner load. - The language-server test harness had an existing intermittent teardown rejection across multiple pools. Normal teardown now awaits the standard LSP ShutdownRequest before disposal, while immediate disposal tests keep their old path. Three post-fix all-core jobs passed, followed by a 704s favorable-runner result. -- JSON-only coverage output showed no measurable phase-tail saving, so the text reporter was restored. Deferring Cloudflare Postgres startup did not improve coverage, so the original workflow order was restored. Five workers produced fast points but failed a legitimate 200ms test on the third sample, so do not oversubscribe. Plain threads had a 764s three-sample median versus 752s for VM threads. The intended product diff is limited to `vitest.config.ts` and language-server harness tests. +- JSON-only coverage output showed no measurable phase-tail saving, so the text reporter was restored. Deferring Cloudflare Postgres startup did not improve coverage, so the original workflow order was restored. Five workers produced fast points but failed a legitimate 200ms test on the third sample, so do not oversubscribe. Plain threads had a 764s three-sample median versus 752s for VM threads. +- Retain a job-local `NODE_COMPILE_CACHE` under `/tmp`: three runs passed at 630s, 774s, and 734s (median 734s), with package coverage phases of 457s, 527s, and 553s (median 527s). The broad no-cache all-core VM distribution centers around ~765s complete and ~580s package coverage. The cache is fresh per hosted job and outside the checkout. +- The intended product diff is limited to root Vitest pool/concurrency, Test-job compile-cache configuration plus its workflow contract test, and language-server harness teardown tests. diff --git a/vitest.config.ts b/vitest.config.ts index 31159ecc5cdc..1d443939fc77 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -7,10 +7,9 @@ const coveragePolicy = composeCoverageConfig(import.meta.dirname); export default defineConfig({ test: { projects: ['packages/**/vitest.config.ts'], - // Reuse all CI runner cores while keeping a fresh VM context per test file. - // Stateful projects can override this default, as the Supabase suite does. - maxWorkers: process.env['CI'] ? '100%' : undefined, - pool: process.env['CI'] ? 'vmThreads' : undefined, + // Keep CI concurrency below the runner's core count so database-backed + // package tests retain CPU headroom for Postgres and PGlite. + maxWorkers: process.env['CI'] ? '50%' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From a221bcd93c1d68a9b0d477b6b66fd3801e8dd462 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sun, 23 Aug 2026 01:26:55 +0000 Subject: [PATCH 65/80] autoresearch: measure Test CI 20260823T012654Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + vitest.config.ts | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 9ea7ebb9dc60..d7a4b6e41df7 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -61,3 +61,4 @@ {"run":60,"commit":"34bb560","metric":630,"metrics":{"packages_coverage_seconds":457,"examples_seconds":111,"startup_seconds":57,"ci_run_id":32601770137},"status":"keep","description":"Enable a job-local Node module compile cache","timestamp":1787440523970,"segment":0,"confidence":10.491228070175438,"asi":{"hypothesis":"NODE_COMPILE_CACHE can reuse V8-compiled module code across the many isolated VM contexts and subprocesses in the Test job.","result":"The full job passed in 630s; coverage took 457s and examples 111s. No clean-tree artifact exists because the cache is under the runner's fresh `/tmp`.","variance_note":"The phase is faster than typical but overlaps prior favorable-runner outliers, so one sample cannot establish cache benefit.","next_action_hint":"Repeat twice with the fixed cache path; compare a three-sample median to the no-cache VM distribution."}} {"run":61,"commit":"a17e873","metric":774,"metrics":{"packages_coverage_seconds":527,"examples_seconds":162,"startup_seconds":79,"ci_run_id":32605440291},"status":"discard","description":"Second Node compile-cache sample","timestamp":1787442412690,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"The compile cache would repeat its package-phase improvement.","result":"The job passed in 774s. Coverage was 527s—70s slower than the first cache sample but still below the no-cache VM median—while startup and examples were unrelated slow outliers.","rollback_reason":"The complete-job primary metric regressed from 630s and did not improve the retained exact-final runs.","next_action_hint":"Collect a third cache sample; judge primarily by coverage-phase median while respecting the complete job as the keep/discard metric."}} {"run":62,"commit":"76add28","metric":734,"metrics":{"packages_coverage_seconds":553,"examples_seconds":113,"startup_seconds":62,"ci_run_id":32606843761},"status":"keep","description":"Third Node compile-cache stability sample","timestamp":1787444297420,"segment":0,"confidence":10.491228070175438,"asi":{"hypothesis":"A third sample would establish whether the job-local compile cache has a repeatable benefit.","result":"The third job passed in 734s with coverage at 553s. Cache samples are 630/774/734 (median 734s); coverage phases are 457/527/553 (median 527s).","comparison":"The broad no-cache all-core VM distribution centers around ~765s complete and ~580s coverage, so cache medians indicate roughly 30s complete-job and 50s package-phase improvement despite runner noise.","correctness":"The cache is fresh per GitHub-hosted job, source-keyed by Node, outside the checkout, and all tests/coverage/clean-tree checks passed three times.","decision":"Retain NODE_COMPILE_CACHE alongside vmThreads."}} +{"run":63,"commit":"a550c0c","metric":873,"metrics":{"packages_coverage_seconds":690,"examples_seconds":118,"startup_seconds":60,"ci_run_id":32608868876},"status":"discard","description":"Use Node compile cache with original fork pool and 50% workers","timestamp":1787447486949,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"The compile cache alone might deliver the import savings without changing the original fork pool.","result":"The job passed but took 873s, with package coverage at 690s—worse than the 847s baseline and far behind compile-cache vmThreads.","rollback_reason":"Primary metric regressed 139s from the cache+VM median; compile caching does not replace worker reuse/all-core concurrency.","next_action_hint":"Restore all-core vmThreads with compile cache. No need to repeat the clearly dominated original pool."}} diff --git a/vitest.config.ts b/vitest.config.ts index 1d443939fc77..31159ecc5cdc 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -7,9 +7,10 @@ const coveragePolicy = composeCoverageConfig(import.meta.dirname); export default defineConfig({ test: { projects: ['packages/**/vitest.config.ts'], - // Keep CI concurrency below the runner's core count so database-backed - // package tests retain CPU headroom for Postgres and PGlite. - maxWorkers: process.env['CI'] ? '50%' : undefined, + // Reuse all CI runner cores while keeping a fresh VM context per test file. + // Stateful projects can override this default, as the Supabase suite does. + maxWorkers: process.env['CI'] ? '100%' : undefined, + pool: process.env['CI'] ? 'vmThreads' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From e542e12af7dc866dbc7c2601ab3141aa9f62653c Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sun, 23 Aug 2026 03:49:28 +0000 Subject: [PATCH 66/80] autoresearch: measure Test CI 20260823T034926Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .github/workflows/ci.yml | 2 +- scripts/coverage-config.test.mjs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index d7a4b6e41df7..5378dceed38a 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -62,3 +62,4 @@ {"run":61,"commit":"a17e873","metric":774,"metrics":{"packages_coverage_seconds":527,"examples_seconds":162,"startup_seconds":79,"ci_run_id":32605440291},"status":"discard","description":"Second Node compile-cache sample","timestamp":1787442412690,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"The compile cache would repeat its package-phase improvement.","result":"The job passed in 774s. Coverage was 527s—70s slower than the first cache sample but still below the no-cache VM median—while startup and examples were unrelated slow outliers.","rollback_reason":"The complete-job primary metric regressed from 630s and did not improve the retained exact-final runs.","next_action_hint":"Collect a third cache sample; judge primarily by coverage-phase median while respecting the complete job as the keep/discard metric."}} {"run":62,"commit":"76add28","metric":734,"metrics":{"packages_coverage_seconds":553,"examples_seconds":113,"startup_seconds":62,"ci_run_id":32606843761},"status":"keep","description":"Third Node compile-cache stability sample","timestamp":1787444297420,"segment":0,"confidence":10.491228070175438,"asi":{"hypothesis":"A third sample would establish whether the job-local compile cache has a repeatable benefit.","result":"The third job passed in 734s with coverage at 553s. Cache samples are 630/774/734 (median 734s); coverage phases are 457/527/553 (median 527s).","comparison":"The broad no-cache all-core VM distribution centers around ~765s complete and ~580s coverage, so cache medians indicate roughly 30s complete-job and 50s package-phase improvement despite runner noise.","correctness":"The cache is fresh per GitHub-hosted job, source-keyed by Node, outside the checkout, and all tests/coverage/clean-tree checks passed three times.","decision":"Retain NODE_COMPILE_CACHE alongside vmThreads."}} {"run":63,"commit":"a550c0c","metric":873,"metrics":{"packages_coverage_seconds":690,"examples_seconds":118,"startup_seconds":60,"ci_run_id":32608868876},"status":"discard","description":"Use Node compile cache with original fork pool and 50% workers","timestamp":1787447486949,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"The compile cache alone might deliver the import savings without changing the original fork pool.","result":"The job passed but took 873s, with package coverage at 690s—worse than the 847s baseline and far behind compile-cache vmThreads.","rollback_reason":"Primary metric regressed 139s from the cache+VM median; compile caching does not replace worker reuse/all-core concurrency.","next_action_hint":"Restore all-core vmThreads with compile cache. No need to repeat the clearly dominated original pool."}} +{"run":64,"commit":"a221bcd","metric":746,"metrics":{"packages_coverage_seconds":565,"examples_seconds":112,"startup_seconds":63,"ci_run_id":32610389255},"status":"keep","description":"Restore final compile-cache VM-thread configuration","timestamp":1787450327844,"segment":0,"confidence":10.491228070175438,"asi":{"hypothesis":"The selected combined optimization would return to its stable below-baseline range after the dominated fork control.","result":"The exact final candidate passed in 746s, 101s (11.9%) below baseline. This fourth compile-cache VM sample yields a 740s median across 630/734/746/774.","decision":"Retain all-core vmThreads, job-local Node compile cache, and graceful language-server teardown."}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cf28fdccda2..98afd83b4a98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,7 +167,7 @@ jobs: runs-on: ubuntu-latest env: TEST_TIMEOUT_MULTIPLIER: 2 - NODE_COMPILE_CACHE: /tmp/prisma-next-node-compile-cache + NODE_COMPILE_CACHE: /tmp/prisma-test-node-compile-cache # Used by examples/prisma-8-cloudflare-worker's vitest-pool-workers # integration test. Mirrors the .env.example pattern; the container is # brought up by `pnpm db:up` below (docker-compose, not a service diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 56a764222313..d2ccde5de37e 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -280,7 +280,7 @@ describe('coverage config', () => { assert.ok(testJob); assert.match(testJob, /^ {4}name: Test$/m); - assert.match(testJob, /^ {6}NODE_COMPILE_CACHE: \/tmp\/prisma-next-node-compile-cache$/m); + assert.match(testJob, /^ {6}NODE_COMPILE_CACHE: \/tmp\/prisma-test-node-compile-cache$/m); assert.match( testJob, /run: pnpm coverage:packages\n {6}- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, From bfa13212270204233c6789d59edf540fe1425cba Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sun, 23 Aug 2026 08:20:27 +0000 Subject: [PATCH 67/80] autoresearch: measure Test CI 20260823T082027Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + packages/3-extensions/sql-orm-client/vitest.config.ts | 1 + scripts/coverage-config.test.mjs | 10 ++++++++++ 3 files changed, 12 insertions(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 5378dceed38a..62f8b564cb87 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -63,3 +63,4 @@ {"run":62,"commit":"76add28","metric":734,"metrics":{"packages_coverage_seconds":553,"examples_seconds":113,"startup_seconds":62,"ci_run_id":32606843761},"status":"keep","description":"Third Node compile-cache stability sample","timestamp":1787444297420,"segment":0,"confidence":10.491228070175438,"asi":{"hypothesis":"A third sample would establish whether the job-local compile cache has a repeatable benefit.","result":"The third job passed in 734s with coverage at 553s. Cache samples are 630/774/734 (median 734s); coverage phases are 457/527/553 (median 527s).","comparison":"The broad no-cache all-core VM distribution centers around ~765s complete and ~580s coverage, so cache medians indicate roughly 30s complete-job and 50s package-phase improvement despite runner noise.","correctness":"The cache is fresh per GitHub-hosted job, source-keyed by Node, outside the checkout, and all tests/coverage/clean-tree checks passed three times.","decision":"Retain NODE_COMPILE_CACHE alongside vmThreads."}} {"run":63,"commit":"a550c0c","metric":873,"metrics":{"packages_coverage_seconds":690,"examples_seconds":118,"startup_seconds":60,"ci_run_id":32608868876},"status":"discard","description":"Use Node compile cache with original fork pool and 50% workers","timestamp":1787447486949,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"The compile cache alone might deliver the import savings without changing the original fork pool.","result":"The job passed but took 873s, with package coverage at 690s—worse than the 847s baseline and far behind compile-cache vmThreads.","rollback_reason":"Primary metric regressed 139s from the cache+VM median; compile caching does not replace worker reuse/all-core concurrency.","next_action_hint":"Restore all-core vmThreads with compile cache. No need to repeat the clearly dominated original pool."}} {"run":64,"commit":"a221bcd","metric":746,"metrics":{"packages_coverage_seconds":565,"examples_seconds":112,"startup_seconds":63,"ci_run_id":32610389255},"status":"keep","description":"Restore final compile-cache VM-thread configuration","timestamp":1787450327844,"segment":0,"confidence":10.491228070175438,"asi":{"hypothesis":"The selected combined optimization would return to its stable below-baseline range after the dominated fork control.","result":"The exact final candidate passed in 746s, 101s (11.9%) below baseline. This fourth compile-cache VM sample yields a 740s median across 630/734/746/774.","decision":"Retain all-core vmThreads, job-local Node compile cache, and graceful language-server teardown."}} +{"run":65,"commit":"e542e12","metric":736,"metrics":{"packages_coverage_seconds":560,"examples_seconds":111,"startup_seconds":60,"ci_run_id":32616764867},"status":"keep","description":"Use a legacy-name-compliant compile-cache path","timestamp":1787459929586,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Renaming the `/tmp` cache directory should preserve performance while satisfying the repository's legacy-name lint.","result":"The Test job passed in 736s with coverage at 560s, effectively matching the 734s cache median sample.","correctness":"Product files no longer introduce the retired name; the remaining lint occurrence is only in the temporary append-only autoresearch log, which is excluded from the final branch."}} diff --git a/packages/3-extensions/sql-orm-client/vitest.config.ts b/packages/3-extensions/sql-orm-client/vitest.config.ts index ad2931656283..4ab307b492e7 100644 --- a/packages/3-extensions/sql-orm-client/vitest.config.ts +++ b/packages/3-extensions/sql-orm-client/vitest.config.ts @@ -5,6 +5,7 @@ export default defineConfig({ test: { globals: true, environment: 'node', + ...(process.env['CI'] ? { isolate: false } : {}), testTimeout: timeouts.typeScriptCompilation, hookTimeout: timeouts.default, typecheck: { diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index d2ccde5de37e..bfcb3572d4cb 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -272,6 +272,16 @@ describe('coverage config', () => { assert.match(rootVitestConfig, /reportOnFailure:\s*true/); }); + it('reuses the stateless SQL ORM client worker on CI', async () => { + const repositoryRoot = join(import.meta.dirname, '..'); + const config = await readFile( + join(repositoryRoot, 'packages/3-extensions/sql-orm-client/vitest.config.ts'), + 'utf8', + ); + + assert.match(config, /\.\.\.\(process\.env\['CI'\] \? \{ isolate: false \} : \{\}\)/); + }); + it('combines package tests and coverage in one CI job', async () => { const repositoryRoot = join(import.meta.dirname, '..'); const workflow = await readFile(join(repositoryRoot, '.github/workflows/ci.yml'), 'utf8'); From 1812f1518d94f445cc2c924297d5b1cb86cf9ae8 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sun, 23 Aug 2026 09:24:43 +0000 Subject: [PATCH 68/80] autoresearch: measure Test CI 20260823T092443Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + packages/3-extensions/sql-orm-client/vitest.config.ts | 2 +- scripts/coverage-config.test.mjs | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 62f8b564cb87..70bf1a73b1b2 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -64,3 +64,4 @@ {"run":63,"commit":"a550c0c","metric":873,"metrics":{"packages_coverage_seconds":690,"examples_seconds":118,"startup_seconds":60,"ci_run_id":32608868876},"status":"discard","description":"Use Node compile cache with original fork pool and 50% workers","timestamp":1787447486949,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"The compile cache alone might deliver the import savings without changing the original fork pool.","result":"The job passed but took 873s, with package coverage at 690s—worse than the 847s baseline and far behind compile-cache vmThreads.","rollback_reason":"Primary metric regressed 139s from the cache+VM median; compile caching does not replace worker reuse/all-core concurrency.","next_action_hint":"Restore all-core vmThreads with compile cache. No need to repeat the clearly dominated original pool."}} {"run":64,"commit":"a221bcd","metric":746,"metrics":{"packages_coverage_seconds":565,"examples_seconds":112,"startup_seconds":63,"ci_run_id":32610389255},"status":"keep","description":"Restore final compile-cache VM-thread configuration","timestamp":1787450327844,"segment":0,"confidence":10.491228070175438,"asi":{"hypothesis":"The selected combined optimization would return to its stable below-baseline range after the dominated fork control.","result":"The exact final candidate passed in 746s, 101s (11.9%) below baseline. This fourth compile-cache VM sample yields a 740s median across 630/734/746/774.","decision":"Retain all-core vmThreads, job-local Node compile cache, and graceful language-server teardown."}} {"run":65,"commit":"e542e12","metric":736,"metrics":{"packages_coverage_seconds":560,"examples_seconds":111,"startup_seconds":60,"ci_run_id":32616764867},"status":"keep","description":"Use a legacy-name-compliant compile-cache path","timestamp":1787459929586,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Renaming the `/tmp` cache directory should preserve performance while satisfying the repository's legacy-name lint.","result":"The Test job passed in 736s with coverage at 560s, effectively matching the 734s cache median sample.","correctness":"Product files no longer introduce the retired name; the remaining lint occurrence is only in the temporary append-only autoresearch log, which is excluded from the final branch."}} +{"run":66,"commit":"bfa1321","metric":0,"metrics":{"packages_coverage_seconds":563,"examples_seconds":0,"startup_seconds":64,"ci_run_id":32628030774},"status":"checks_failed","description":"Reuse SQL ORM client workers only on CI","timestamp":1787475634040,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Disabling file isolation only for the 69-file stateless SQL ORM client project would remove VM-context churn without risking cross-file state leakage.","result":"All SQL ORM client tests passed, and the package phase reached its summary after 563s, but a repository guard rejected the source-level `process.env.CI` branch in the package config.","error_details":"`@internal/cli test/utils/no-parallel-ci-detection.test.ts` found `packages/3-extensions/sql-orm-client/vitest.config.ts: ...(process.env['CI'] ? { isolate: false } : {})`. The guard requires package behavior not to branch directly on CI.","rollback_reason":"The candidate violates the repository's no-target-branches rule, so no timing result is valid.","next_action_hint":"Apply `isolate: false` unconditionally in this demonstrably stateless project and rerun; this matches the existing CLI project pattern and avoids a target-specific branch."}} diff --git a/packages/3-extensions/sql-orm-client/vitest.config.ts b/packages/3-extensions/sql-orm-client/vitest.config.ts index 4ab307b492e7..3b91e3863602 100644 --- a/packages/3-extensions/sql-orm-client/vitest.config.ts +++ b/packages/3-extensions/sql-orm-client/vitest.config.ts @@ -5,7 +5,7 @@ export default defineConfig({ test: { globals: true, environment: 'node', - ...(process.env['CI'] ? { isolate: false } : {}), + isolate: false, testTimeout: timeouts.typeScriptCompilation, hookTimeout: timeouts.default, typecheck: { diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index bfcb3572d4cb..8e1c4ef290a3 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -272,14 +272,14 @@ describe('coverage config', () => { assert.match(rootVitestConfig, /reportOnFailure:\s*true/); }); - it('reuses the stateless SQL ORM client worker on CI', async () => { + it('reuses workers for the stateless SQL ORM client', async () => { const repositoryRoot = join(import.meta.dirname, '..'); const config = await readFile( join(repositoryRoot, 'packages/3-extensions/sql-orm-client/vitest.config.ts'), 'utf8', ); - assert.match(config, /\.\.\.\(process\.env\['CI'\] \? \{ isolate: false \} : \{\}\)/); + assert.match(config, /^ {4}isolate: false,$/m); }); it('combines package tests and coverage in one CI job', async () => { From 4ef026aba2f1afa5073b37c8037b2b911c75d2de Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sun, 23 Aug 2026 14:38:29 +0000 Subject: [PATCH 69/80] autoresearch: measure Test CI 20260823T143829Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .github/workflows/ci.yml | 2 +- .../3-extensions/sql-orm-client/vitest.config.ts | 1 - scripts/coverage-config.test.mjs | 12 +----------- 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 70bf1a73b1b2..aa4cf68ee06b 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -65,3 +65,4 @@ {"run":64,"commit":"a221bcd","metric":746,"metrics":{"packages_coverage_seconds":565,"examples_seconds":112,"startup_seconds":63,"ci_run_id":32610389255},"status":"keep","description":"Restore final compile-cache VM-thread configuration","timestamp":1787450327844,"segment":0,"confidence":10.491228070175438,"asi":{"hypothesis":"The selected combined optimization would return to its stable below-baseline range after the dominated fork control.","result":"The exact final candidate passed in 746s, 101s (11.9%) below baseline. This fourth compile-cache VM sample yields a 740s median across 630/734/746/774.","decision":"Retain all-core vmThreads, job-local Node compile cache, and graceful language-server teardown."}} {"run":65,"commit":"e542e12","metric":736,"metrics":{"packages_coverage_seconds":560,"examples_seconds":111,"startup_seconds":60,"ci_run_id":32616764867},"status":"keep","description":"Use a legacy-name-compliant compile-cache path","timestamp":1787459929586,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Renaming the `/tmp` cache directory should preserve performance while satisfying the repository's legacy-name lint.","result":"The Test job passed in 736s with coverage at 560s, effectively matching the 734s cache median sample.","correctness":"Product files no longer introduce the retired name; the remaining lint occurrence is only in the temporary append-only autoresearch log, which is excluded from the final branch."}} {"run":66,"commit":"bfa1321","metric":0,"metrics":{"packages_coverage_seconds":563,"examples_seconds":0,"startup_seconds":64,"ci_run_id":32628030774},"status":"checks_failed","description":"Reuse SQL ORM client workers only on CI","timestamp":1787475634040,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Disabling file isolation only for the 69-file stateless SQL ORM client project would remove VM-context churn without risking cross-file state leakage.","result":"All SQL ORM client tests passed, and the package phase reached its summary after 563s, but a repository guard rejected the source-level `process.env.CI` branch in the package config.","error_details":"`@internal/cli test/utils/no-parallel-ci-detection.test.ts` found `packages/3-extensions/sql-orm-client/vitest.config.ts: ...(process.env['CI'] ? { isolate: false } : {})`. The guard requires package behavior not to branch directly on CI.","rollback_reason":"The candidate violates the repository's no-target-branches rule, so no timing result is valid.","next_action_hint":"Apply `isolate: false` unconditionally in this demonstrably stateless project and rerun; this matches the existing CLI project pattern and avoids a target-specific branch."}} +{"run":67,"commit":"1812f15","metric":766,"metrics":{"packages_coverage_seconds":574,"examples_seconds":115,"startup_seconds":69,"ci_run_id":32631001906},"status":"discard","description":"Reuse workers only for the stateless SQL ORM client project","timestamp":1787478133948,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Setting `isolate: false` only for the 69-file stateless SQL ORM client project would reduce VM-context setup while preserving isolation for stateful projects.","result":"The full hosted Test job passed in 766s with package coverage at 574s. Vitest reported 952 isolated worker contexts, down from roughly 998 in earlier fully isolated runs, but the complete job and coverage phase were slower than the selected cache+vmThreads median (~736s complete, ~553s coverage).","rollback_reason":"The targeted reuse added package-specific semantic/configuration complexity without improving the primary metric; the saved contexts are too small a fraction of the dominant workload to rise above runner variance.","next_action_hint":"Restore isolation for SQL ORM client. If revisiting targeted reuse, batch several independently proven stateless projects in one experiment so the structural reduction is large enough to measure."}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98afd83b4a98..a64c6655ff3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,7 +167,7 @@ jobs: runs-on: ubuntu-latest env: TEST_TIMEOUT_MULTIPLIER: 2 - NODE_COMPILE_CACHE: /tmp/prisma-test-node-compile-cache + NODE_COMPILE_CACHE: /dev/shm/prisma-test-node-compile-cache # Used by examples/prisma-8-cloudflare-worker's vitest-pool-workers # integration test. Mirrors the .env.example pattern; the container is # brought up by `pnpm db:up` below (docker-compose, not a service diff --git a/packages/3-extensions/sql-orm-client/vitest.config.ts b/packages/3-extensions/sql-orm-client/vitest.config.ts index 3b91e3863602..ad2931656283 100644 --- a/packages/3-extensions/sql-orm-client/vitest.config.ts +++ b/packages/3-extensions/sql-orm-client/vitest.config.ts @@ -5,7 +5,6 @@ export default defineConfig({ test: { globals: true, environment: 'node', - isolate: false, testTimeout: timeouts.typeScriptCompilation, hookTimeout: timeouts.default, typecheck: { diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 8e1c4ef290a3..bc1c82591e23 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -272,16 +272,6 @@ describe('coverage config', () => { assert.match(rootVitestConfig, /reportOnFailure:\s*true/); }); - it('reuses workers for the stateless SQL ORM client', async () => { - const repositoryRoot = join(import.meta.dirname, '..'); - const config = await readFile( - join(repositoryRoot, 'packages/3-extensions/sql-orm-client/vitest.config.ts'), - 'utf8', - ); - - assert.match(config, /^ {4}isolate: false,$/m); - }); - it('combines package tests and coverage in one CI job', async () => { const repositoryRoot = join(import.meta.dirname, '..'); const workflow = await readFile(join(repositoryRoot, '.github/workflows/ci.yml'), 'utf8'); @@ -290,7 +280,7 @@ describe('coverage config', () => { assert.ok(testJob); assert.match(testJob, /^ {4}name: Test$/m); - assert.match(testJob, /^ {6}NODE_COMPILE_CACHE: \/tmp\/prisma-test-node-compile-cache$/m); + assert.match(testJob, /^ {6}NODE_COMPILE_CACHE: \/dev\/shm\/prisma-test-node-compile-cache$/m); assert.match( testJob, /run: pnpm coverage:packages\n {6}- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, From 536522304ad9e5ec20af66bf5ce1f49c528e2a98 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sun, 23 Aug 2026 17:20:28 +0000 Subject: [PATCH 70/80] autoresearch: measure Test CI 20260823T172024Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .github/workflows/ci.yml | 2 +- package.json | 2 +- scripts/coverage-config.test.mjs | 6 +++++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index aa4cf68ee06b..8ad23d9ceed7 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -66,3 +66,4 @@ {"run":65,"commit":"e542e12","metric":736,"metrics":{"packages_coverage_seconds":560,"examples_seconds":111,"startup_seconds":60,"ci_run_id":32616764867},"status":"keep","description":"Use a legacy-name-compliant compile-cache path","timestamp":1787459929586,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Renaming the `/tmp` cache directory should preserve performance while satisfying the repository's legacy-name lint.","result":"The Test job passed in 736s with coverage at 560s, effectively matching the 734s cache median sample.","correctness":"Product files no longer introduce the retired name; the remaining lint occurrence is only in the temporary append-only autoresearch log, which is excluded from the final branch."}} {"run":66,"commit":"bfa1321","metric":0,"metrics":{"packages_coverage_seconds":563,"examples_seconds":0,"startup_seconds":64,"ci_run_id":32628030774},"status":"checks_failed","description":"Reuse SQL ORM client workers only on CI","timestamp":1787475634040,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Disabling file isolation only for the 69-file stateless SQL ORM client project would remove VM-context churn without risking cross-file state leakage.","result":"All SQL ORM client tests passed, and the package phase reached its summary after 563s, but a repository guard rejected the source-level `process.env.CI` branch in the package config.","error_details":"`@internal/cli test/utils/no-parallel-ci-detection.test.ts` found `packages/3-extensions/sql-orm-client/vitest.config.ts: ...(process.env['CI'] ? { isolate: false } : {})`. The guard requires package behavior not to branch directly on CI.","rollback_reason":"The candidate violates the repository's no-target-branches rule, so no timing result is valid.","next_action_hint":"Apply `isolate: false` unconditionally in this demonstrably stateless project and rerun; this matches the existing CLI project pattern and avoids a target-specific branch."}} {"run":67,"commit":"1812f15","metric":766,"metrics":{"packages_coverage_seconds":574,"examples_seconds":115,"startup_seconds":69,"ci_run_id":32631001906},"status":"discard","description":"Reuse workers only for the stateless SQL ORM client project","timestamp":1787478133948,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Setting `isolate: false` only for the 69-file stateless SQL ORM client project would reduce VM-context setup while preserving isolation for stateful projects.","result":"The full hosted Test job passed in 766s with package coverage at 574s. Vitest reported 952 isolated worker contexts, down from roughly 998 in earlier fully isolated runs, but the complete job and coverage phase were slower than the selected cache+vmThreads median (~736s complete, ~553s coverage).","rollback_reason":"The targeted reuse added package-specific semantic/configuration complexity without improving the primary metric; the saved contexts are too small a fraction of the dominant workload to rise above runner variance.","next_action_hint":"Restore isolation for SQL ORM client. If revisiting targeted reuse, batch several independently proven stateless projects in one experiment so the structural reduction is large enough to measure."}} +{"run":68,"commit":"4ef026a","metric":776,"metrics":{"packages_coverage_seconds":587,"examples_seconds":115,"startup_seconds":67,"ci_run_id":32646852824},"status":"discard","description":"Place the Node compile cache on runner tmpfs","timestamp":1787498325499,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"Moving the job-local Node compile cache from disk-backed `/tmp` to memory-backed `/dev/shm` would reduce cache lookup/write latency across isolated workers without changing cache semantics.","result":"The hosted Test job passed in 776s with package coverage at 587s. Both are slower than the selected disk-cache distribution (~736s complete and ~553s coverage), with no evidence that cache storage I/O is material.","rollback_reason":"The primary metric regressed by roughly 40s from the selected median, and `/dev/shm` adds platform/capacity assumptions without demonstrated benefit.","next_action_hint":"Restore the `/tmp` cache path. Focus on execution scheduling or larger structural costs rather than the compile-cache storage medium."}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a64c6655ff3d..98afd83b4a98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,7 +167,7 @@ jobs: runs-on: ubuntu-latest env: TEST_TIMEOUT_MULTIPLIER: 2 - NODE_COMPILE_CACHE: /dev/shm/prisma-test-node-compile-cache + NODE_COMPILE_CACHE: /tmp/prisma-test-node-compile-cache # Used by examples/prisma-8-cloudflare-worker's vitest-pool-workers # integration test. Mirrors the .env.example pattern; the container is # brought up by `pnpm db:up` below (docker-compose, not a service diff --git a/package.json b/package.json index 044e19e7f43b..c9711f0f5091 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "test:all": "pnpm test:packages && pnpm test:examples && pnpm test:integration && pnpm test:e2e", "test:packages": "turbo run build --filter='!./examples/**' --filter='!./test/**' && vitest run", "test:packages:agent": "node scripts/run-logged.mjs test-packages pnpm test:packages", - "test:examples": "turbo run test --filter='./examples/**' --continue", + "test:examples": "turbo run test --filter='./examples/**' --continue --concurrency=4", "test:e2e": "pnpm --filter e2e-tests test", "test:e2e:agent": "node scripts/run-logged.mjs test-e2e pnpm test:e2e", "test:integration": "pnpm --filter integration-tests test", diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index bc1c82591e23..f01fde9ec973 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -255,6 +255,10 @@ describe('coverage config', () => { ); assert.equal(rootManifest.scripts['test:coverage'], 'pnpm coverage:packages'); assert.equal(rootManifest.scripts['coverage:report'], 'node scripts/coverage-report.mjs'); + assert.equal( + rootManifest.scripts['test:examples'], + "turbo run test --filter='./examples/**' --continue --concurrency=4", + ); for await (const path of glob('packages/**/package.json', { cwd: repositoryRoot })) { const manifest = JSON.parse(await readFile(join(repositoryRoot, path), 'utf8')); @@ -280,7 +284,7 @@ describe('coverage config', () => { assert.ok(testJob); assert.match(testJob, /^ {4}name: Test$/m); - assert.match(testJob, /^ {6}NODE_COMPILE_CACHE: \/dev\/shm\/prisma-test-node-compile-cache$/m); + assert.match(testJob, /^ {6}NODE_COMPILE_CACHE: \/tmp\/prisma-test-node-compile-cache$/m); assert.match( testJob, /run: pnpm coverage:packages\n {6}- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, From 2cc2dc0191ce12165b8e8fb304a68e71e02ad4d1 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sun, 23 Aug 2026 17:53:42 +0000 Subject: [PATCH 71/80] autoresearch: measure Test CI 20260823T175341Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 8ad23d9ceed7..036aa80a88ab 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -67,3 +67,4 @@ {"run":66,"commit":"bfa1321","metric":0,"metrics":{"packages_coverage_seconds":563,"examples_seconds":0,"startup_seconds":64,"ci_run_id":32628030774},"status":"checks_failed","description":"Reuse SQL ORM client workers only on CI","timestamp":1787475634040,"segment":0,"confidence":9.966666666666667,"asi":{"hypothesis":"Disabling file isolation only for the 69-file stateless SQL ORM client project would remove VM-context churn without risking cross-file state leakage.","result":"All SQL ORM client tests passed, and the package phase reached its summary after 563s, but a repository guard rejected the source-level `process.env.CI` branch in the package config.","error_details":"`@internal/cli test/utils/no-parallel-ci-detection.test.ts` found `packages/3-extensions/sql-orm-client/vitest.config.ts: ...(process.env['CI'] ? { isolate: false } : {})`. The guard requires package behavior not to branch directly on CI.","rollback_reason":"The candidate violates the repository's no-target-branches rule, so no timing result is valid.","next_action_hint":"Apply `isolate: false` unconditionally in this demonstrably stateless project and rerun; this matches the existing CLI project pattern and avoids a target-specific branch."}} {"run":67,"commit":"1812f15","metric":766,"metrics":{"packages_coverage_seconds":574,"examples_seconds":115,"startup_seconds":69,"ci_run_id":32631001906},"status":"discard","description":"Reuse workers only for the stateless SQL ORM client project","timestamp":1787478133948,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Setting `isolate: false` only for the 69-file stateless SQL ORM client project would reduce VM-context setup while preserving isolation for stateful projects.","result":"The full hosted Test job passed in 766s with package coverage at 574s. Vitest reported 952 isolated worker contexts, down from roughly 998 in earlier fully isolated runs, but the complete job and coverage phase were slower than the selected cache+vmThreads median (~736s complete, ~553s coverage).","rollback_reason":"The targeted reuse added package-specific semantic/configuration complexity without improving the primary metric; the saved contexts are too small a fraction of the dominant workload to rise above runner variance.","next_action_hint":"Restore isolation for SQL ORM client. If revisiting targeted reuse, batch several independently proven stateless projects in one experiment so the structural reduction is large enough to measure."}} {"run":68,"commit":"4ef026a","metric":776,"metrics":{"packages_coverage_seconds":587,"examples_seconds":115,"startup_seconds":67,"ci_run_id":32646852824},"status":"discard","description":"Place the Node compile cache on runner tmpfs","timestamp":1787498325499,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"Moving the job-local Node compile cache from disk-backed `/tmp` to memory-backed `/dev/shm` would reduce cache lookup/write latency across isolated workers without changing cache semantics.","result":"The hosted Test job passed in 776s with package coverage at 587s. Both are slower than the selected disk-cache distribution (~736s complete and ~553s coverage), with no evidence that cache storage I/O is material.","rollback_reason":"The primary metric regressed by roughly 40s from the selected median, and `/dev/shm` adds platform/capacity assumptions without demonstrated benefit.","next_action_hint":"Restore the `/tmp` cache path. Focus on execution scheduling or larger structural costs rather than the compile-cache storage medium."}} +{"run":69,"commit":"5365223","metric":732,"metrics":{"packages_coverage_seconds":555,"examples_seconds":113,"startup_seconds":59,"ci_run_id":32654539897},"status":"keep","description":"Cap example-test task concurrency at runner core count","timestamp":1787506590558,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Limiting Turbo to four concurrent example test tasks would avoid oversubscribing the four-core hosted runner with roughly eleven Vitest processes, improving or stabilizing the examples phase without changing test coverage.","result":"The full job passed in 732s, 4s below the selected ~736s median. The examples phase took 113s, however, which is indistinguishable from the existing 111–115s normal range; the small primary improvement came from package/startup variance.","confidence_note":"This first point does not establish a causal benefit because the directly affected phase did not improve. Retain provisionally only because the primary metric improved, and require a repeat before adding configuration complexity.","next_action_hint":"Repeat the four-task example cap unchanged. Discard unless the examples phase is consistently lower or materially less variable than the uncapped distribution."}} From 3219a6aac7e2d10deac3c5a5208a6661ad4758dd Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sun, 23 Aug 2026 19:38:21 +0000 Subject: [PATCH 72/80] autoresearch: measure Test CI 20260823T193821Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .github/workflows/ci.yml | 19 ++++++++----------- package.json | 2 +- scripts/coverage-config.test.mjs | 8 ++++---- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 036aa80a88ab..ef505bd403fb 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -68,3 +68,4 @@ {"run":67,"commit":"1812f15","metric":766,"metrics":{"packages_coverage_seconds":574,"examples_seconds":115,"startup_seconds":69,"ci_run_id":32631001906},"status":"discard","description":"Reuse workers only for the stateless SQL ORM client project","timestamp":1787478133948,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Setting `isolate: false` only for the 69-file stateless SQL ORM client project would reduce VM-context setup while preserving isolation for stateful projects.","result":"The full hosted Test job passed in 766s with package coverage at 574s. Vitest reported 952 isolated worker contexts, down from roughly 998 in earlier fully isolated runs, but the complete job and coverage phase were slower than the selected cache+vmThreads median (~736s complete, ~553s coverage).","rollback_reason":"The targeted reuse added package-specific semantic/configuration complexity without improving the primary metric; the saved contexts are too small a fraction of the dominant workload to rise above runner variance.","next_action_hint":"Restore isolation for SQL ORM client. If revisiting targeted reuse, batch several independently proven stateless projects in one experiment so the structural reduction is large enough to measure."}} {"run":68,"commit":"4ef026a","metric":776,"metrics":{"packages_coverage_seconds":587,"examples_seconds":115,"startup_seconds":67,"ci_run_id":32646852824},"status":"discard","description":"Place the Node compile cache on runner tmpfs","timestamp":1787498325499,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"Moving the job-local Node compile cache from disk-backed `/tmp` to memory-backed `/dev/shm` would reduce cache lookup/write latency across isolated workers without changing cache semantics.","result":"The hosted Test job passed in 776s with package coverage at 587s. Both are slower than the selected disk-cache distribution (~736s complete and ~553s coverage), with no evidence that cache storage I/O is material.","rollback_reason":"The primary metric regressed by roughly 40s from the selected median, and `/dev/shm` adds platform/capacity assumptions without demonstrated benefit.","next_action_hint":"Restore the `/tmp` cache path. Focus on execution scheduling or larger structural costs rather than the compile-cache storage medium."}} {"run":69,"commit":"5365223","metric":732,"metrics":{"packages_coverage_seconds":555,"examples_seconds":113,"startup_seconds":59,"ci_run_id":32654539897},"status":"keep","description":"Cap example-test task concurrency at runner core count","timestamp":1787506590558,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Limiting Turbo to four concurrent example test tasks would avoid oversubscribing the four-core hosted runner with roughly eleven Vitest processes, improving or stabilizing the examples phase without changing test coverage.","result":"The full job passed in 732s, 4s below the selected ~736s median. The examples phase took 113s, however, which is indistinguishable from the existing 111–115s normal range; the small primary improvement came from package/startup variance.","confidence_note":"This first point does not establish a causal benefit because the directly affected phase did not improve. Retain provisionally only because the primary metric improved, and require a repeat before adding configuration complexity.","next_action_hint":"Repeat the four-task example cap unchanged. Discard unless the examples phase is consistently lower or materially less variable than the uncapped distribution."}} +{"run":70,"commit":"2cc2dc0","metric":792,"metrics":{"packages_coverage_seconds":611,"examples_seconds":115,"startup_seconds":61,"ci_run_id":32656308373},"status":"discard","description":"Repeat four-task example-test concurrency cap","timestamp":1787509296641,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"A second run would show whether limiting Turbo to four example-test tasks improves the directly affected examples phase rather than merely coinciding with favorable package-test variance.","result":"The full job passed in 792s and examples took 115s. Across the two capped samples, examples took 113s and 115s (median 114s), identical to the normal uncapped 111–115s range.","rollback_reason":"The concurrency cap produced no phase-level benefit and adds a hardware-specific scheduling knob; the first run's 732s total was package-phase variance, not an examples optimization.","next_action_hint":"Restore Turbo's default example concurrency. Do not tune this phase further without per-example resource/timing evidence."}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98afd83b4a98..8fc2fc37f3c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,18 +193,15 @@ jobs: with: persist-credentials: false - uses: ./.github/actions/setup - - name: Install dependencies (skip bin linking) + - name: Install, build, link bins, and start cloudflare-worker Postgres if: needs.changes.outputs.inert != 'true' - run: pnpm install --frozen-lockfile --ignore-scripts - - name: Build packages (restored from Turbo cache; needed for bin linking) - if: needs.changes.outputs.inert != 'true' - run: pnpm build - - name: Link bins - if: needs.changes.outputs.inert != 'true' - run: pnpm install --frozen-lockfile - - name: Start cloudflare-worker Postgres (5433, pg_stat_statements) - if: needs.changes.outputs.inert != 'true' - run: pnpm --filter prisma-8-cloudflare-worker db:up + run: | + pnpm --filter prisma-8-cloudflare-worker db:up & + cloudflare_db_pid=$! + pnpm install --frozen-lockfile --ignore-scripts + pnpm build + pnpm install --frozen-lockfile + wait "$cloudflare_db_pid" - name: Test packages with coverage if: needs.changes.outputs.inert != 'true' run: pnpm coverage:packages diff --git a/package.json b/package.json index c9711f0f5091..044e19e7f43b 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "test:all": "pnpm test:packages && pnpm test:examples && pnpm test:integration && pnpm test:e2e", "test:packages": "turbo run build --filter='!./examples/**' --filter='!./test/**' && vitest run", "test:packages:agent": "node scripts/run-logged.mjs test-packages pnpm test:packages", - "test:examples": "turbo run test --filter='./examples/**' --continue --concurrency=4", + "test:examples": "turbo run test --filter='./examples/**' --continue", "test:e2e": "pnpm --filter e2e-tests test", "test:e2e:agent": "node scripts/run-logged.mjs test-e2e pnpm test:e2e", "test:integration": "pnpm --filter integration-tests test", diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index f01fde9ec973..74e33b31fc54 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -255,10 +255,6 @@ describe('coverage config', () => { ); assert.equal(rootManifest.scripts['test:coverage'], 'pnpm coverage:packages'); assert.equal(rootManifest.scripts['coverage:report'], 'node scripts/coverage-report.mjs'); - assert.equal( - rootManifest.scripts['test:examples'], - "turbo run test --filter='./examples/**' --continue --concurrency=4", - ); for await (const path of glob('packages/**/package.json', { cwd: repositoryRoot })) { const manifest = JSON.parse(await readFile(join(repositoryRoot, path), 'utf8')); @@ -285,6 +281,10 @@ describe('coverage config', () => { assert.ok(testJob); assert.match(testJob, /^ {4}name: Test$/m); assert.match(testJob, /^ {6}NODE_COMPILE_CACHE: \/tmp\/prisma-test-node-compile-cache$/m); + assert.match( + testJob, + /- name: Install, build, link bins, and start cloudflare-worker Postgres\n {8}if: needs\.changes\.outputs\.inert != 'true'\n {8}run: \|\n {10}pnpm --filter prisma-8-cloudflare-worker db:up &\n {10}cloudflare_db_pid=\$!\n {10}pnpm install --frozen-lockfile --ignore-scripts\n {10}pnpm build\n {10}pnpm install --frozen-lockfile\n {10}wait "\$cloudflare_db_pid"/, + ); assert.match( testJob, /run: pnpm coverage:packages\n {6}- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, From 8173d1e4bf335fa8505fadf4f061c1e0845af451 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Sun, 23 Aug 2026 22:18:36 +0000 Subject: [PATCH 73/80] autoresearch: measure Test CI 20260823T221835Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + .github/workflows/ci.yml | 19 +++++++++++-------- scripts/coverage-config.test.mjs | 8 ++++---- vitest.config.ts | 1 + 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index ef505bd403fb..12dac8365b27 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -69,3 +69,4 @@ {"run":68,"commit":"4ef026a","metric":776,"metrics":{"packages_coverage_seconds":587,"examples_seconds":115,"startup_seconds":67,"ci_run_id":32646852824},"status":"discard","description":"Place the Node compile cache on runner tmpfs","timestamp":1787498325499,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"Moving the job-local Node compile cache from disk-backed `/tmp` to memory-backed `/dev/shm` would reduce cache lookup/write latency across isolated workers without changing cache semantics.","result":"The hosted Test job passed in 776s with package coverage at 587s. Both are slower than the selected disk-cache distribution (~736s complete and ~553s coverage), with no evidence that cache storage I/O is material.","rollback_reason":"The primary metric regressed by roughly 40s from the selected median, and `/dev/shm` adds platform/capacity assumptions without demonstrated benefit.","next_action_hint":"Restore the `/tmp` cache path. Focus on execution scheduling or larger structural costs rather than the compile-cache storage medium."}} {"run":69,"commit":"5365223","metric":732,"metrics":{"packages_coverage_seconds":555,"examples_seconds":113,"startup_seconds":59,"ci_run_id":32654539897},"status":"keep","description":"Cap example-test task concurrency at runner core count","timestamp":1787506590558,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Limiting Turbo to four concurrent example test tasks would avoid oversubscribing the four-core hosted runner with roughly eleven Vitest processes, improving or stabilizing the examples phase without changing test coverage.","result":"The full job passed in 732s, 4s below the selected ~736s median. The examples phase took 113s, however, which is indistinguishable from the existing 111–115s normal range; the small primary improvement came from package/startup variance.","confidence_note":"This first point does not establish a causal benefit because the directly affected phase did not improve. Retain provisionally only because the primary metric improved, and require a repeat before adding configuration complexity.","next_action_hint":"Repeat the four-task example cap unchanged. Discard unless the examples phase is consistently lower or materially less variable than the uncapped distribution."}} {"run":70,"commit":"2cc2dc0","metric":792,"metrics":{"packages_coverage_seconds":611,"examples_seconds":115,"startup_seconds":61,"ci_run_id":32656308373},"status":"discard","description":"Repeat four-task example-test concurrency cap","timestamp":1787509296641,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"A second run would show whether limiting Turbo to four example-test tasks improves the directly affected examples phase rather than merely coinciding with favorable package-test variance.","result":"The full job passed in 792s and examples took 115s. Across the two capped samples, examples took 113s and 115s (median 114s), identical to the normal uncapped 111–115s range.","rollback_reason":"The concurrency cap produced no phase-level benefit and adds a hardware-specific scheduling knob; the first run's 732s total was package-phase variance, not an examples optimization.","next_action_hint":"Restore Turbo's default example concurrency. Do not tune this phase further without per-example resource/timing evidence."}} +{"run":71,"commit":"3219a6a","metric":792,"metrics":{"packages_coverage_seconds":603,"examples_seconds":116,"startup_seconds":65,"ci_run_id":32663104401},"status":"discard","description":"Overlap Cloudflare Postgres startup with Test-job preparation","timestamp":1787516622482,"segment":0,"confidence":10.872727272727273,"asi":{"hypothesis":"Starting the example-only Cloudflare Postgres container in parallel with dependency installation, cached build, and bin linking would remove its roughly 10s startup from the Test job's critical path while preserving readiness before tests.","result":"The full job passed in 792s. Startup took 65s, versus roughly 59–63s for the selected sequential workflow; package coverage and examples were also ordinary slow-runner samples.","rollback_reason":"The directly affected startup phase did not improve, so combining four clear workflow steps into one complex background-process step is unjustified.","next_action_hint":"Restore separate preparation and database-start steps. The container startup is likely contending with install/network work rather than being hidden by it."}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fc2fc37f3c6..98afd83b4a98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,15 +193,18 @@ jobs: with: persist-credentials: false - uses: ./.github/actions/setup - - name: Install, build, link bins, and start cloudflare-worker Postgres + - name: Install dependencies (skip bin linking) if: needs.changes.outputs.inert != 'true' - run: | - pnpm --filter prisma-8-cloudflare-worker db:up & - cloudflare_db_pid=$! - pnpm install --frozen-lockfile --ignore-scripts - pnpm build - pnpm install --frozen-lockfile - wait "$cloudflare_db_pid" + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Build packages (restored from Turbo cache; needed for bin linking) + if: needs.changes.outputs.inert != 'true' + run: pnpm build + - name: Link bins + if: needs.changes.outputs.inert != 'true' + run: pnpm install --frozen-lockfile + - name: Start cloudflare-worker Postgres (5433, pg_stat_statements) + if: needs.changes.outputs.inert != 'true' + run: pnpm --filter prisma-8-cloudflare-worker db:up - name: Test packages with coverage if: needs.changes.outputs.inert != 'true' run: pnpm coverage:packages diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 74e33b31fc54..0dbfedb492f1 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -270,6 +270,10 @@ describe('coverage config', () => { assert.doesNotMatch(rootVitestConfig, /from ['"]\.\/scripts\/coverage-config\.[^'"]+['"]/); assert.match(rootVitestConfig, /provider:\s*['"]v8['"]/); assert.match(rootVitestConfig, /reportOnFailure:\s*true/); + assert.match( + rootVitestConfig, + /\.\.\.\(process\.env\['CI'\] \? \{ disableConsoleIntercept: true \} : \{\}\)/, + ); }); it('combines package tests and coverage in one CI job', async () => { @@ -281,10 +285,6 @@ describe('coverage config', () => { assert.ok(testJob); assert.match(testJob, /^ {4}name: Test$/m); assert.match(testJob, /^ {6}NODE_COMPILE_CACHE: \/tmp\/prisma-test-node-compile-cache$/m); - assert.match( - testJob, - /- name: Install, build, link bins, and start cloudflare-worker Postgres\n {8}if: needs\.changes\.outputs\.inert != 'true'\n {8}run: \|\n {10}pnpm --filter prisma-8-cloudflare-worker db:up &\n {10}cloudflare_db_pid=\$!\n {10}pnpm install --frozen-lockfile --ignore-scripts\n {10}pnpm build\n {10}pnpm install --frozen-lockfile\n {10}wait "\$cloudflare_db_pid"/, - ); assert.match( testJob, /run: pnpm coverage:packages\n {6}- name: Report package coverage\n {8}if: \$\{\{ !cancelled\(\) && needs\.changes\.outputs\.inert != 'true' \}\}\n {8}run: pnpm coverage:report/, diff --git a/vitest.config.ts b/vitest.config.ts index 31159ecc5cdc..473232b81135 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,6 +11,7 @@ export default defineConfig({ // Stateful projects can override this default, as the Supabase suite does. maxWorkers: process.env['CI'] ? '100%' : undefined, pool: process.env['CI'] ? 'vmThreads' : undefined, + ...(process.env['CI'] ? { disableConsoleIntercept: true } : {}), // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 34336087c9b55c23d118b78046d6546942508584 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Mon, 24 Aug 2026 00:09:48 +0000 Subject: [PATCH 74/80] autoresearch: measure Test CI 20260824T000945Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + scripts/coverage-config.test.mjs | 2 +- vitest.config.ts | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 12dac8365b27..ebe2f0f42dad 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -70,3 +70,4 @@ {"run":69,"commit":"5365223","metric":732,"metrics":{"packages_coverage_seconds":555,"examples_seconds":113,"startup_seconds":59,"ci_run_id":32654539897},"status":"keep","description":"Cap example-test task concurrency at runner core count","timestamp":1787506590558,"segment":0,"confidence":10.310344827586206,"asi":{"hypothesis":"Limiting Turbo to four concurrent example test tasks would avoid oversubscribing the four-core hosted runner with roughly eleven Vitest processes, improving or stabilizing the examples phase without changing test coverage.","result":"The full job passed in 732s, 4s below the selected ~736s median. The examples phase took 113s, however, which is indistinguishable from the existing 111–115s normal range; the small primary improvement came from package/startup variance.","confidence_note":"This first point does not establish a causal benefit because the directly affected phase did not improve. Retain provisionally only because the primary metric improved, and require a repeat before adding configuration complexity.","next_action_hint":"Repeat the four-task example cap unchanged. Discard unless the examples phase is consistently lower or materially less variable than the uncapped distribution."}} {"run":70,"commit":"2cc2dc0","metric":792,"metrics":{"packages_coverage_seconds":611,"examples_seconds":115,"startup_seconds":61,"ci_run_id":32656308373},"status":"discard","description":"Repeat four-task example-test concurrency cap","timestamp":1787509296641,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"A second run would show whether limiting Turbo to four example-test tasks improves the directly affected examples phase rather than merely coinciding with favorable package-test variance.","result":"The full job passed in 792s and examples took 115s. Across the two capped samples, examples took 113s and 115s (median 114s), identical to the normal uncapped 111–115s range.","rollback_reason":"The concurrency cap produced no phase-level benefit and adds a hardware-specific scheduling knob; the first run's 732s total was package-phase variance, not an examples optimization.","next_action_hint":"Restore Turbo's default example concurrency. Do not tune this phase further without per-example resource/timing evidence."}} {"run":71,"commit":"3219a6a","metric":792,"metrics":{"packages_coverage_seconds":603,"examples_seconds":116,"startup_seconds":65,"ci_run_id":32663104401},"status":"discard","description":"Overlap Cloudflare Postgres startup with Test-job preparation","timestamp":1787516622482,"segment":0,"confidence":10.872727272727273,"asi":{"hypothesis":"Starting the example-only Cloudflare Postgres container in parallel with dependency installation, cached build, and bin linking would remove its roughly 10s startup from the Test job's critical path while preserving readiness before tests.","result":"The full job passed in 792s. Startup took 65s, versus roughly 59–63s for the selected sequential workflow; package coverage and examples were also ordinary slow-runner samples.","rollback_reason":"The directly affected startup phase did not improve, so combining four clear workflow steps into one complex background-process step is unjustified.","next_action_hint":"Restore separate preparation and database-start steps. The container startup is likely contending with install/network work rather than being hidden by it."}} +{"run":72,"commit":"8173d1e","metric":761,"metrics":{"packages_coverage_seconds":579,"examples_seconds":114,"startup_seconds":63,"ci_run_id":32670208286},"status":"discard","description":"Disable Vitest console interception on CI","timestamp":1787524490671,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Avoiding per-file console interception on CI would reduce setup and output-buffering overhead across roughly 1,165 isolated test files while preserving test execution and pass/fail behavior.","result":"The hosted Test job passed in 761s with package coverage at 579s. This is slower than the selected ~736s complete-job and ~553s package-phase medians, with no observable execution benefit.","rollback_reason":"Console interception is not a material bottleneck, and disabling it would reduce the diagnostic association of logs with tests without improving the primary metric.","next_action_hint":"Restore normal console interception. Avoid further reporter/output micro-optimizations; prior text coverage and task-output experiments also showed negligible impact."}} diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 0dbfedb492f1..f19771043c42 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -272,7 +272,7 @@ describe('coverage config', () => { assert.match(rootVitestConfig, /reportOnFailure:\s*true/); assert.match( rootVitestConfig, - /\.\.\.\(process\.env\['CI'\] \? \{ disableConsoleIntercept: true \} : \{\}\)/, + /\.\.\.\(process\.env\['CI'\] \? \{ experimental: \{ diagnostics: false \} \} : \{\}\)/, ); }); diff --git a/vitest.config.ts b/vitest.config.ts index 473232b81135..d6d05b2085b3 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,7 +11,7 @@ export default defineConfig({ // Stateful projects can override this default, as the Supabase suite does. maxWorkers: process.env['CI'] ? '100%' : undefined, pool: process.env['CI'] ? 'vmThreads' : undefined, - ...(process.env['CI'] ? { disableConsoleIntercept: true } : {}), + ...(process.env['CI'] ? { experimental: { diagnostics: false } } : {}), // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 3d02748a16540a6cf41d0242dc347a361604a745 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Mon, 24 Aug 2026 01:13:35 +0000 Subject: [PATCH 75/80] autoresearch: measure Test CI 20260824T011335Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + scripts/coverage-config.test.mjs | 4 ---- vitest.config.ts | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index ebe2f0f42dad..9f37e6c34081 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -71,3 +71,4 @@ {"run":70,"commit":"2cc2dc0","metric":792,"metrics":{"packages_coverage_seconds":611,"examples_seconds":115,"startup_seconds":61,"ci_run_id":32656308373},"status":"discard","description":"Repeat four-task example-test concurrency cap","timestamp":1787509296641,"segment":0,"confidence":10.678571428571429,"asi":{"hypothesis":"A second run would show whether limiting Turbo to four example-test tasks improves the directly affected examples phase rather than merely coinciding with favorable package-test variance.","result":"The full job passed in 792s and examples took 115s. Across the two capped samples, examples took 113s and 115s (median 114s), identical to the normal uncapped 111–115s range.","rollback_reason":"The concurrency cap produced no phase-level benefit and adds a hardware-specific scheduling knob; the first run's 732s total was package-phase variance, not an examples optimization.","next_action_hint":"Restore Turbo's default example concurrency. Do not tune this phase further without per-example resource/timing evidence."}} {"run":71,"commit":"3219a6a","metric":792,"metrics":{"packages_coverage_seconds":603,"examples_seconds":116,"startup_seconds":65,"ci_run_id":32663104401},"status":"discard","description":"Overlap Cloudflare Postgres startup with Test-job preparation","timestamp":1787516622482,"segment":0,"confidence":10.872727272727273,"asi":{"hypothesis":"Starting the example-only Cloudflare Postgres container in parallel with dependency installation, cached build, and bin linking would remove its roughly 10s startup from the Test job's critical path while preserving readiness before tests.","result":"The full job passed in 792s. Startup took 65s, versus roughly 59–63s for the selected sequential workflow; package coverage and examples were also ordinary slow-runner samples.","rollback_reason":"The directly affected startup phase did not improve, so combining four clear workflow steps into one complex background-process step is unjustified.","next_action_hint":"Restore separate preparation and database-start steps. The container startup is likely contending with install/network work rather than being hidden by it."}} {"run":72,"commit":"8173d1e","metric":761,"metrics":{"packages_coverage_seconds":579,"examples_seconds":114,"startup_seconds":63,"ci_run_id":32670208286},"status":"discard","description":"Disable Vitest console interception on CI","timestamp":1787524490671,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Avoiding per-file console interception on CI would reduce setup and output-buffering overhead across roughly 1,165 isolated test files while preserving test execution and pass/fail behavior.","result":"The hosted Test job passed in 761s with package coverage at 579s. This is slower than the selected ~736s complete-job and ~553s package-phase medians, with no observable execution benefit.","rollback_reason":"Console interception is not a material bottleneck, and disabling it would reduce the diagnostic association of logs with tests without improving the primary metric.","next_action_hint":"Restore normal console interception. Avoid further reporter/output micro-optimizations; prior text coverage and task-output experiments also showed negligible impact."}} +{"run":73,"commit":"3433608","metric":752,"metrics":{"packages_coverage_seconds":575,"examples_seconds":113,"startup_seconds":56,"ci_run_id":32676681550},"status":"discard","description":"Disable Vitest performance diagnostics on CI","timestamp":1787532103462,"segment":0,"confidence":11.283018867924529,"asi":{"hypothesis":"Disabling Vitest's default performance diagnostics would avoid timing/bookkeeping overhead across roughly 1,165 files while preserving all tests, coverage, and normal failure reporting.","result":"The hosted Test job passed in 752s with package coverage at 575s. The package phase is slower than the selected ~553s median, and removing the final diagnostic hint produced no measurable benefit.","rollback_reason":"Diagnostic collection is not a material bottleneck; retaining Vitest's actionable isolation/transform hints is preferable without a primary-metric improvement.","next_action_hint":"Restore diagnostics. The remaining dominant cost is actual test/import execution, not reporting or diagnostic bookkeeping."}} diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index f19771043c42..d2ccde5de37e 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -270,10 +270,6 @@ describe('coverage config', () => { assert.doesNotMatch(rootVitestConfig, /from ['"]\.\/scripts\/coverage-config\.[^'"]+['"]/); assert.match(rootVitestConfig, /provider:\s*['"]v8['"]/); assert.match(rootVitestConfig, /reportOnFailure:\s*true/); - assert.match( - rootVitestConfig, - /\.\.\.\(process\.env\['CI'\] \? \{ experimental: \{ diagnostics: false \} \} : \{\}\)/, - ); }); it('combines package tests and coverage in one CI job', async () => { diff --git a/vitest.config.ts b/vitest.config.ts index d6d05b2085b3..31159ecc5cdc 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,7 +11,6 @@ export default defineConfig({ // Stateful projects can override this default, as the Supabase suite does. maxWorkers: process.env['CI'] ? '100%' : undefined, pool: process.env['CI'] ? 'vmThreads' : undefined, - ...(process.env['CI'] ? { experimental: { diagnostics: false } } : {}), // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From dc4efe8e2b7aeea4490e08d8a8516687da02eeef Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Mon, 24 Aug 2026 03:51:27 +0000 Subject: [PATCH 76/80] autoresearch: measure Test CI 20260824T035127Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + scripts/coverage-config.test.mjs | 1 + vitest.config.ts | 1 + 3 files changed, 3 insertions(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 9f37e6c34081..edbfab4ed783 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -72,3 +72,4 @@ {"run":71,"commit":"3219a6a","metric":792,"metrics":{"packages_coverage_seconds":603,"examples_seconds":116,"startup_seconds":65,"ci_run_id":32663104401},"status":"discard","description":"Overlap Cloudflare Postgres startup with Test-job preparation","timestamp":1787516622482,"segment":0,"confidence":10.872727272727273,"asi":{"hypothesis":"Starting the example-only Cloudflare Postgres container in parallel with dependency installation, cached build, and bin linking would remove its roughly 10s startup from the Test job's critical path while preserving readiness before tests.","result":"The full job passed in 792s. Startup took 65s, versus roughly 59–63s for the selected sequential workflow; package coverage and examples were also ordinary slow-runner samples.","rollback_reason":"The directly affected startup phase did not improve, so combining four clear workflow steps into one complex background-process step is unjustified.","next_action_hint":"Restore separate preparation and database-start steps. The container startup is likely contending with install/network work rather than being hidden by it."}} {"run":72,"commit":"8173d1e","metric":761,"metrics":{"packages_coverage_seconds":579,"examples_seconds":114,"startup_seconds":63,"ci_run_id":32670208286},"status":"discard","description":"Disable Vitest console interception on CI","timestamp":1787524490671,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Avoiding per-file console interception on CI would reduce setup and output-buffering overhead across roughly 1,165 isolated test files while preserving test execution and pass/fail behavior.","result":"The hosted Test job passed in 761s with package coverage at 579s. This is slower than the selected ~736s complete-job and ~553s package-phase medians, with no observable execution benefit.","rollback_reason":"Console interception is not a material bottleneck, and disabling it would reduce the diagnostic association of logs with tests without improving the primary metric.","next_action_hint":"Restore normal console interception. Avoid further reporter/output micro-optimizations; prior text coverage and task-output experiments also showed negligible impact."}} {"run":73,"commit":"3433608","metric":752,"metrics":{"packages_coverage_seconds":575,"examples_seconds":113,"startup_seconds":56,"ci_run_id":32676681550},"status":"discard","description":"Disable Vitest performance diagnostics on CI","timestamp":1787532103462,"segment":0,"confidence":11.283018867924529,"asi":{"hypothesis":"Disabling Vitest's default performance diagnostics would avoid timing/bookkeeping overhead across roughly 1,165 files while preserving all tests, coverage, and normal failure reporting.","result":"The hosted Test job passed in 752s with package coverage at 575s. The package phase is slower than the selected ~553s median, and removing the final diagnostic hint produced no measurable benefit.","rollback_reason":"Diagnostic collection is not a material bottleneck; retaining Vitest's actionable isolation/transform hints is preferable without a primary-metric improvement.","next_action_hint":"Restore diagnostics. The remaining dominant cost is actual test/import execution, not reporting or diagnostic bookkeeping."}} +{"run":74,"commit":"3d02748","metric":766,"metrics":{"packages_coverage_seconds":587,"examples_seconds":115,"startup_seconds":59,"ci_run_id":32679905499},"status":"keep","description":"Restore and revalidate the selected Test-job configuration","timestamp":1787535900044,"segment":0,"confidence":11.5,"asi":{"hypothesis":"After rejecting several micro-optimizations, restoring the selected all-core vmThreads plus disk-backed Node compile cache configuration would remain stable and materially faster than the 847s baseline.","result":"The exact selected configuration passed in 766s with package coverage at 587s. This is an ordinary slow-runner point but remains 81s (9.6%) below baseline, with all tests, coverage enforcement, examples, services, and clean-tree verification intact.","distribution":"Exact compile-cache candidate runs now span 630–774s and center around roughly 741s; this result is consistent with known hosted-runner variance rather than a regression in configuration.","decision":"Keep the restored minimal candidate. Recent console, diagnostics, tmpfs, example-concurrency, and startup-overlap experiments added complexity without phase-level gains."}} diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index d2ccde5de37e..48e1c41515ab 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -270,6 +270,7 @@ describe('coverage config', () => { assert.doesNotMatch(rootVitestConfig, /from ['"]\.\/scripts\/coverage-config\.[^'"]+['"]/); assert.match(rootVitestConfig, /provider:\s*['"]v8['"]/); assert.match(rootVitestConfig, /reportOnFailure:\s*true/); + assert.match(rootVitestConfig, /vmMemoryLimit: process\.env\['CI'\] \? '2GB' : undefined/); }); it('combines package tests and coverage in one CI job', async () => { diff --git a/vitest.config.ts b/vitest.config.ts index 31159ecc5cdc..50f843e92698 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,6 +11,7 @@ export default defineConfig({ // Stateful projects can override this default, as the Supabase suite does. maxWorkers: process.env['CI'] ? '100%' : undefined, pool: process.env['CI'] ? 'vmThreads' : undefined, + vmMemoryLimit: process.env['CI'] ? '2GB' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 67da9e12823c1cca25ea53d60003558ecc5b544c Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Mon, 24 Aug 2026 04:47:57 +0000 Subject: [PATCH 77/80] autoresearch: measure Test CI 20260824T044757Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index edbfab4ed783..dd022322b2ec 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -73,3 +73,4 @@ {"run":72,"commit":"8173d1e","metric":761,"metrics":{"packages_coverage_seconds":579,"examples_seconds":114,"startup_seconds":63,"ci_run_id":32670208286},"status":"discard","description":"Disable Vitest console interception on CI","timestamp":1787524490671,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Avoiding per-file console interception on CI would reduce setup and output-buffering overhead across roughly 1,165 isolated test files while preserving test execution and pass/fail behavior.","result":"The hosted Test job passed in 761s with package coverage at 579s. This is slower than the selected ~736s complete-job and ~553s package-phase medians, with no observable execution benefit.","rollback_reason":"Console interception is not a material bottleneck, and disabling it would reduce the diagnostic association of logs with tests without improving the primary metric.","next_action_hint":"Restore normal console interception. Avoid further reporter/output micro-optimizations; prior text coverage and task-output experiments also showed negligible impact."}} {"run":73,"commit":"3433608","metric":752,"metrics":{"packages_coverage_seconds":575,"examples_seconds":113,"startup_seconds":56,"ci_run_id":32676681550},"status":"discard","description":"Disable Vitest performance diagnostics on CI","timestamp":1787532103462,"segment":0,"confidence":11.283018867924529,"asi":{"hypothesis":"Disabling Vitest's default performance diagnostics would avoid timing/bookkeeping overhead across roughly 1,165 files while preserving all tests, coverage, and normal failure reporting.","result":"The hosted Test job passed in 752s with package coverage at 575s. The package phase is slower than the selected ~553s median, and removing the final diagnostic hint produced no measurable benefit.","rollback_reason":"Diagnostic collection is not a material bottleneck; retaining Vitest's actionable isolation/transform hints is preferable without a primary-metric improvement.","next_action_hint":"Restore diagnostics. The remaining dominant cost is actual test/import execution, not reporting or diagnostic bookkeeping."}} {"run":74,"commit":"3d02748","metric":766,"metrics":{"packages_coverage_seconds":587,"examples_seconds":115,"startup_seconds":59,"ci_run_id":32679905499},"status":"keep","description":"Restore and revalidate the selected Test-job configuration","timestamp":1787535900044,"segment":0,"confidence":11.5,"asi":{"hypothesis":"After rejecting several micro-optimizations, restoring the selected all-core vmThreads plus disk-backed Node compile cache configuration would remain stable and materially faster than the 847s baseline.","result":"The exact selected configuration passed in 766s with package coverage at 587s. This is an ordinary slow-runner point but remains 81s (9.6%) below baseline, with all tests, coverage enforcement, examples, services, and clean-tree verification intact.","distribution":"Exact compile-cache candidate runs now span 630–774s and center around roughly 741s; this result is consistent with known hosted-runner variance rather than a regression in configuration.","decision":"Keep the restored minimal candidate. Recent console, diagnostics, tmpfs, example-concurrency, and startup-overlap experiments added complexity without phase-level gains."}} +{"run":75,"commit":"dc4efe8","metric":618,"metrics":{"packages_coverage_seconds":443,"examples_seconds":97,"startup_seconds":72,"ci_run_id":32688137929},"status":"keep","description":"Recycle VM workers at a 2 GB heap limit","timestamp":1787544979718,"segment":0,"confidence":11.283018867924529,"asi":{"hypothesis":"Lowering vmThreads' per-worker heap limit from the default ~4 GB to 2 GB would recycle workers before pinned VM module contexts create GC/memory pressure, trading occasional worker restart cost for lower sustained heap pressure.","result":"The full hosted job passed in 618s with package coverage at 443s. This is among the fastest safe runs, but examples were also an unusually fast 97s while startup was slow, indicating strongly favorable execution conditions rather than clean attribution.","mechanism":"Vitest documents that Node pins SourceTextModule VM contexts until vmMemoryLimit triggers worker recycling; with ~998 isolated files, a 2 GB threshold is a plausible workload-level control rather than a benchmark shortcut.","confidence_note":"One favorable-runner sample cannot establish benefit. Retain provisionally because the primary metric improved, but require at least two repeats and reject on any OOM, worker crash, or ordinary-runner regression.","next_action_hint":"Repeat 2 GB vmMemoryLimit unchanged twice; compare package-phase median to the ~553s compile-cache candidate median."}} From 405b266cd8cfac00db21e3fff12b2cdf2b3693d0 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Mon, 24 Aug 2026 05:41:11 +0000 Subject: [PATCH 78/80] autoresearch: measure Test CI 20260824T054110Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index dd022322b2ec..e3f8f2d2e00d 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -74,3 +74,4 @@ {"run":73,"commit":"3433608","metric":752,"metrics":{"packages_coverage_seconds":575,"examples_seconds":113,"startup_seconds":56,"ci_run_id":32676681550},"status":"discard","description":"Disable Vitest performance diagnostics on CI","timestamp":1787532103462,"segment":0,"confidence":11.283018867924529,"asi":{"hypothesis":"Disabling Vitest's default performance diagnostics would avoid timing/bookkeeping overhead across roughly 1,165 files while preserving all tests, coverage, and normal failure reporting.","result":"The hosted Test job passed in 752s with package coverage at 575s. The package phase is slower than the selected ~553s median, and removing the final diagnostic hint produced no measurable benefit.","rollback_reason":"Diagnostic collection is not a material bottleneck; retaining Vitest's actionable isolation/transform hints is preferable without a primary-metric improvement.","next_action_hint":"Restore diagnostics. The remaining dominant cost is actual test/import execution, not reporting or diagnostic bookkeeping."}} {"run":74,"commit":"3d02748","metric":766,"metrics":{"packages_coverage_seconds":587,"examples_seconds":115,"startup_seconds":59,"ci_run_id":32679905499},"status":"keep","description":"Restore and revalidate the selected Test-job configuration","timestamp":1787535900044,"segment":0,"confidence":11.5,"asi":{"hypothesis":"After rejecting several micro-optimizations, restoring the selected all-core vmThreads plus disk-backed Node compile cache configuration would remain stable and materially faster than the 847s baseline.","result":"The exact selected configuration passed in 766s with package coverage at 587s. This is an ordinary slow-runner point but remains 81s (9.6%) below baseline, with all tests, coverage enforcement, examples, services, and clean-tree verification intact.","distribution":"Exact compile-cache candidate runs now span 630–774s and center around roughly 741s; this result is consistent with known hosted-runner variance rather than a regression in configuration.","decision":"Keep the restored minimal candidate. Recent console, diagnostics, tmpfs, example-concurrency, and startup-overlap experiments added complexity without phase-level gains."}} {"run":75,"commit":"dc4efe8","metric":618,"metrics":{"packages_coverage_seconds":443,"examples_seconds":97,"startup_seconds":72,"ci_run_id":32688137929},"status":"keep","description":"Recycle VM workers at a 2 GB heap limit","timestamp":1787544979718,"segment":0,"confidence":11.283018867924529,"asi":{"hypothesis":"Lowering vmThreads' per-worker heap limit from the default ~4 GB to 2 GB would recycle workers before pinned VM module contexts create GC/memory pressure, trading occasional worker restart cost for lower sustained heap pressure.","result":"The full hosted job passed in 618s with package coverage at 443s. This is among the fastest safe runs, but examples were also an unusually fast 97s while startup was slow, indicating strongly favorable execution conditions rather than clean attribution.","mechanism":"Vitest documents that Node pins SourceTextModule VM contexts until vmMemoryLimit triggers worker recycling; with ~998 isolated files, a 2 GB threshold is a plausible workload-level control rather than a benchmark shortcut.","confidence_note":"One favorable-runner sample cannot establish benefit. Retain provisionally because the primary metric improved, but require at least two repeats and reject on any OOM, worker crash, or ordinary-runner regression.","next_action_hint":"Repeat 2 GB vmMemoryLimit unchanged twice; compare package-phase median to the ~553s compile-cache candidate median."}} +{"run":76,"commit":"67da9e1","metric":781,"metrics":{"packages_coverage_seconds":590,"examples_seconds":116,"startup_seconds":68,"ci_run_id":32691359747},"status":"discard","description":"Second 2 GB VM worker recycling sample","timestamp":1787548174898,"segment":0,"confidence":11.5,"asi":{"hypothesis":"A repeat on ordinary runner conditions would show whether the 2 GB VM heap threshold reduces package-test GC pressure consistently.","result":"The full job passed in 781s with package coverage at 590s, materially slower than both the first 443s package-phase outlier and the selected ~553s package median.","rollback_reason":"This sample did not improve the primary metric and shows the first 618s result was dominated by favorable runner conditions.","next_action_hint":"Collect the precommitted third 2 GB sample promised in run #75. Retain only if its three-sample package median beats the selected candidate and all runs remain stable."}} From f91e07342f3bbd46d57764d6e102a454a64672d9 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Mon, 24 Aug 2026 07:03:49 +0000 Subject: [PATCH 79/80] autoresearch: measure Test CI 20260824T070348Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + scripts/coverage-config.test.mjs | 2 +- vitest.config.ts | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index e3f8f2d2e00d..d7e4aafa1bb4 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -75,3 +75,4 @@ {"run":74,"commit":"3d02748","metric":766,"metrics":{"packages_coverage_seconds":587,"examples_seconds":115,"startup_seconds":59,"ci_run_id":32679905499},"status":"keep","description":"Restore and revalidate the selected Test-job configuration","timestamp":1787535900044,"segment":0,"confidence":11.5,"asi":{"hypothesis":"After rejecting several micro-optimizations, restoring the selected all-core vmThreads plus disk-backed Node compile cache configuration would remain stable and materially faster than the 847s baseline.","result":"The exact selected configuration passed in 766s with package coverage at 587s. This is an ordinary slow-runner point but remains 81s (9.6%) below baseline, with all tests, coverage enforcement, examples, services, and clean-tree verification intact.","distribution":"Exact compile-cache candidate runs now span 630–774s and center around roughly 741s; this result is consistent with known hosted-runner variance rather than a regression in configuration.","decision":"Keep the restored minimal candidate. Recent console, diagnostics, tmpfs, example-concurrency, and startup-overlap experiments added complexity without phase-level gains."}} {"run":75,"commit":"dc4efe8","metric":618,"metrics":{"packages_coverage_seconds":443,"examples_seconds":97,"startup_seconds":72,"ci_run_id":32688137929},"status":"keep","description":"Recycle VM workers at a 2 GB heap limit","timestamp":1787544979718,"segment":0,"confidence":11.283018867924529,"asi":{"hypothesis":"Lowering vmThreads' per-worker heap limit from the default ~4 GB to 2 GB would recycle workers before pinned VM module contexts create GC/memory pressure, trading occasional worker restart cost for lower sustained heap pressure.","result":"The full hosted job passed in 618s with package coverage at 443s. This is among the fastest safe runs, but examples were also an unusually fast 97s while startup was slow, indicating strongly favorable execution conditions rather than clean attribution.","mechanism":"Vitest documents that Node pins SourceTextModule VM contexts until vmMemoryLimit triggers worker recycling; with ~998 isolated files, a 2 GB threshold is a plausible workload-level control rather than a benchmark shortcut.","confidence_note":"One favorable-runner sample cannot establish benefit. Retain provisionally because the primary metric improved, but require at least two repeats and reject on any OOM, worker crash, or ordinary-runner regression.","next_action_hint":"Repeat 2 GB vmMemoryLimit unchanged twice; compare package-phase median to the ~553s compile-cache candidate median."}} {"run":76,"commit":"67da9e1","metric":781,"metrics":{"packages_coverage_seconds":590,"examples_seconds":116,"startup_seconds":68,"ci_run_id":32691359747},"status":"discard","description":"Second 2 GB VM worker recycling sample","timestamp":1787548174898,"segment":0,"confidence":11.5,"asi":{"hypothesis":"A repeat on ordinary runner conditions would show whether the 2 GB VM heap threshold reduces package-test GC pressure consistently.","result":"The full job passed in 781s with package coverage at 590s, materially slower than both the first 443s package-phase outlier and the selected ~553s package median.","rollback_reason":"This sample did not improve the primary metric and shows the first 618s result was dominated by favorable runner conditions.","next_action_hint":"Collect the precommitted third 2 GB sample promised in run #75. Retain only if its three-sample package median beats the selected candidate and all runs remain stable."}} +{"run":77,"commit":"405b266","metric":703,"metrics":{"packages_coverage_seconds":520,"examples_seconds":108,"startup_seconds":69,"ci_run_id":32694431383},"status":"keep","description":"Third 2 GB VM worker recycling sample","timestamp":1787552669977,"segment":0,"confidence":11.283018867924529,"asi":{"hypothesis":"A third hosted sample would determine whether earlier worker recycling prevents enough VM-context heap accumulation to improve the median despite runner variance.","result":"The job passed in 703s with package coverage at 520s. Three 2 GB samples are 618/781/703 (median 703s); package phases are 443/590/520 (median 520s).","comparison":"The exact compile-cache candidate without an explicit VM limit has a broader center around ~740s complete and ~556s package coverage after recent samples. The 2 GB medians are roughly 37s lower on both measures, with three clean runs.","decision":"Retain provisionally: the three-sample median now indicates a meaningful benefit, but two samples also had fast examples, so run a contemporaneous default-memory control before finalizing.","next_action_hint":"Remove the explicit vmMemoryLimit for one A/B control while keeping every other selected setting unchanged. Restore 2 GB if the default control is materially slower."}} diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 48e1c41515ab..065d4a3ac184 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -270,7 +270,7 @@ describe('coverage config', () => { assert.doesNotMatch(rootVitestConfig, /from ['"]\.\/scripts\/coverage-config\.[^'"]+['"]/); assert.match(rootVitestConfig, /provider:\s*['"]v8['"]/); assert.match(rootVitestConfig, /reportOnFailure:\s*true/); - assert.match(rootVitestConfig, /vmMemoryLimit: process\.env\['CI'\] \? '2GB' : undefined/); + assert.doesNotMatch(rootVitestConfig, /vmMemoryLimit/); }); it('combines package tests and coverage in one CI job', async () => { diff --git a/vitest.config.ts b/vitest.config.ts index 50f843e92698..31159ecc5cdc 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,7 +11,6 @@ export default defineConfig({ // Stateful projects can override this default, as the Supabase suite does. maxWorkers: process.env['CI'] ? '100%' : undefined, pool: process.env['CI'] ? 'vmThreads' : undefined, - vmMemoryLimit: process.env['CI'] ? '2GB' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process. From 467ed1d672eab74d5a52bca75ebe61451ede6051 Mon Sep 17 00:00:00 2001 From: Steven McClankerton Date: Mon, 24 Aug 2026 07:29:54 +0000 Subject: [PATCH 80/80] autoresearch: measure Test CI 20260824T072953Z Signed-off-by: Steven McClankerton --- .auto/log.jsonl | 1 + scripts/coverage-config.test.mjs | 2 +- vitest.config.ts | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.auto/log.jsonl b/.auto/log.jsonl index d7e4aafa1bb4..0e4dbce954d5 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -76,3 +76,4 @@ {"run":75,"commit":"dc4efe8","metric":618,"metrics":{"packages_coverage_seconds":443,"examples_seconds":97,"startup_seconds":72,"ci_run_id":32688137929},"status":"keep","description":"Recycle VM workers at a 2 GB heap limit","timestamp":1787544979718,"segment":0,"confidence":11.283018867924529,"asi":{"hypothesis":"Lowering vmThreads' per-worker heap limit from the default ~4 GB to 2 GB would recycle workers before pinned VM module contexts create GC/memory pressure, trading occasional worker restart cost for lower sustained heap pressure.","result":"The full hosted job passed in 618s with package coverage at 443s. This is among the fastest safe runs, but examples were also an unusually fast 97s while startup was slow, indicating strongly favorable execution conditions rather than clean attribution.","mechanism":"Vitest documents that Node pins SourceTextModule VM contexts until vmMemoryLimit triggers worker recycling; with ~998 isolated files, a 2 GB threshold is a plausible workload-level control rather than a benchmark shortcut.","confidence_note":"One favorable-runner sample cannot establish benefit. Retain provisionally because the primary metric improved, but require at least two repeats and reject on any OOM, worker crash, or ordinary-runner regression.","next_action_hint":"Repeat 2 GB vmMemoryLimit unchanged twice; compare package-phase median to the ~553s compile-cache candidate median."}} {"run":76,"commit":"67da9e1","metric":781,"metrics":{"packages_coverage_seconds":590,"examples_seconds":116,"startup_seconds":68,"ci_run_id":32691359747},"status":"discard","description":"Second 2 GB VM worker recycling sample","timestamp":1787548174898,"segment":0,"confidence":11.5,"asi":{"hypothesis":"A repeat on ordinary runner conditions would show whether the 2 GB VM heap threshold reduces package-test GC pressure consistently.","result":"The full job passed in 781s with package coverage at 590s, materially slower than both the first 443s package-phase outlier and the selected ~553s package median.","rollback_reason":"This sample did not improve the primary metric and shows the first 618s result was dominated by favorable runner conditions.","next_action_hint":"Collect the precommitted third 2 GB sample promised in run #75. Retain only if its three-sample package median beats the selected candidate and all runs remain stable."}} {"run":77,"commit":"405b266","metric":703,"metrics":{"packages_coverage_seconds":520,"examples_seconds":108,"startup_seconds":69,"ci_run_id":32694431383},"status":"keep","description":"Third 2 GB VM worker recycling sample","timestamp":1787552669977,"segment":0,"confidence":11.283018867924529,"asi":{"hypothesis":"A third hosted sample would determine whether earlier worker recycling prevents enough VM-context heap accumulation to improve the median despite runner variance.","result":"The job passed in 703s with package coverage at 520s. Three 2 GB samples are 618/781/703 (median 703s); package phases are 443/590/520 (median 520s).","comparison":"The exact compile-cache candidate without an explicit VM limit has a broader center around ~740s complete and ~556s package coverage after recent samples. The 2 GB medians are roughly 37s lower on both measures, with three clean runs.","decision":"Retain provisionally: the three-sample median now indicates a meaningful benefit, but two samples also had fast examples, so run a contemporaneous default-memory control before finalizing.","next_action_hint":"Remove the explicit vmMemoryLimit for one A/B control while keeping every other selected setting unchanged. Restore 2 GB if the default control is materially slower."}} +{"run":78,"commit":"f91e073","metric":726,"metrics":{"packages_coverage_seconds":552,"examples_seconds":110,"startup_seconds":59,"ci_run_id":32699891817},"status":"discard","description":"Contemporaneous default VM memory-limit control","timestamp":1787556424720,"segment":0,"confidence":11.074074074074074,"asi":{"hypothesis":"Removing the explicit 2 GB VM heap threshold would reveal whether recent faster runs came from runner conditions rather than earlier worker recycling.","result":"The default-memory control passed in 726s with package coverage at 552s. It is 23s slower complete and 32s slower in package coverage than the three-sample 2 GB medians (703s and 520s), despite normal 110s examples and favorable 59s startup.","rollback_reason":"The contemporaneous default control does not beat the retained 2 GB distribution and aligns with the older default-memory center, strengthening rather than falsifying the recycling hypothesis.","next_action_hint":"Restore the 2 GB vmMemoryLimit and collect one final exact-candidate stability sample before treating it as selected."}} diff --git a/scripts/coverage-config.test.mjs b/scripts/coverage-config.test.mjs index 065d4a3ac184..48e1c41515ab 100644 --- a/scripts/coverage-config.test.mjs +++ b/scripts/coverage-config.test.mjs @@ -270,7 +270,7 @@ describe('coverage config', () => { assert.doesNotMatch(rootVitestConfig, /from ['"]\.\/scripts\/coverage-config\.[^'"]+['"]/); assert.match(rootVitestConfig, /provider:\s*['"]v8['"]/); assert.match(rootVitestConfig, /reportOnFailure:\s*true/); - assert.doesNotMatch(rootVitestConfig, /vmMemoryLimit/); + assert.match(rootVitestConfig, /vmMemoryLimit: process\.env\['CI'\] \? '2GB' : undefined/); }); it('combines package tests and coverage in one CI job', async () => { diff --git a/vitest.config.ts b/vitest.config.ts index 31159ecc5cdc..50f843e92698 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,6 +11,7 @@ export default defineConfig({ // Stateful projects can override this default, as the Supabase suite does. maxWorkers: process.env['CI'] ? '100%' : undefined, pool: process.env['CI'] ? 'vmThreads' : undefined, + vmMemoryLimit: process.env['CI'] ? '2GB' : undefined, // Hard-suppress telemetry across every package test suite. The CLI's // `program.hook('preAction', …)` would otherwise fork the sender // child every time a test invokes the CLI in-process.