From 31793419c3d1f74c1bb7c50cf1e04fcf55504bc8 Mon Sep 17 00:00:00 2001 From: Drake Costa Date: Wed, 3 Jun 2026 14:27:42 -0700 Subject: [PATCH] feat: Add Vitest bench suite + CPU profile script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Iteration-time performance tooling — the second of the three concerns from the perf-architecture discussion. Answers "is this change actually faster?" via Vitest's built-in `bench()` mode (Tinybench under the hood), and provides a one-command CPU profiler wrapper for finding hotspots. ## What's new **Four `.bench.ts` files** in `src/__benchmarks__/`, one per fixture schema. Each measures ops/sec for `mock(schema)` via `bench()`. Baseline numbers at v1.5.4 (Node 26, AMD Ryzen 5): | Schema | ops/sec | RME | | -------------------------------------------- | ------: | -----: | | `mock(messageLikeSchema)` recursive-lazy | 3,188 | ±3.68% | | `mock(applicationLikeSchema)` wide-flat | 13,147 | ±2.28% | | `mock(channelLikeSchema)` intersect+variant | 27,419 | ±2.47% | | `mock(interactionLikeSchema)` multi-lazy | 25,933 | ±2.39% | These are within-process ratios — absolute numbers vary by hardware, but ratios between two runs in the same session are stable for comparing baseline vs feature-branch perf. **Vitest discovery configured** via a new `test.benchmark` block in `vite.config.ts` so `.bench.ts` files are picked up by `vp test bench` without polluting the default `vp test` run. The bench suite is explicitly opt-in — it doesn't gate CI. **`scripts/profile.mjs` convenience wrapper** runs the bench suite under `node --cpu-prof` and drops the resulting `.cpuprofile` in `./profiles/` (gitignored). Prints instructions for opening it in Chrome DevTools. Invoke via `vp run profile`. **`src/__benchmarks__/README.md`** documents how to use the suite, how to interpret ops/sec and RME, and how to add new benchmarks. The "5% threshold" guidance there is the practical rule for distinguishing real regressions from noise. ## What's NOT in this PR This is the iteration-time perf tooling. The catastrophe-detection canary (asserting call-count ceilings on the same fixtures) shipped in #77 — they share the fixture library deliberately. CPU profile output viewing is left as a manual step (open in Chrome DevTools). A separate flame-graph generator could be wired in later via the npm script if anyone wants the on-CLI variant; for now the manual workflow is what the original perf report used. Co-Authored-By: Claude Opus 4.7 --- .bumpy/perf-bench-suite.md | 5 ++ .gitignore | 4 ++ package.json | 4 +- scripts/profile.mjs | 70 +++++++++++++++++++ src/__benchmarks__/README.md | 66 +++++++++++++++++ .../applicationLikeSchema.bench.ts | 19 +++++ src/__benchmarks__/channelLikeSchema.bench.ts | 18 +++++ .../interactionLikeSchema.bench.ts | 18 +++++ src/__benchmarks__/messageLikeSchema.bench.ts | 21 ++++++ vite.config.ts | 11 ++- 10 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 .bumpy/perf-bench-suite.md create mode 100644 scripts/profile.mjs create mode 100644 src/__benchmarks__/README.md create mode 100644 src/__benchmarks__/applicationLikeSchema.bench.ts create mode 100644 src/__benchmarks__/channelLikeSchema.bench.ts create mode 100644 src/__benchmarks__/interactionLikeSchema.bench.ts create mode 100644 src/__benchmarks__/messageLikeSchema.bench.ts diff --git a/.bumpy/perf-bench-suite.md b/.bumpy/perf-bench-suite.md new file mode 100644 index 0000000..49ee4f4 --- /dev/null +++ b/.bumpy/perf-bench-suite.md @@ -0,0 +1,5 @@ +--- +valimock: patch +--- + +Add Vitest `bench()` files for iteration-time performance evaluation, reusing the fixture library from the regression canary. Includes a `scripts/profile.mjs` convenience wrapper for CPU profiling. Pure tooling — no public-API or runtime changes. diff --git a/.gitignore b/.gitignore index bf32a7e..523631e 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,7 @@ cypress/e2e/build # Claude Code local agent state .claude/ + +# CPU profiles emitted by scripts/profile.mjs +profiles/ +*.cpuprofile diff --git a/package.json b/package.json index 0cca7de..4c16ce2 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,9 @@ "check": "vp check", "format": "vp check --fix", "test": "vp test --fileParallelism", - "typecheck": "vp typecheck" + "typecheck": "vp typecheck", + "bench": "vp test bench --run", + "profile": "node scripts/profile.mjs" }, "dependencies": { "randexp": "^0.5.3" diff --git a/scripts/profile.mjs b/scripts/profile.mjs new file mode 100644 index 0000000..5b45357 --- /dev/null +++ b/scripts/profile.mjs @@ -0,0 +1,70 @@ +#!/usr/bin/env node +/** + * Run the Valimock benchmark suite under node's V8 CPU profiler and drop a + * `.cpuprofile` file in `./profiles/` for analysis in Chrome DevTools. + * + * Usage: + * vp run profile # all benchmarks + * vp run profile -- messageLikeSchema # filter to one bench file + * + * Open the resulting file in Chrome: + * 1. devtools → ⋮ menu → More tools → Performance insights + * 2. drag the .cpuprofile file into the panel + * + * The Self Time column surfaces hotspots; ranked self-time is what surfaced + * `findFakerForKeyName` as the top regression cause in the original perf + * report. + */ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readdirSync, statSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repoRoot = resolve(fileURLToPath(import.meta.url), `../..`); +const profilesDir = join(repoRoot, `profiles`); + +if (!existsSync(profilesDir)) mkdirSync(profilesDir); + +console.log(`>> Running benchmarks under --cpu-prof, output dir: ./profiles/`); +console.log(`>> Bench filter: ${process.argv.slice(2).join(` `) || `(all)`}`); + +// Use Vitest's CLI binary directly with --cpu-prof. We can't use `vp test bench` +// here because the --cpu-prof flag has to be passed to the actual node process, +// not the vp wrapper. +const vitestBin = join(repoRoot, `node_modules/vitest/dist/cli.js`); +if (!existsSync(vitestBin)) { + console.error(`error: vitest CLI not found at ${vitestBin}. Run \`vp install\` first.`); + process.exit(1); +} + +const result = spawnSync( + process.execPath, + [`--cpu-prof`, `--cpu-prof-dir=${profilesDir}`, vitestBin, `bench`, `--run`, ...process.argv.slice(2)], + { stdio: `inherit`, cwd: repoRoot } +); + +if (result.status !== 0) { + console.error(`>> Bench run exited with status ${result.status}`); + process.exit(result.status ?? 1); +} + +// Find the most recently-written .cpuprofile in profiles/ and surface its path. +const cpuProfiles = readdirSync(profilesDir) + .filter((f) => f.endsWith(`.cpuprofile`)) + .map((f) => ({ name: f, mtime: statSync(join(profilesDir, f)).mtimeMs })) + .sort((a, b) => b.mtime - a.mtime); + +if (cpuProfiles.length === 0) { + console.warn(`>> No .cpuprofile files written — check that vitest actually ran.`); + process.exit(1); +} + +const latest = cpuProfiles[0]; +console.log(``); +console.log(`>> Wrote: ./profiles/${latest.name}`); +console.log(`>> Open in Chrome DevTools:`); +console.log(` devtools → ⋮ menu → More tools → Performance insights`); +console.log(` drag ./profiles/${latest.name} into the panel`); +console.log(``); +console.log(`>> Or inspect via the CLI:`); +console.log(` npx -y v8-cpu-prof-flamegraph ./profiles/${latest.name}`); diff --git a/src/__benchmarks__/README.md b/src/__benchmarks__/README.md new file mode 100644 index 0000000..e2301ca --- /dev/null +++ b/src/__benchmarks__/README.md @@ -0,0 +1,66 @@ +# Performance benchmarks + +This directory holds Vitest `bench()` files for iteration-time performance evaluation. They are **not run by default** — they're tools for contributors evaluating a perf change, not catastrophe-detection canaries. + +For the regression canary that runs in normal CI, see [`src/__tests__/perfRegression.spec.ts`](../__tests__/perfRegression.spec.ts). + +## Running + +```bash +# Run all benchmarks +vp test bench + +# Run a single bench file +vp test bench messageLikeSchema + +# Run with a custom number of iterations (default: tinybench-driven) +vp test bench --benchmark.iterations=200 +``` + +Output looks like: + +``` + BENCH Summary + mock(messageLikeSchema) > default options + 3,420 ops/sec ±0.42% (456 samples) +``` + +`ops/sec` is the headline number. Higher is faster. The `±0.42%` is the relative margin of error — anything under ~1% is stable enough to compare runs. + +## How to evaluate a perf change + +1. Run the bench on `main` (or whatever baseline you're comparing against) and save the output. +2. Switch to your branch, run the bench again. +3. Compare ops/sec across runs **on the same machine, in the same session**. Absolute numbers vary by hardware and load — ratios within a session are stable. + +A change that drops ops/sec by >5% (well above the RME) is a regression. A change that improves it by >5% is a win. Anything in the middle is noise and shouldn't drive decisions. + +## What's here + +| File | Fixture | Pattern | +| -------------------------------- | ----------------------------------- | --------------------------------------- | +| `messageLikeSchema.bench.ts` | recursive lazy through nullish | the v1.5.0 wrapper-recursion shape | +| `channelLikeSchema.bench.ts` | `intersect([common, variant(...)])` | discriminated unions with shared fields | +| `applicationLikeSchema.bench.ts` | wide-flat object | string-pipeline-dominated, no recursion | +| `interactionLikeSchema.bench.ts` | multi-`v.lazy()` union | cross-schema references | + +The fixture schemas themselves live in [`fixtures/`](./fixtures/) and are shared with the regression canary spec. + +## Adding a new benchmark + +A new bench file should: + +1. Live in this directory with a `.bench.ts` suffix (Vitest's discovery glob). +2. Import only from `fixtures/` and Valimock's public exports. No discordkit imports, no network, no filesystem. +3. Wrap each measurement in a `describe()` block named after the schema being mocked. Multiple `bench()` calls inside the same `describe` get side-by-side comparison in the reporter — use that to compare configurations of the same workload. +4. Construct the `Valimock` instance _outside_ the `bench()` callback when possible. Constructor cost is real (Map + Set initialization) but is a one-time hit, not a per-mock cost — including it in the timing measurement would obscure the actual mock cost. + +## Profiling for hotspot discovery + +The bench suite tells you _whether_ a change is faster. To find out _where_ the time is going, use the CPU profiler: + +```bash +node --cpu-prof --cpu-prof-dir=./profiles node_modules/vitest/dist/cli.js bench --run +``` + +Open the resulting `.cpuprofile` in Chrome DevTools → Performance → "Load Profile". Self-time columns surface the top hotspots; that's how the original perf report identified `findFakerForKeyName` as 21.4% of CPU time. diff --git a/src/__benchmarks__/applicationLikeSchema.bench.ts b/src/__benchmarks__/applicationLikeSchema.bench.ts new file mode 100644 index 0000000..364bc51 --- /dev/null +++ b/src/__benchmarks__/applicationLikeSchema.bench.ts @@ -0,0 +1,19 @@ +import { bench, describe } from "vite-plus/test"; +import { Valimock } from "../Valimock.js"; +import { applicationLikeSchema } from "./fixtures/index.js"; + +/** + * Benchmark: wide-flat object, no recursion. + * + * Dominated by string-pipeline cost (keyNameGenerators, format generators, + * the findFakerForKeyName cache, etc.). Useful as the canonical + * "no-recursion" comparison case — if THIS bench regresses, the cause + * is in the string pipeline or the dispatch loop, not a wrapper handler. + */ +describe(`mock(applicationLikeSchema)`, () => { + const m = new Valimock({ onWarn: () => {} }); + + bench(`default options`, () => { + m.mock(applicationLikeSchema); + }); +}); diff --git a/src/__benchmarks__/channelLikeSchema.bench.ts b/src/__benchmarks__/channelLikeSchema.bench.ts new file mode 100644 index 0000000..c902d84 --- /dev/null +++ b/src/__benchmarks__/channelLikeSchema.bench.ts @@ -0,0 +1,18 @@ +import { bench, describe } from "vite-plus/test"; +import { Valimock } from "../Valimock.js"; +import { channelLikeSchema } from "./fixtures/index.js"; + +/** + * Benchmark: `intersect([common, variant(...)])` discriminated unions. + * + * Exercises the intersect deepMerge path (the v1.5.1 variant-discriminator + * regression) and variant resolution. No recursion — useful as a control + * for the difference between shape-driven and recursion-driven cost. + */ +describe(`mock(channelLikeSchema)`, () => { + const m = new Valimock({ onWarn: () => {} }); + + bench(`default options`, () => { + m.mock(channelLikeSchema); + }); +}); diff --git a/src/__benchmarks__/interactionLikeSchema.bench.ts b/src/__benchmarks__/interactionLikeSchema.bench.ts new file mode 100644 index 0000000..163c9e6 --- /dev/null +++ b/src/__benchmarks__/interactionLikeSchema.bench.ts @@ -0,0 +1,18 @@ +import { bench, describe } from "vite-plus/test"; +import { Valimock } from "../Valimock.js"; +import { interactionLikeSchema } from "./fixtures/index.js"; + +/** + * Benchmark: multi-`v.lazy()` union with cross-schema references. + * + * Exercises lazy resolution interleaved with discriminated union picking + * and nested nullish wrappers. Closest match to discordkit's Interaction + * schema, which mixes all three patterns. + */ +describe(`mock(interactionLikeSchema)`, () => { + const m = new Valimock({ onWarn: () => {} }); + + bench(`default options`, () => { + m.mock(interactionLikeSchema); + }); +}); diff --git a/src/__benchmarks__/messageLikeSchema.bench.ts b/src/__benchmarks__/messageLikeSchema.bench.ts new file mode 100644 index 0000000..2dd88c5 --- /dev/null +++ b/src/__benchmarks__/messageLikeSchema.bench.ts @@ -0,0 +1,21 @@ +import { bench, describe } from "vite-plus/test"; +import { Valimock } from "../Valimock.js"; +import { messageLikeSchema } from "./fixtures/index.js"; + +/** + * Benchmark: the recursive-lazy-through-nullish pattern. + * + * messageLikeSchema mirrors discordkit's Message schema — the shape that + * drove the v1.5.0 regression. Healthy ops/sec is in the low thousands; + * the v1.5.2 eager-recursion bug dropped it to ~38 ops/sec. + * + * Run all benches: vp test bench + * Run only this file: vp test bench messageLikeSchema + */ +describe(`mock(messageLikeSchema)`, () => { + const m = new Valimock({ onWarn: () => {} }); + + bench(`default options`, () => { + m.mock(messageLikeSchema); + }); +}); diff --git a/vite.config.ts b/vite.config.ts index 1e218f4..c573116 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -643,6 +643,15 @@ export default defineConfig({ include: ["**/*.{test,spec}.{ts,tsx}"], exclude: ["**/node_modules/**", "**/dist/**"], environment: "node", - passWithNoTests: true + passWithNoTests: true, + // Benchmark suite (run via `vp test bench`). Files live under + // src/__benchmarks__/ and reuse the fixture library there. Kept out of + // the default `vp test` run so the iteration-time bench suite doesn't + // gate CI — it answers "is this change actually faster?" not "did + // anything break". + benchmark: { + include: ["src/__benchmarks__/**/*.bench.{ts,tsx}"], + exclude: ["**/node_modules/**", "**/dist/**"] + } } });