From 95c0dafd3f01d4d6d3e6a0c251bc6ae665a2e845 Mon Sep 17 00:00:00 2001 From: MerhiOPS Date: Sun, 15 Mar 2026 19:52:26 +0200 Subject: [PATCH] test(plugins): benchmark cold-start before trust groups --- docs/plugin-startup-benchmark.md | 31 +++ docs/runtime-model.md | 10 + package.json | 1 + scripts/ci/plugin-startup-bench.mjs | 300 ++++++++++++++++++++++++++++ 4 files changed, 342 insertions(+) create mode 100644 docs/plugin-startup-benchmark.md create mode 100644 scripts/ci/plugin-startup-bench.mjs diff --git a/docs/plugin-startup-benchmark.md b/docs/plugin-startup-benchmark.md new file mode 100644 index 0000000..a490aa4 --- /dev/null +++ b/docs/plugin-startup-benchmark.md @@ -0,0 +1,31 @@ +# Plugin Startup Benchmark + +This benchmark measures the current cold-start cost of the plugin host before introducing trust groups or shared-process pooling. + +## Method + +1. Build the plugin host in release mode with `cargo build -p volt-plugin-host --release`. +2. Run `node scripts/ci/plugin-startup-bench.mjs`. +3. The harness creates a minimal plugin that: + - imports `definePlugin` from `volt:plugin` + - registers a single `ping` command during `activate` +4. Each iteration measures: + - process spawn to `ready` signal + - process spawn to `activate` plus the first `plugin:invoke-command` response +5. The benchmark host answers the plugin's `plugin:register-command` request so the measurement uses the real stdio protocol and activation path. + +## Current Result + +Measured on the Session 6 development machine on March 15, 2026 with 10 iterations: + +| Metric | Min | Median | Max | P95 | +| --- | ---: | ---: | ---: | ---: | +| Spawn -> ready | 9.79 ms | 15.48 ms | 16.25 ms | 16.25 ms | +| Spawn -> first command response | 13.97 ms | 17.42 ms | 18.08 ms | 18.08 ms | + +Decision threshold: + +- spawn -> ready must stay under `200 ms` +- spawn -> first command response must stay under `500 ms` + +Current measurements are well below both thresholds, so trust groups remain deferred. The plugin system keeps one-process-per-plugin isolation by default, and shared-process pooling stays an escape hatch until a later benchmark shows a real need. diff --git a/docs/runtime-model.md b/docs/runtime-model.md index fb2c7cd..f3f1dec 100644 --- a/docs/runtime-model.md +++ b/docs/runtime-model.md @@ -94,6 +94,16 @@ structurally consistent after create/close/quit transitions. - failed command sends - Dropped callback dispatch counter in N-API layer +## Plugin Startup Benchmark + +- Session 6 measured plugin cold-start before adding trust groups or process pooling. +- The benchmark uses the real release `volt-plugin-host` binary plus the framed stdio protocol and activation path. +- Current result on the development machine: + - spawn -> ready: 15.48 ms median, 16.25 ms p95 + - spawn -> first command response: 17.42 ms median, 18.08 ms p95 +- Those numbers are far below the Session 6 decision gate of 200 ms and 500 ms, so shared-process trust groups remain deferred instead of becoming a co-equal execution model. +- Re-run with `cargo build -p volt-plugin-host --release` and `node scripts/ci/plugin-startup-bench.mjs`. + ## Known Architectural Limits in v0.1 diff --git a/package.json b/package.json index a06feab..3632aa6 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "test:benchmarks:release": "node scripts/ci/volt-benchmarks.mjs --release", "test:benchmarks:sweep": "node scripts/ci/volt-benchmarks.mjs --sweep", "test:benchmarks:sweep:release": "node scripts/ci/volt-benchmarks.mjs --sweep --release", + "test:plugin-startup-bench": "node scripts/ci/plugin-startup-bench.mjs", "test:soak": "node scripts/soak/nightly-soak.mjs", "test:contract-compat": "node scripts/contracts/check-compatibility.mjs", "check:loc": "node scripts/ci/loc-guard.mjs", diff --git a/scripts/ci/plugin-startup-bench.mjs b/scripts/ci/plugin-startup-bench.mjs new file mode 100644 index 0000000..986fdf3 --- /dev/null +++ b/scripts/ci/plugin-startup-bench.mjs @@ -0,0 +1,300 @@ +import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { spawn } from 'node:child_process'; + +const repoRoot = path.resolve(import.meta.dirname, '..', '..'); +const defaultIterations = 10; +const spawnReadyThresholdMs = 200, firstResponseThresholdMs = 500; + +function parseIterations(argv) { + const flag = argv.find((arg) => arg.startsWith('--iterations=')); + if (!flag) { + return defaultIterations; + } + const value = Number.parseInt(flag.split('=')[1] ?? '', 10); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`invalid --iterations value: ${flag}`); + } + return value; +} + +function resolvePluginHostBinary() { + const binaryName = process.platform === 'win32' ? 'volt-plugin-host.exe' : 'volt-plugin-host'; + return path.join(repoRoot, 'target', 'release', binaryName); +} + +function encodeMessage(message) { + const body = Buffer.from(`${JSON.stringify(message)}\n`, 'utf8'); + const header = Buffer.alloc(4); + header.writeUInt32LE(body.length, 0); + return Buffer.concat([header, body]); +} + +function createFrameReader(onMessage) { + let buffer = Buffer.alloc(0); + return (chunk) => { + buffer = Buffer.concat([buffer, chunk]); + while (buffer.length >= 4) { + const length = buffer.readUInt32LE(0); + if (buffer.length < 4 + length) { + return; + } + const body = buffer.subarray(4, 4 + length).toString('utf8').trimEnd(); + buffer = buffer.subarray(4 + length); + onMessage(JSON.parse(body)); + } + }; +} + +function median(values) { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; +} + +function percentile95(values) { + const sorted = [...values].sort((a, b) => a - b); + const index = Math.ceil(0.95 * sorted.length) - 1; + return sorted[Math.max(0, Math.min(index, sorted.length - 1))]; +} + +function buildSummary(values) { + return { + min: Math.min(...values), + median: median(values), + max: Math.max(...values), + p95: percentile95(values), + }; +} + +function createMinimalPlugin(tempRoot) { + const backendEntry = path.join(tempRoot, 'plugin.js'); + const dataRoot = path.join(tempRoot, 'data'); + mkdirSync(dataRoot, { recursive: true }); + writeFileSync( + backendEntry, + [ + "import { definePlugin } from 'volt:plugin';", + 'definePlugin({', + ' async activate(context) {', + " context.commands.register('ping', async (args) => ({ ok: true, echoed: args ?? null }));", + ' },', + ' async deactivate() {}', + '});', + '', + ].join('\n'), + 'utf8', + ); + return { backendEntry, dataRoot }; +} + +function waitForExit(child) { + return new Promise((resolve, reject) => { + child.once('exit', (code, signal) => resolve({ code, signal })); + child.once('error', reject); + }); +} + +async function runIteration(binary, iteration) { + const tempRoot = mkdtempSync(path.join(os.tmpdir(), 'volt-plugin-bench-')); + const { backendEntry, dataRoot } = createMinimalPlugin(tempRoot); + const pluginId = `bench.plugin.${iteration}`; + const config = { + pluginId, + backendEntry, + manifest: { + id: pluginId, + name: 'Bench Plugin', + version: '0.0.0', + apiVersion: 1, + engine: { volt: '>=0.1.0' }, + backend: './plugin.js', + capabilities: [], + }, + capabilities: [], + dataRoot, + delegatedGrants: [], + hostIpcSettings: { + heartbeatIntervalMs: 5000, + heartbeatTimeoutMs: 3000, + callTimeoutMs: 30000, + maxInflight: 64, + maxQueueDepth: 256, + }, + }; + + const child = spawn( + binary, + ['--plugin', '--config', Buffer.from(JSON.stringify(config), 'utf8').toString('base64')], + { cwd: repoRoot, stdio: ['pipe', 'pipe', 'pipe'] }, + ); + + const start = performance.now(); + let stderr = ''; + const pending = new Map(); + let readyResolve; + let readyReject; + const readyPromise = new Promise((resolve, reject) => { + readyResolve = resolve; + readyReject = reject; + }); + + child.stderr.on('data', (chunk) => { + stderr += chunk.toString('utf8'); + }); + child.once('error', readyReject); + child.once('exit', (code, signal) => { + const error = new Error(`plugin host exited early code=${code} signal=${signal} stderr=${stderr}`); + readyReject(error); + for (const waiter of pending.values()) { + waiter.reject(error); + } + pending.clear(); + }); + + child.stdout.on('data', createFrameReader((message) => { + if (message.type === 'signal' && message.method === 'ready') { + readyResolve({ time: performance.now() }); + return; + } + if (message.type === 'request' && message.method === 'plugin:register-command') { + child.stdin.write( + encodeMessage({ + type: 'response', + id: message.id, + method: message.method, + payload: null, + error: null, + }), + ); + return; + } + const waiter = pending.get(message.id); + if (!waiter) { + return; + } + pending.delete(message.id); + waiter.resolve({ message, time: performance.now() }); + })); + + function sendAndWait(message, timeoutMs = 30000) { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + if (pending.delete(message.id)) { + reject(new Error(`timeout waiting for ${message.method}; stderr=${stderr}`)); + } + }, timeoutMs); + pending.set(message.id, { + resolve: (value) => { + clearTimeout(timeout); + resolve(value); + }, + reject: (error) => { + clearTimeout(timeout); + reject(error); + }, + }); + child.stdin.write(encodeMessage(message), (error) => { + if (!error) { + return; + } + clearTimeout(timeout); + pending.delete(message.id); + reject(error); + }); + }); + } + + const readyTimeout = setTimeout(() => { + readyReject(new Error(`timeout waiting for ready; stderr=${stderr}`)); + child.kill(); + }, 30000); + + try { + const ready = await readyPromise; + clearTimeout(readyTimeout); + + const activate = await sendAndWait({ + type: 'signal', + id: 'activate-1', + method: 'activate', + payload: null, + error: null, + }); + const command = await sendAndWait({ + type: 'request', + id: 'command-1', + method: 'plugin:invoke-command', + payload: { id: 'ping', args: { hello: 'world' } }, + error: null, + }); + const exitPromise = waitForExit(child); + child.stdin.write( + encodeMessage({ + type: 'signal', + id: 'deactivate-1', + method: 'deactivate', + payload: null, + error: null, + }), + ); + const exit = await exitPromise; + if (activate.message.error || command.message.error || exit.code !== 0) { + throw new Error(`benchmark iteration failed stderr=${stderr}`); + } + + return { + spawnToReadyMs: ready.time - start, + spawnToFirstResponseMs: command.time - start, + readyToActivateAckMs: activate.time - ready.time, + activateToCommandResponseMs: command.time - activate.time, + }; + } finally { + clearTimeout(readyTimeout); + child.kill(); + rmSync(tempRoot, { recursive: true, force: true }); + } +} + +async function main() { + const iterations = parseIterations(process.argv.slice(2)); + const binary = resolvePluginHostBinary(); + if (!existsSync(binary)) { + throw new Error(`missing release plugin host binary: ${binary}`); + } + const results = []; + + for (let index = 0; index < iterations; index += 1) { + const result = await runIteration(binary, index + 1); + results.push(result); + console.log( + `iteration ${index + 1}: spawn->ready=${result.spawnToReadyMs.toFixed(2)}ms ` + + `spawn->first-response=${result.spawnToFirstResponseMs.toFixed(2)}ms`, + ); + } + + const readyValues = results.map((result) => result.spawnToReadyMs); + const firstResponseValues = results.map((result) => result.spawnToFirstResponseMs); + const readySummary = buildSummary(readyValues); + const firstResponseSummary = buildSummary(firstResponseValues); + const summary = { + thresholds: { + spawnToReadyMs: spawnReadyThresholdMs, + spawnToFirstResponseMs: firstResponseThresholdMs, + }, + spawnToReadyMs: readySummary, + spawnToFirstResponseMs: firstResponseSummary, + trustGroupsRecommended: + readySummary.p95 > spawnReadyThresholdMs || + firstResponseSummary.p95 > firstResponseThresholdMs, + iterations: results, + }; + + console.log(JSON.stringify(summary, null, 2)); +} + +await main();