From 1d02adeaf9fdf0814d6d909833328825c127f8c1 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Mon, 2 Mar 2026 10:47:13 +0100 Subject: [PATCH 01/38] Consolidate view comoser --- src/BladeService.php | 19 ------------------ src/BlazeServiceProvider.php | 39 ++++++++++++++---------------------- 2 files changed, 15 insertions(+), 43 deletions(-) diff --git a/src/BladeService.php b/src/BladeService.php index 746f8060..5f7c35f2 100644 --- a/src/BladeService.php +++ b/src/BladeService.php @@ -3,7 +3,6 @@ namespace Livewire\Blaze; use Illuminate\Support\Arr; -use Illuminate\Support\Facades\Event; use Illuminate\Support\Facades\File; use Illuminate\Support\Str; use Illuminate\View\Compilers\ComponentTagCompiler; @@ -354,24 +353,6 @@ public static function stripQuotes(string $input): string return (new ComponentTagCompiler(blade: app('blade.compiler')))->stripQuotes($input); } - /** - * Register a callback to intercept view cache invalidation events. - */ - public static function viewCacheInvalidationHook(callable $callback): void - { - Event::listen('composing:*', function ($event, $params) use ($callback) { - $view = $params[0]; - - if (! $view instanceof \Illuminate\View\View) { - return; - } - - $invalidate = fn () => app('blade.compiler')->compile($view->getPath()); - - $callback($view, $invalidate); - }); - } - /** * Resolve a component name to its file path using registered anonymous component paths. */ diff --git a/src/BlazeServiceProvider.php b/src/BlazeServiceProvider.php index 35036f98..dd77e780 100644 --- a/src/BlazeServiceProvider.php +++ b/src/BlazeServiceProvider.php @@ -40,25 +40,32 @@ protected function registerConfig(): void public function boot(): void { $this->registerBlazeDirectives(); - $this->registerBlazeRuntime(); + $this->registerViewComposer(); $this->registerBladeMacros(); - $this->interceptViewCacheInvalidation(); $this->interceptBladeCompilation(); $this->registerDebuggerMiddleware(); } /** - * Make the BlazeRuntime instance available to Blade views. + * Register the view composer that handles cache invalidation + * and makes the BlazeRuntime instance available to Blade views. */ - protected function registerBlazeRuntime(): void + protected function registerViewComposer(): void { - View::composer('*', function (\Illuminate\View\View $view) { - if (Blaze::isDisabled() && ! Blaze::isDebugging()) { + $blaze = $this->app->make(BlazeManager::class); + $runtime = $this->app->make(BlazeRuntime::class); + + View::composer('*', function (\Illuminate\View\View $view) use ($blaze, $runtime) { + if (! str_ends_with($view->getPath(), '.blade.php')) { return; } - if (str_ends_with($view->getPath(), '.blade.php')) { - $view->with('__blaze', $this->app->make(BlazeRuntime::class)); + if ($blaze->isEnabled() && $blaze->viewContainsExpiredFrontMatter($view)) { + $view->getEngine()->getCompiler()->compile($view->getPath()); + } + + if ($blaze->isEnabled() || $blaze->isDebugging()) { + $view->with('__blaze', $runtime); } }); } @@ -125,22 +132,6 @@ protected function interceptBladeCompilation(): void }); } - /** - * Recompile views when folded component dependencies have changed. - */ - protected function interceptViewCacheInvalidation(): void - { - BladeService::viewCacheInvalidationHook(function ($view, $invalidate) { - if (Blaze::isDisabled()) { - return; - } - - if (Blaze::viewContainsExpiredFrontMatter($view)) { - $invalidate(); - } - }); - } - /** * Register the Debugger middleware. */ From 38f30aa3329c3dd50e837a35c8e8a18179d927b1 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Mon, 2 Mar 2026 10:58:22 +0100 Subject: [PATCH 02/38] Revert "Consolidate view comoser" This reverts commit 1d02adeaf9fdf0814d6d909833328825c127f8c1. --- src/BladeService.php | 19 ++++++++++++++++++ src/BlazeServiceProvider.php | 39 ++++++++++++++++++++++-------------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/src/BladeService.php b/src/BladeService.php index 5f7c35f2..746f8060 100644 --- a/src/BladeService.php +++ b/src/BladeService.php @@ -3,6 +3,7 @@ namespace Livewire\Blaze; use Illuminate\Support\Arr; +use Illuminate\Support\Facades\Event; use Illuminate\Support\Facades\File; use Illuminate\Support\Str; use Illuminate\View\Compilers\ComponentTagCompiler; @@ -353,6 +354,24 @@ public static function stripQuotes(string $input): string return (new ComponentTagCompiler(blade: app('blade.compiler')))->stripQuotes($input); } + /** + * Register a callback to intercept view cache invalidation events. + */ + public static function viewCacheInvalidationHook(callable $callback): void + { + Event::listen('composing:*', function ($event, $params) use ($callback) { + $view = $params[0]; + + if (! $view instanceof \Illuminate\View\View) { + return; + } + + $invalidate = fn () => app('blade.compiler')->compile($view->getPath()); + + $callback($view, $invalidate); + }); + } + /** * Resolve a component name to its file path using registered anonymous component paths. */ diff --git a/src/BlazeServiceProvider.php b/src/BlazeServiceProvider.php index dd77e780..35036f98 100644 --- a/src/BlazeServiceProvider.php +++ b/src/BlazeServiceProvider.php @@ -40,32 +40,25 @@ protected function registerConfig(): void public function boot(): void { $this->registerBlazeDirectives(); - $this->registerViewComposer(); + $this->registerBlazeRuntime(); $this->registerBladeMacros(); + $this->interceptViewCacheInvalidation(); $this->interceptBladeCompilation(); $this->registerDebuggerMiddleware(); } /** - * Register the view composer that handles cache invalidation - * and makes the BlazeRuntime instance available to Blade views. + * Make the BlazeRuntime instance available to Blade views. */ - protected function registerViewComposer(): void + protected function registerBlazeRuntime(): void { - $blaze = $this->app->make(BlazeManager::class); - $runtime = $this->app->make(BlazeRuntime::class); - - View::composer('*', function (\Illuminate\View\View $view) use ($blaze, $runtime) { - if (! str_ends_with($view->getPath(), '.blade.php')) { + View::composer('*', function (\Illuminate\View\View $view) { + if (Blaze::isDisabled() && ! Blaze::isDebugging()) { return; } - if ($blaze->isEnabled() && $blaze->viewContainsExpiredFrontMatter($view)) { - $view->getEngine()->getCompiler()->compile($view->getPath()); - } - - if ($blaze->isEnabled() || $blaze->isDebugging()) { - $view->with('__blaze', $runtime); + if (str_ends_with($view->getPath(), '.blade.php')) { + $view->with('__blaze', $this->app->make(BlazeRuntime::class)); } }); } @@ -132,6 +125,22 @@ protected function interceptBladeCompilation(): void }); } + /** + * Recompile views when folded component dependencies have changed. + */ + protected function interceptViewCacheInvalidation(): void + { + BladeService::viewCacheInvalidationHook(function ($view, $invalidate) { + if (Blaze::isDisabled()) { + return; + } + + if (Blaze::viewContainsExpiredFrontMatter($view)) { + $invalidate(); + } + }); + } + /** * Register the Debugger middleware. */ From cc20bc85b5aa35205d4cbf070fbb7c72734b4641 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Mon, 2 Mar 2026 10:59:21 +0100 Subject: [PATCH 03/38] Access errors via Arr::get() --- src/Runtime/BlazeRuntime.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runtime/BlazeRuntime.php b/src/Runtime/BlazeRuntime.php index cc82112a..57ced054 100644 --- a/src/Runtime/BlazeRuntime.php +++ b/src/Runtime/BlazeRuntime.php @@ -213,7 +213,7 @@ private function getCompiledPath(): string public function __get(string $name): mixed { return match ($name) { - 'errors' => $this->env->getShared()['errors'] ?? new ViewErrorBag, + 'errors' => $this->env->shared('errors') ?? new ViewErrorBag, 'compiledPath' => $this->getCompiledPath(), default => throw new \InvalidArgumentException("Property {$name} does not exist"), }; From 1ff548e6d58f2cce76ffd1740ee5a8bf378124cc Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Mon, 2 Mar 2026 11:03:57 +0100 Subject: [PATCH 04/38] Revert "Access errors via Arr::get()" This reverts commit cc20bc85b5aa35205d4cbf070fbb7c72734b4641. --- src/Runtime/BlazeRuntime.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runtime/BlazeRuntime.php b/src/Runtime/BlazeRuntime.php index 57ced054..cc82112a 100644 --- a/src/Runtime/BlazeRuntime.php +++ b/src/Runtime/BlazeRuntime.php @@ -213,7 +213,7 @@ private function getCompiledPath(): string public function __get(string $name): mixed { return match ($name) { - 'errors' => $this->env->shared('errors') ?? new ViewErrorBag, + 'errors' => $this->env->getShared()['errors'] ?? new ViewErrorBag, 'compiledPath' => $this->getCompiledPath(), default => throw new \InvalidArgumentException("Property {$name} does not exist"), }; From 90eef7e7707d4543d124f2febb20e6e322e28790 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Mon, 2 Mar 2026 11:04:01 +0100 Subject: [PATCH 05/38] Update Wrapper.php --- src/Compiler/Wrapper.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Compiler/Wrapper.php b/src/Compiler/Wrapper.php index 8254c287..c8faced5 100644 --- a/src/Compiler/Wrapper.php +++ b/src/Compiler/Wrapper.php @@ -58,6 +58,8 @@ public function wrap(string $compiled, string $path, ?string $source = null): st $output .= $this->globalVariables($source, $compiled); $output .= 'if (($__data[\'attributes\'] ?? null) instanceof \Illuminate\View\ComponentAttributeBag) { $__data = $__data + $__data[\'attributes\']->all(); unset($__data[\'attributes\']); }'."\n"; $output .= '$attributes = \\Livewire\\Blaze\\Runtime\\BlazeAttributeBag::sanitized($__data, $__bound);'."\n"; + $output .= '$attributes = \\Livewire\\Blaze\\Runtime\\BlazeAttributeBag::sanitized($__data, $__bound);'."\n"; + $output .= '$attributes = \\Livewire\\Blaze\\Runtime\\BlazeAttributeBag::sanitized($__data, $__bound);'."\n"; $output .= 'extract($__slots, EXTR_SKIP); unset($__slots);'."\n"; $output .= 'extract($__data, EXTR_SKIP); unset($__data, $__bound);'."\n"; $output .= 'ob_start();' . "\n"; From 0eac8e6851ce4d653d2ae1599939cfef692ea4c6 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Mon, 2 Mar 2026 12:15:09 +0100 Subject: [PATCH 06/38] Revert "Update Wrapper.php" This reverts commit 90eef7e7707d4543d124f2febb20e6e322e28790. --- src/Compiler/Wrapper.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Compiler/Wrapper.php b/src/Compiler/Wrapper.php index c8faced5..8254c287 100644 --- a/src/Compiler/Wrapper.php +++ b/src/Compiler/Wrapper.php @@ -58,8 +58,6 @@ public function wrap(string $compiled, string $path, ?string $source = null): st $output .= $this->globalVariables($source, $compiled); $output .= 'if (($__data[\'attributes\'] ?? null) instanceof \Illuminate\View\ComponentAttributeBag) { $__data = $__data + $__data[\'attributes\']->all(); unset($__data[\'attributes\']); }'."\n"; $output .= '$attributes = \\Livewire\\Blaze\\Runtime\\BlazeAttributeBag::sanitized($__data, $__bound);'."\n"; - $output .= '$attributes = \\Livewire\\Blaze\\Runtime\\BlazeAttributeBag::sanitized($__data, $__bound);'."\n"; - $output .= '$attributes = \\Livewire\\Blaze\\Runtime\\BlazeAttributeBag::sanitized($__data, $__bound);'."\n"; $output .= 'extract($__slots, EXTR_SKIP); unset($__slots);'."\n"; $output .= 'extract($__data, EXTR_SKIP); unset($__data, $__bound);'."\n"; $output .= 'ob_start();' . "\n"; From 4bff9e9411a04b8b8ccd19b46b9290c3ae392586 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Mon, 2 Mar 2026 12:15:02 +0100 Subject: [PATCH 07/38] Test stability --- .github/scripts/aggregate-stability.php | 207 ++++++++++++++++++ .github/workflows/benchmark-stability.yml | 107 +++++++++ .github/workflows/ci.yml | 8 +- .../app/Console/Commands/BenchmarkCommand.php | 83 ++++--- 4 files changed, 374 insertions(+), 31 deletions(-) create mode 100644 .github/scripts/aggregate-stability.php create mode 100644 .github/workflows/benchmark-stability.yml diff --git a/.github/scripts/aggregate-stability.php b/.github/scripts/aggregate-stability.php new file mode 100644 index 00000000..56a0a569 --- /dev/null +++ b/.github/scripts/aggregate-stability.php @@ -0,0 +1,207 @@ + + */ + +$dir = $argv[1] ?? 'stats'; + +// Each config directory contains stats-rep1.json .. stats-rep5.json. +$files = glob("$dir/*/stats-rep*.json"); + +if (empty($files)) { + fwrite(STDERR, "No stats files found in $dir\n"); + exit(1); +} + +// Group runs by config (iterations x rounds x warmup). +$byConfig = []; + +foreach ($files as $file) { + $data = json_decode(file_get_contents($file), true); + + if (! $data || ! isset($data['config'], $data['benchmarks'])) { + fwrite(STDERR, "Skipping invalid file: $file\n"); + continue; + } + + $key = sprintf( + 'i%05d-r%02d-w%d', + $data['config']['iterations'], + $data['config']['rounds'], + $data['config']['warmup'], + ); + + $byConfig[$key]['config'] = $data['config']; + $byConfig[$key]['runs'][] = $data['benchmarks']; +} + +ksort($byConfig); + +function computeStats(array $values): array +{ + $n = count($values); + + if ($n === 0) { + return ['mean' => 0, 'stddev' => 0, 'cv' => 0, 'min' => 0, 'max' => 0]; + } + + $mean = array_sum($values) / $n; + $variance = array_reduce( + $values, + fn ($carry, $v) => $carry + ($v - $mean) ** 2, + 0 + ) / max($n - 1, 1); + + $stddev = sqrt($variance); + $cv = $mean > 0 ? ($stddev / $mean) * 100 : 0; + + return [ + 'mean' => $mean, + 'stddev' => $stddev, + 'cv' => $cv, + 'min' => min($values), + 'max' => max($values), + ]; +} + +// --- Report --- + +echo "# Benchmark Stability Report\n\n"; + +// Overall ranking table. +echo "## Ranking (sorted by Blaze cross-run CV%)\n\n"; +echo "Lower CV% = more stable across repeated runs on the same machine.\n\n"; +echo "| Config | Runs | Blaze cross-run CV% | Blade cross-run CV% | Avg Blaze within-run CV% | Verdict |\n"; +echo "|--------|------|--------------------:|--------------------:|-------------------------:|--------:|\n"; + +$rankings = []; + +foreach ($byConfig as $key => $group) { + $config = $group['config']; + $runs = $group['runs']; + $benchNames = array_keys($runs[0]); + + $blazeCrossRunCvs = []; + $bladeCrossRunCvs = []; + $blazeWithinRunCvs = []; + + foreach ($benchNames as $bench) { + $blazeMedians = array_map(fn ($r) => $r[$bench]['blaze']['median'], $runs); + $bladeMedians = array_map(fn ($r) => $r[$bench]['blade']['median'], $runs); + + $blazeCrossRunCvs[] = computeStats($blazeMedians)['cv']; + $bladeCrossRunCvs[] = computeStats($bladeMedians)['cv']; + + foreach ($runs as $run) { + $blazeWithinRunCvs[] = $run[$bench]['blaze']['cv_percent']; + } + } + + $avgBlazeCrossRunCv = array_sum($blazeCrossRunCvs) / count($blazeCrossRunCvs); + $avgBladeCrossRunCv = array_sum($bladeCrossRunCvs) / count($bladeCrossRunCvs); + $avgBlazeWithinRunCv = array_sum($blazeWithinRunCvs) / count($blazeWithinRunCvs); + + if ($avgBlazeCrossRunCv < 3.0) { + $verdict = 'EXCELLENT'; + } elseif ($avgBlazeCrossRunCv < 5.0) { + $verdict = 'GOOD'; + } elseif ($avgBlazeCrossRunCv < 10.0) { + $verdict = 'FAIR'; + } else { + $verdict = 'POOR'; + } + + $label = sprintf( + '%dk iter x %d rounds x %d warmup', + $config['iterations'] / 1000, + $config['rounds'], + $config['warmup'], + ); + + $rankings[] = [ + 'key' => $key, + 'label' => $label, + 'runs' => count($runs), + 'blaze_cross_cv' => $avgBlazeCrossRunCv, + 'blade_cross_cv' => $avgBladeCrossRunCv, + 'blaze_within_cv' => $avgBlazeWithinRunCv, + 'verdict' => $verdict, + ]; +} + +// Sort by blaze cross-run CV ascending (most stable first). +usort($rankings, fn ($a, $b) => $a['blaze_cross_cv'] <=> $b['blaze_cross_cv']); + +foreach ($rankings as $r) { + printf( + "| %-30s | %d | %5.1f%% | %5.1f%% | %5.1f%% | %-9s |\n", + $r['label'], + $r['runs'], + $r['blaze_cross_cv'], + $r['blade_cross_cv'], + $r['blaze_within_cv'], + $r['verdict'], + ); +} + +echo "\n---\n\n"; + +// Detailed per-config tables. +foreach ($byConfig as $key => $group) { + $config = $group['config']; + $runs = $group['runs']; + $numRuns = count($runs); + + $label = sprintf( + '%dk iterations x %d rounds x %d warmup', + $config['iterations'] / 1000, + $config['rounds'], + $config['warmup'], + ); + + echo "## $label ($numRuns runs)\n\n"; + echo "| Benchmark | Engine | Medians (per run) | Cross-run CV% | Avg within-run CV% | Stable? |\n"; + echo "|-----------|--------|-------------------|:-------------:|:-------------------:|:-------:|\n"; + + $benchNames = array_keys($runs[0]); + + foreach ($benchNames as $bench) { + foreach (['blade', 'blaze'] as $engine) { + $medians = array_map(fn ($r) => $r[$bench][$engine]['median'], $runs); + $withinCvs = array_map(fn ($r) => $r[$bench][$engine]['cv_percent'], $runs); + + $stats = computeStats($medians); + $avgWithinCv = array_sum($withinCvs) / count($withinCvs); + + $medianStr = implode(', ', array_map(fn ($v) => sprintf('%.1f', $v), $medians)); + + if ($stats['cv'] < 3.0) { + $stable = 'YES'; + } elseif ($stats['cv'] < 5.0) { + $stable = 'OK'; + } elseif ($stats['cv'] < 10.0) { + $stable = 'FAIR'; + } else { + $stable = 'NO'; + } + + printf( + "| %-25s | %-5s | %s | %.1f%% | %.1f%% | %s |\n", + $bench, + strtoupper($engine), + $medianStr, + $stats['cv'], + $avgWithinCv, + $stable, + ); + } + } + + echo "\n"; +} diff --git a/.github/workflows/benchmark-stability.yml b/.github/workflows/benchmark-stability.yml new file mode 100644 index 00000000..21ef3a8b --- /dev/null +++ b/.github/workflows/benchmark-stability.yml @@ -0,0 +1,107 @@ +name: Benchmark Stability + +on: + push: + branches-ignore: + - main + +jobs: + stability: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # Low iterations + - { iterations: 5000, rounds: 5, warmup: 2, label: 5k-r5-w2 } + - { iterations: 5000, rounds: 10, warmup: 3, label: 5k-r10-w3 } + # Current default + - { iterations: 10000, rounds: 5, warmup: 2, label: 10k-r5-w2 } + # More rounds + - { iterations: 10000, rounds: 7, warmup: 2, label: 10k-r7-w2 } + - { iterations: 10000, rounds: 10, warmup: 2, label: 10k-r10-w2 } + # Heavier warmup + - { iterations: 10000, rounds: 5, warmup: 5, label: 10k-r5-w5 } + - { iterations: 10000, rounds: 7, warmup: 5, label: 10k-r7-w5 } + # Higher iterations + - { iterations: 15000, rounds: 5, warmup: 2, label: 15k-r5-w2 } + - { iterations: 15000, rounds: 5, warmup: 5, label: 15k-r5-w5 } + - { iterations: 15000, rounds: 7, warmup: 3, label: 15k-r7-w3 } + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + tools: composer:v2 + coverage: none + extensions: mbstring, dom, curl, json, libxml, xml, xmlwriter, simplexml, tokenizer + + - name: Determine composer cache directory + id: composer-cache + run: echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT + + - name: Cache composer + uses: actions/cache@v3 + with: + path: ${{ steps.composer-cache.outputs.dir }} + key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} + restore-keys: | + ${{ runner.os }}-composer- + + - name: Install dependencies + run: composer install --prefer-dist --no-progress --no-interaction + + - name: Run benchmarks (5 repeats) + run: | + for i in 1 2 3 4 5; do + echo "--- Repeat $i ---" + vendor/bin/testbench benchmark \ + --ci \ + --iterations=${{ matrix.iterations }} \ + --rounds=${{ matrix.rounds }} \ + --warmup=${{ matrix.warmup }} \ + --dump=stats-rep${i}.json + echo "" + done + + - name: Upload stats + uses: actions/upload-artifact@v4 + with: + name: stats-${{ matrix.label }} + path: stats-rep*.json + + aggregate: + needs: stability + if: always() + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + + - name: Download all stats + uses: actions/download-artifact@v4 + with: + path: stats + pattern: stats-* + + - name: Generate stability report + run: php .github/scripts/aggregate-stability.php stats > stability-report.md + + - name: Show report + run: cat stability-report.md >> $GITHUB_STEP_SUMMARY + + - name: Upload report + uses: actions/upload-artifact@v4 + with: + name: stability-report + path: stability-report.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1180fa16..bee5ead3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,10 +95,16 @@ jobs: run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md - name: Run benchmark - run: vendor/bin/testbench benchmark --ci >> benchmark-result.md + run: vendor/bin/testbench benchmark --ci --dump=benchmark-stats.json >> benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 with: name: benchmark-result path: benchmark-result.md + + - name: Upload benchmark stats + uses: actions/upload-artifact@v4 + with: + name: benchmark-stats + path: benchmark-stats.json diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index faad1577..497e052d 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -16,7 +16,8 @@ class BenchmarkCommand extends Command {--rounds=5 : Number of timed rounds per benchmark} {--warmup=2 : Number of untimed warmup rounds} {--snapshot : Save results as the baseline snapshot} - {--ci : Output a markdown table with no progress (for CI)}'; + {--ci : Output a markdown table with no progress (for CI)} + {--dump= : Dump detailed per-round stats to a JSON file}'; protected $description = 'Run Blaze performance benchmarks'; @@ -44,6 +45,10 @@ public function handle(): int $this->saveSnapshot($results); } + if ($dump = $this->option('dump')) { + $this->dumpStats($results, $dump); + } + return Command::SUCCESS; } @@ -97,6 +102,8 @@ protected function runBenchmarks(): array $name => [ 'blade_ms' => round(collect($bladeTimes[$name])->median(), 2), 'blaze_ms' => round(collect($blazeTimes[$name])->median(), 2), + 'blade_rounds' => $bladeTimes[$name], + 'blaze_rounds' => $blazeTimes[$name], ], ])->all(); } @@ -113,9 +120,9 @@ protected function buildTable(array $results): array $improvement = $this->improvement($result) . '%'; if ($prev = $snapshot['benchmarks'][$name] ?? null) { - $blade .= ' ' . $this->formatChange($prev['blade_ms'], $result['blade_ms'], 10.0); - $blaze .= ' ' . $this->formatChange($prev['blaze_ms'], $result['blaze_ms'], 5.0); - $improvement .= ' ' . $this->formatChange($prev['improvement'], $this->improvement($result), 1.0); + $blade .= ' ' . $this->formatChange($prev['blade_ms'], $result['blade_ms']); + $blaze .= ' ' . $this->formatChange($prev['blaze_ms'], $result['blaze_ms']); + $improvement .= ' ' . $this->formatChange($prev['improvement'], $this->improvement($result)); } return [$name, $blade, $blaze, $improvement]; @@ -172,15 +179,6 @@ protected function outputMarkdown(array $results): void protected function saveSnapshot(array $results): void { - $existing = $this->loadSnapshot(); - - if ($existing && ! $this->hasSignificantChange($results, $existing)) { - $this->newLine(); - $this->comment('Snapshot unchanged (no significant difference).'); - - return; - } - $snapshot = [ 'iterations' => $this->iterations, 'rounds' => $this->rounds, @@ -199,24 +197,54 @@ protected function saveSnapshot(array $results): void $this->info("Snapshot saved to {$path}"); } - protected function hasSignificantChange(array $results, array $snapshot): bool + protected function dumpStats(array $results, string $path): void { - if (array_keys($results) !== array_keys($snapshot['benchmarks'])) { - return true; + if (! str_starts_with($path, '/')) { + $path = dirname(__DIR__, 4) . '/' . $path; } - return collect($results)->contains(function ($result, $name) use ($snapshot) { - $prev = $snapshot['benchmarks'][$name]; + $stats = [ + 'config' => [ + 'iterations' => $this->iterations, + 'rounds' => $this->rounds, + 'warmup' => $this->warmupRounds, + 'timestamp' => now()->toIso8601String(), + ], + 'benchmarks' => collect($results)->map(fn ($result, $name) => [ + 'blade' => $this->computeStats($result['blade_rounds']), + 'blaze' => $this->computeStats($result['blaze_rounds']), + 'improvement' => $this->improvement($result), + ])->all(), + ]; - return $this->exceedsThreshold($prev['blade_ms'], $result['blade_ms'], 10.0) - || $this->exceedsThreshold($prev['blaze_ms'], $result['blaze_ms'], 5.0) - || $this->exceedsThreshold($prev['improvement'], $this->improvement($result), 0.5); - }); + File::put($path, json_encode($stats, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n"); } - protected function exceedsThreshold(float $old, float $new, float $threshold): bool + protected function computeStats(array $values): array { - return $old > 0 && abs(($new - $old) / $old * 100) >= $threshold; + $collection = collect($values); + $mean = $collection->avg(); + $median = $collection->median(); + $min = $collection->min(); + $max = $collection->max(); + $count = $collection->count(); + + $variance = $collection->reduce( + fn ($carry, $val) => $carry + pow($val - $mean, 2), 0 + ) / max($count - 1, 1); + + $stddev = sqrt($variance); + $cv = $mean > 0 ? ($stddev / $mean) * 100 : 0; + + return [ + 'mean' => round($mean, 2), + 'median' => round($median, 2), + 'min' => round($min, 2), + 'max' => round($max, 2), + 'stddev' => round($stddev, 2), + 'cv_percent' => round($cv, 1), + 'rounds' => array_map(fn ($v) => round($v, 2), $values), + ]; } protected function loadSnapshot(): ?array @@ -244,18 +272,13 @@ protected function improvement(array $result): float : 0; } - protected function formatChange(float $old, float $new, float $threshold): string + protected function formatChange(float $old, float $new): string { if ($old == 0) { return '(~)'; } $change = ($new - $old) / abs($old) * 100; - - if (abs($change) < $threshold) { - return '(~)'; - } - $sign = $change > 0 ? '+' : ''; return "({$sign}" . round($change, 1) . '%)'; From d5d9fb81491cd9fc6080579df3127f412d1e8a1d Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Mon, 2 Mar 2026 12:38:08 +0100 Subject: [PATCH 08/38] Test stability --- .github/workflows/benchmark-stability.yml | 24 ++++++++----------- .../app/Console/Commands/BenchmarkCommand.php | 1 - 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/.github/workflows/benchmark-stability.yml b/.github/workflows/benchmark-stability.yml index 21ef3a8b..4e83bf44 100644 --- a/.github/workflows/benchmark-stability.yml +++ b/.github/workflows/benchmark-stability.yml @@ -12,21 +12,17 @@ jobs: fail-fast: false matrix: include: - # Low iterations - - { iterations: 5000, rounds: 5, warmup: 2, label: 5k-r5-w2 } - - { iterations: 5000, rounds: 10, warmup: 3, label: 5k-r10-w3 } - # Current default - - { iterations: 10000, rounds: 5, warmup: 2, label: 10k-r5-w2 } - # More rounds + # Rounds sweep (warmup 2) + - { iterations: 10000, rounds: 6, warmup: 2, label: 10k-r6-w2 } - { iterations: 10000, rounds: 7, warmup: 2, label: 10k-r7-w2 } - - { iterations: 10000, rounds: 10, warmup: 2, label: 10k-r10-w2 } - # Heavier warmup - - { iterations: 10000, rounds: 5, warmup: 5, label: 10k-r5-w5 } - - { iterations: 10000, rounds: 7, warmup: 5, label: 10k-r7-w5 } - # Higher iterations - - { iterations: 15000, rounds: 5, warmup: 2, label: 15k-r5-w2 } - - { iterations: 15000, rounds: 5, warmup: 5, label: 15k-r5-w5 } - - { iterations: 15000, rounds: 7, warmup: 3, label: 15k-r7-w3 } + - { iterations: 10000, rounds: 8, warmup: 2, label: 10k-r8-w2 } + - { iterations: 10000, rounds: 9, warmup: 2, label: 10k-r9-w2 } + # Warmup sweep (7 rounds) + - { iterations: 10000, rounds: 7, warmup: 1, label: 10k-r7-w1 } + - { iterations: 10000, rounds: 7, warmup: 3, label: 10k-r7-w3 } + # Combo tweaks + - { iterations: 10000, rounds: 8, warmup: 1, label: 10k-r8-w1 } + - { iterations: 10000, rounds: 8, warmup: 3, label: 10k-r8-w3 } steps: - name: Checkout diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index 497e052d..7f573244 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -122,7 +122,6 @@ protected function buildTable(array $results): array if ($prev = $snapshot['benchmarks'][$name] ?? null) { $blade .= ' ' . $this->formatChange($prev['blade_ms'], $result['blade_ms']); $blaze .= ' ' . $this->formatChange($prev['blaze_ms'], $result['blaze_ms']); - $improvement .= ' ' . $this->formatChange($prev['improvement'], $this->improvement($result)); } return [$name, $blade, $blaze, $improvement]; From 928410d49b9dc34f89092ebfead3a5a9bfe485a7 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Mon, 2 Mar 2026 13:38:03 +0100 Subject: [PATCH 09/38] Cleanup --- .github/scripts/aggregate-stability.php | 207 ------------------ .github/workflows/benchmark-stability.yml | 103 --------- .github/workflows/ci.yml | 8 +- .../app/Console/Commands/BenchmarkCommand.php | 68 +----- 4 files changed, 9 insertions(+), 377 deletions(-) delete mode 100644 .github/scripts/aggregate-stability.php delete mode 100644 .github/workflows/benchmark-stability.yml diff --git a/.github/scripts/aggregate-stability.php b/.github/scripts/aggregate-stability.php deleted file mode 100644 index 56a0a569..00000000 --- a/.github/scripts/aggregate-stability.php +++ /dev/null @@ -1,207 +0,0 @@ - - */ - -$dir = $argv[1] ?? 'stats'; - -// Each config directory contains stats-rep1.json .. stats-rep5.json. -$files = glob("$dir/*/stats-rep*.json"); - -if (empty($files)) { - fwrite(STDERR, "No stats files found in $dir\n"); - exit(1); -} - -// Group runs by config (iterations x rounds x warmup). -$byConfig = []; - -foreach ($files as $file) { - $data = json_decode(file_get_contents($file), true); - - if (! $data || ! isset($data['config'], $data['benchmarks'])) { - fwrite(STDERR, "Skipping invalid file: $file\n"); - continue; - } - - $key = sprintf( - 'i%05d-r%02d-w%d', - $data['config']['iterations'], - $data['config']['rounds'], - $data['config']['warmup'], - ); - - $byConfig[$key]['config'] = $data['config']; - $byConfig[$key]['runs'][] = $data['benchmarks']; -} - -ksort($byConfig); - -function computeStats(array $values): array -{ - $n = count($values); - - if ($n === 0) { - return ['mean' => 0, 'stddev' => 0, 'cv' => 0, 'min' => 0, 'max' => 0]; - } - - $mean = array_sum($values) / $n; - $variance = array_reduce( - $values, - fn ($carry, $v) => $carry + ($v - $mean) ** 2, - 0 - ) / max($n - 1, 1); - - $stddev = sqrt($variance); - $cv = $mean > 0 ? ($stddev / $mean) * 100 : 0; - - return [ - 'mean' => $mean, - 'stddev' => $stddev, - 'cv' => $cv, - 'min' => min($values), - 'max' => max($values), - ]; -} - -// --- Report --- - -echo "# Benchmark Stability Report\n\n"; - -// Overall ranking table. -echo "## Ranking (sorted by Blaze cross-run CV%)\n\n"; -echo "Lower CV% = more stable across repeated runs on the same machine.\n\n"; -echo "| Config | Runs | Blaze cross-run CV% | Blade cross-run CV% | Avg Blaze within-run CV% | Verdict |\n"; -echo "|--------|------|--------------------:|--------------------:|-------------------------:|--------:|\n"; - -$rankings = []; - -foreach ($byConfig as $key => $group) { - $config = $group['config']; - $runs = $group['runs']; - $benchNames = array_keys($runs[0]); - - $blazeCrossRunCvs = []; - $bladeCrossRunCvs = []; - $blazeWithinRunCvs = []; - - foreach ($benchNames as $bench) { - $blazeMedians = array_map(fn ($r) => $r[$bench]['blaze']['median'], $runs); - $bladeMedians = array_map(fn ($r) => $r[$bench]['blade']['median'], $runs); - - $blazeCrossRunCvs[] = computeStats($blazeMedians)['cv']; - $bladeCrossRunCvs[] = computeStats($bladeMedians)['cv']; - - foreach ($runs as $run) { - $blazeWithinRunCvs[] = $run[$bench]['blaze']['cv_percent']; - } - } - - $avgBlazeCrossRunCv = array_sum($blazeCrossRunCvs) / count($blazeCrossRunCvs); - $avgBladeCrossRunCv = array_sum($bladeCrossRunCvs) / count($bladeCrossRunCvs); - $avgBlazeWithinRunCv = array_sum($blazeWithinRunCvs) / count($blazeWithinRunCvs); - - if ($avgBlazeCrossRunCv < 3.0) { - $verdict = 'EXCELLENT'; - } elseif ($avgBlazeCrossRunCv < 5.0) { - $verdict = 'GOOD'; - } elseif ($avgBlazeCrossRunCv < 10.0) { - $verdict = 'FAIR'; - } else { - $verdict = 'POOR'; - } - - $label = sprintf( - '%dk iter x %d rounds x %d warmup', - $config['iterations'] / 1000, - $config['rounds'], - $config['warmup'], - ); - - $rankings[] = [ - 'key' => $key, - 'label' => $label, - 'runs' => count($runs), - 'blaze_cross_cv' => $avgBlazeCrossRunCv, - 'blade_cross_cv' => $avgBladeCrossRunCv, - 'blaze_within_cv' => $avgBlazeWithinRunCv, - 'verdict' => $verdict, - ]; -} - -// Sort by blaze cross-run CV ascending (most stable first). -usort($rankings, fn ($a, $b) => $a['blaze_cross_cv'] <=> $b['blaze_cross_cv']); - -foreach ($rankings as $r) { - printf( - "| %-30s | %d | %5.1f%% | %5.1f%% | %5.1f%% | %-9s |\n", - $r['label'], - $r['runs'], - $r['blaze_cross_cv'], - $r['blade_cross_cv'], - $r['blaze_within_cv'], - $r['verdict'], - ); -} - -echo "\n---\n\n"; - -// Detailed per-config tables. -foreach ($byConfig as $key => $group) { - $config = $group['config']; - $runs = $group['runs']; - $numRuns = count($runs); - - $label = sprintf( - '%dk iterations x %d rounds x %d warmup', - $config['iterations'] / 1000, - $config['rounds'], - $config['warmup'], - ); - - echo "## $label ($numRuns runs)\n\n"; - echo "| Benchmark | Engine | Medians (per run) | Cross-run CV% | Avg within-run CV% | Stable? |\n"; - echo "|-----------|--------|-------------------|:-------------:|:-------------------:|:-------:|\n"; - - $benchNames = array_keys($runs[0]); - - foreach ($benchNames as $bench) { - foreach (['blade', 'blaze'] as $engine) { - $medians = array_map(fn ($r) => $r[$bench][$engine]['median'], $runs); - $withinCvs = array_map(fn ($r) => $r[$bench][$engine]['cv_percent'], $runs); - - $stats = computeStats($medians); - $avgWithinCv = array_sum($withinCvs) / count($withinCvs); - - $medianStr = implode(', ', array_map(fn ($v) => sprintf('%.1f', $v), $medians)); - - if ($stats['cv'] < 3.0) { - $stable = 'YES'; - } elseif ($stats['cv'] < 5.0) { - $stable = 'OK'; - } elseif ($stats['cv'] < 10.0) { - $stable = 'FAIR'; - } else { - $stable = 'NO'; - } - - printf( - "| %-25s | %-5s | %s | %.1f%% | %.1f%% | %s |\n", - $bench, - strtoupper($engine), - $medianStr, - $stats['cv'], - $avgWithinCv, - $stable, - ); - } - } - - echo "\n"; -} diff --git a/.github/workflows/benchmark-stability.yml b/.github/workflows/benchmark-stability.yml deleted file mode 100644 index 4e83bf44..00000000 --- a/.github/workflows/benchmark-stability.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Benchmark Stability - -on: - push: - branches-ignore: - - main - -jobs: - stability: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - # Rounds sweep (warmup 2) - - { iterations: 10000, rounds: 6, warmup: 2, label: 10k-r6-w2 } - - { iterations: 10000, rounds: 7, warmup: 2, label: 10k-r7-w2 } - - { iterations: 10000, rounds: 8, warmup: 2, label: 10k-r8-w2 } - - { iterations: 10000, rounds: 9, warmup: 2, label: 10k-r9-w2 } - # Warmup sweep (7 rounds) - - { iterations: 10000, rounds: 7, warmup: 1, label: 10k-r7-w1 } - - { iterations: 10000, rounds: 7, warmup: 3, label: 10k-r7-w3 } - # Combo tweaks - - { iterations: 10000, rounds: 8, warmup: 1, label: 10k-r8-w1 } - - { iterations: 10000, rounds: 8, warmup: 3, label: 10k-r8-w3 } - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup PHP - uses: shivammathur/setup-php@v2 - with: - php-version: '8.4' - tools: composer:v2 - coverage: none - extensions: mbstring, dom, curl, json, libxml, xml, xmlwriter, simplexml, tokenizer - - - name: Determine composer cache directory - id: composer-cache - run: echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT - - - name: Cache composer - uses: actions/cache@v3 - with: - path: ${{ steps.composer-cache.outputs.dir }} - key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} - restore-keys: | - ${{ runner.os }}-composer- - - - name: Install dependencies - run: composer install --prefer-dist --no-progress --no-interaction - - - name: Run benchmarks (5 repeats) - run: | - for i in 1 2 3 4 5; do - echo "--- Repeat $i ---" - vendor/bin/testbench benchmark \ - --ci \ - --iterations=${{ matrix.iterations }} \ - --rounds=${{ matrix.rounds }} \ - --warmup=${{ matrix.warmup }} \ - --dump=stats-rep${i}.json - echo "" - done - - - name: Upload stats - uses: actions/upload-artifact@v4 - with: - name: stats-${{ matrix.label }} - path: stats-rep*.json - - aggregate: - needs: stability - if: always() - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup PHP - uses: shivammathur/setup-php@v2 - with: - php-version: '8.4' - - - name: Download all stats - uses: actions/download-artifact@v4 - with: - path: stats - pattern: stats-* - - - name: Generate stability report - run: php .github/scripts/aggregate-stability.php stats > stability-report.md - - - name: Show report - run: cat stability-report.md >> $GITHUB_STEP_SUMMARY - - - name: Upload report - uses: actions/upload-artifact@v4 - with: - name: stability-report - path: stability-report.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bee5ead3..1180fa16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,16 +95,10 @@ jobs: run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md - name: Run benchmark - run: vendor/bin/testbench benchmark --ci --dump=benchmark-stats.json >> benchmark-result.md + run: vendor/bin/testbench benchmark --ci >> benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 with: name: benchmark-result path: benchmark-result.md - - - name: Upload benchmark stats - uses: actions/upload-artifact@v4 - with: - name: benchmark-stats - path: benchmark-stats.json diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index 7f573244..c3e6524e 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -13,11 +13,10 @@ class BenchmarkCommand extends Command { protected $signature = 'benchmark {--iterations=10000 : Number of component renders per benchmark} - {--rounds=5 : Number of timed rounds per benchmark} + {--rounds=7 : Number of timed rounds per benchmark} {--warmup=2 : Number of untimed warmup rounds} {--snapshot : Save results as the baseline snapshot} - {--ci : Output a markdown table with no progress (for CI)} - {--dump= : Dump detailed per-round stats to a JSON file}'; + {--ci : Output a markdown table with no progress (for CI)}'; protected $description = 'Run Blaze performance benchmarks'; @@ -45,10 +44,6 @@ public function handle(): int $this->saveSnapshot($results); } - if ($dump = $this->option('dump')) { - $this->dumpStats($results, $dump); - } - return Command::SUCCESS; } @@ -102,8 +97,6 @@ protected function runBenchmarks(): array $name => [ 'blade_ms' => round(collect($bladeTimes[$name])->median(), 2), 'blaze_ms' => round(collect($blazeTimes[$name])->median(), 2), - 'blade_rounds' => $bladeTimes[$name], - 'blaze_rounds' => $blazeTimes[$name], ], ])->all(); } @@ -196,56 +189,6 @@ protected function saveSnapshot(array $results): void $this->info("Snapshot saved to {$path}"); } - protected function dumpStats(array $results, string $path): void - { - if (! str_starts_with($path, '/')) { - $path = dirname(__DIR__, 4) . '/' . $path; - } - - $stats = [ - 'config' => [ - 'iterations' => $this->iterations, - 'rounds' => $this->rounds, - 'warmup' => $this->warmupRounds, - 'timestamp' => now()->toIso8601String(), - ], - 'benchmarks' => collect($results)->map(fn ($result, $name) => [ - 'blade' => $this->computeStats($result['blade_rounds']), - 'blaze' => $this->computeStats($result['blaze_rounds']), - 'improvement' => $this->improvement($result), - ])->all(), - ]; - - File::put($path, json_encode($stats, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n"); - } - - protected function computeStats(array $values): array - { - $collection = collect($values); - $mean = $collection->avg(); - $median = $collection->median(); - $min = $collection->min(); - $max = $collection->max(); - $count = $collection->count(); - - $variance = $collection->reduce( - fn ($carry, $val) => $carry + pow($val - $mean, 2), 0 - ) / max($count - 1, 1); - - $stddev = sqrt($variance); - $cv = $mean > 0 ? ($stddev / $mean) * 100 : 0; - - return [ - 'mean' => round($mean, 2), - 'median' => round($median, 2), - 'min' => round($min, 2), - 'max' => round($max, 2), - 'stddev' => round($stddev, 2), - 'cv_percent' => round($cv, 1), - 'rounds' => array_map(fn ($v) => round($v, 2), $values), - ]; - } - protected function loadSnapshot(): ?array { $path = $this->snapshotPath(); @@ -271,13 +214,18 @@ protected function improvement(array $result): float : 0; } - protected function formatChange(float $old, float $new): string + protected function formatChange(float $old, float $new, float $threshold = 2.0): string { if ($old == 0) { return '(~)'; } $change = ($new - $old) / abs($old) * 100; + + if (abs($change) < $threshold) { + return '(~)'; + } + $sign = $change > 0 ? '+' : ''; return "({$sign}" . round($change, 1) . '%)'; From fb7a40234ea4a587c0c652001aecead45b7ab679 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Mon, 2 Mar 2026 13:53:01 +0100 Subject: [PATCH 10/38] Update BenchmarkCommand.php --- workbench/app/Console/Commands/BenchmarkCommand.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index c3e6524e..3193353e 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -12,8 +12,8 @@ class BenchmarkCommand extends Command { protected $signature = 'benchmark - {--iterations=10000 : Number of component renders per benchmark} - {--rounds=7 : Number of timed rounds per benchmark} + {--iterations=5000 : Number of component renders per benchmark} + {--rounds=15 : Number of timed rounds per benchmark} {--warmup=2 : Number of untimed warmup rounds} {--snapshot : Save results as the baseline snapshot} {--ci : Output a markdown table with no progress (for CI)}'; From 718048583a25e30d4f4106a6c8c17e5f44418071 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Mon, 2 Mar 2026 13:54:38 +0100 Subject: [PATCH 11/38] Update ci.yml --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1180fa16..f5e78e07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,7 @@ jobs: run: composer install --prefer-dist --no-progress --no-interaction - name: Generate baseline snapshot - run: vendor/bin/testbench benchmark --snapshot --ci + run: vendor/bin/testbench benchmark --snapshot --ci --iterations=5000 --rounds=15 --warmup=2 - name: Save baseline snapshot run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json @@ -95,7 +95,7 @@ jobs: run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md - name: Run benchmark - run: vendor/bin/testbench benchmark --ci >> benchmark-result.md + run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=15 --warmup=2 >> benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 From 8b926cedad61631af5570b8766912b4bffd0487c Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Mon, 2 Mar 2026 14:03:33 +0100 Subject: [PATCH 12/38] Update BenchmarkCommand.php --- .../app/Console/Commands/BenchmarkCommand.php | 60 ++++++++++++++++--- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index 3193353e..fd6f6ea0 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -15,6 +15,7 @@ class BenchmarkCommand extends Command {--iterations=5000 : Number of component renders per benchmark} {--rounds=15 : Number of timed rounds per benchmark} {--warmup=2 : Number of untimed warmup rounds} + {--filter-outliers : Exclude outlier rounds using the IQR method} {--snapshot : Save results as the baseline snapshot} {--ci : Output a markdown table with no progress (for CI)}'; @@ -93,12 +94,24 @@ protected function runBenchmarks(): array $this->newLine(2); } - return collect($names)->mapWithKeys(fn ($name) => [ - $name => [ - 'blade_ms' => round(collect($bladeTimes[$name])->median(), 2), - 'blaze_ms' => round(collect($blazeTimes[$name])->median(), 2), - ], - ])->all(); + $filterOutliers = $this->option('filter-outliers'); + + return collect($names)->mapWithKeys(function ($name) use ($bladeTimes, $blazeTimes, $filterOutliers) { + $blade = collect($bladeTimes[$name]); + $blaze = collect($blazeTimes[$name]); + + if ($filterOutliers) { + $blade = $this->filterOutliers($blade); + $blaze = $this->filterOutliers($blaze); + } + + return [ + $name => [ + 'blade_ms' => round($blade->median(), 2), + 'blaze_ms' => round($blaze->median(), 2), + ], + ]; + })->all(); } protected function buildTable(array $results): array @@ -133,9 +146,13 @@ protected function displayResults(array $results): void $this->newLine(); $this->info("{$this->iterations} iterations x {$this->rounds} rounds per benchmark"); + if ($this->option('filter-outliers')) { + $this->comment('Outlier rounds excluded (IQR method)'); + } + if ($snapshot) { $rounds = $snapshot['rounds'] ?? 1; - $this->comment("Compared against snapshot ({$snapshot['iterations']} iterations x {$rounds} rounds)"); + $this->comment("Compared against baseline snapshot ({$snapshot['iterations']} iterations x {$rounds} rounds)"); } } @@ -162,7 +179,8 @@ protected function outputMarkdown(array $results): void ...collect($rows)->map($formatRow), '', '' . "{$this->iterations} iterations x {$this->rounds} rounds per benchmark" - . ($snapshot ? ' — compared against committed snapshot' : '') + . ($this->option('filter-outliers') ? ' — outliers excluded (IQR)' : '') + . ($snapshot ? ' — compared against baseline snapshot' : '') . '', ])->implode("\n"); @@ -214,7 +232,7 @@ protected function improvement(array $result): float : 0; } - protected function formatChange(float $old, float $new, float $threshold = 2.0): string + protected function formatChange(float $old, float $new, float $threshold = 0.1): string { if ($old == 0) { return '(~)'; @@ -273,6 +291,30 @@ protected function getBenchmarks(): array ]; } + /** + * Remove outliers using the Interquartile Range (IQR) method. + * + * Values below Q1 - 1.5*IQR or above Q3 + 1.5*IQR are excluded. + */ + protected function filterOutliers(\Illuminate\Support\Collection $values): \Illuminate\Support\Collection + { + if ($values->count() < 4) { + return $values; + } + + $sorted = $values->sort()->values(); + $count = $sorted->count(); + + $q1 = $sorted[intdiv($count, 4)]; + $q3 = $sorted[intdiv($count * 3, 4)]; + $iqr = $q3 - $q1; + + $lower = $q1 - 1.5 * $iqr; + $upper = $q3 + 1.5 * $iqr; + + return $values->filter(fn ($v) => $v >= $lower && $v <= $upper)->values(); + } + protected function renderView(string $view): string { return View::make($view, ['iterations' => $this->iterations])->render(); From dbfd59eef44b999e81fa78edde298448f61aff30 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Mon, 2 Mar 2026 18:11:34 +0100 Subject: [PATCH 13/38] Add benchmark:variance --- src/Runtime/BlazeRuntime.php | 2 +- .../app/Console/Commands/BenchmarkCommand.php | 68 +++++----- .../Commands/BenchmarkVarianceCommand.php | 116 ++++++++++++++++++ 3 files changed, 151 insertions(+), 35 deletions(-) create mode 100644 workbench/app/Console/Commands/BenchmarkVarianceCommand.php diff --git a/src/Runtime/BlazeRuntime.php b/src/Runtime/BlazeRuntime.php index cc82112a..9131ce75 100644 --- a/src/Runtime/BlazeRuntime.php +++ b/src/Runtime/BlazeRuntime.php @@ -45,7 +45,7 @@ public function __construct() */ public function ensureCompiled(string $path, string $compiledPath): void { - if (isset($this->compiled[$path])) { + if (isset($this->compiled[$path]) && file_exists($compiledPath)) { return; } diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index fd6f6ea0..ffaa462a 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -12,8 +12,8 @@ class BenchmarkCommand extends Command { protected $signature = 'benchmark - {--iterations=5000 : Number of component renders per benchmark} - {--rounds=15 : Number of timed rounds per benchmark} + {--iterations=2500 : Number of component renders per benchmark} + {--rounds=100 : Number of timed rounds per benchmark} {--warmup=2 : Number of untimed warmup rounds} {--filter-outliers : Exclude outlier rounds using the IQR method} {--snapshot : Save results as the baseline snapshot} @@ -252,42 +252,42 @@ protected function formatChange(float $old, float $new, float $threshold = 0.1): protected function getBenchmarks(): array { return [ - 'No attributes' => [ - 'blade' => 'bench.blade.no-attributes', - 'blaze' => 'bench.blaze.no-attributes', - ], - 'Attributes only' => [ - 'blade' => 'bench.blade.attributes', - 'blaze' => 'bench.blaze.attributes', - ], - 'Attributes + merge()' => [ - 'blade' => 'bench.blade.merge', - 'blaze' => 'bench.blaze.merge', - ], - 'Attributes + class()' => [ - 'blade' => 'bench.blade.class', - 'blaze' => 'bench.blaze.class', - ], + // 'No attributes' => [ + // 'blade' => 'bench.blade.no-attributes', + // 'blaze' => 'bench.blaze.no-attributes', + // ], + // 'Attributes only' => [ + // 'blade' => 'bench.blade.attributes', + // 'blaze' => 'bench.blaze.attributes', + // ], + // 'Attributes + merge()' => [ + // 'blade' => 'bench.blade.merge', + // 'blaze' => 'bench.blaze.merge', + // ], + // 'Attributes + class()' => [ + // 'blade' => 'bench.blade.class', + // 'blaze' => 'bench.blaze.class', + // ], 'Props + attributes' => [ 'blade' => 'bench.blade.props', 'blaze' => 'bench.blaze.props', ], - 'Default slot' => [ - 'blade' => 'bench.blade.slot', - 'blaze' => 'bench.blaze.slot', - ], - 'Named slots' => [ - 'blade' => 'bench.blade.named-slots', - 'blaze' => 'bench.blaze.named-slots', - ], - '`@aware` (nested)' => [ - 'blade' => 'bench.blade.aware', - 'blaze' => 'bench.blaze.aware', - ], - 'Attribute forwarding' => [ - 'blade' => 'bench.blade.forwarding', - 'blaze' => 'bench.blaze.forwarding', - ], + // 'Default slot' => [ + // 'blade' => 'bench.blade.slot', + // 'blaze' => 'bench.blaze.slot', + // ], + // 'Named slots' => [ + // 'blade' => 'bench.blade.named-slots', + // 'blaze' => 'bench.blaze.named-slots', + // ], + // '`@aware` (nested)' => [ + // 'blade' => 'bench.blade.aware', + // 'blaze' => 'bench.blaze.aware', + // ], + // 'Attribute forwarding' => [ + // 'blade' => 'bench.blade.forwarding', + // 'blaze' => 'bench.blaze.forwarding', + // ], ]; } diff --git a/workbench/app/Console/Commands/BenchmarkVarianceCommand.php b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php new file mode 100644 index 00000000..61874274 --- /dev/null +++ b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php @@ -0,0 +1,116 @@ +iterations = (int) $this->option('iterations'); + $this->rounds = (int) $this->option('rounds'); + $this->warmupRounds = (int) $this->option('warmup'); + $runs = (int) $this->option('runs'); + + if ($runs < 1) { + $this->error('--runs must be at least 1.'); + + return Command::FAILURE; + } + + $totalRuns = $runs + 1; + + Artisan::call('view:clear'); + + // Step 1: Snapshot run + $this->info("Run 1/{$totalRuns}: Creating baseline snapshot..."); + $snapshotResults = $this->runBenchmarks(); + $this->saveSnapshot($snapshotResults); + + // Step 2: Benchmark runs + $allRuns = []; + + for ($i = 0; $i < $runs; $i++) { + $this->newLine(); + $this->info('Run '.($i + 2)."/{$totalRuns}: Benchmarking..."); + Artisan::call('view:clear'); + $allRuns[] = $this->runBenchmarks(); + } + + // Step 3: Display variance report + $this->displayVarianceResults($snapshotResults, $allRuns); + + return Command::SUCCESS; + } + + protected function displayVarianceResults(array $snapshot, array $allRuns): void + { + $benchmarkNames = array_keys($snapshot); + $headers = ['', 'Blade', 'Blaze', 'Improvement']; + $rows = []; + + foreach ($benchmarkNames as $name) { + $snapshotImprovement = $this->improvement($snapshot[$name]); + + $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot[$name]['blade_ms'], $run[$name]['blade_ms'])); + $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot[$name]['blaze_ms'], $run[$name]['blaze_ms'])); + $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run[$name]) - $snapshotImprovement, 1)); + + $rows[] = [ + 'Snapshot', + $this->formatTime($snapshot[$name]['blade_ms']), + $this->formatTime($snapshot[$name]['blaze_ms']), + $snapshotImprovement.'%', + ]; + + $rows[] = [ + 'Variance', + $this->formatVarianceRange($bladeChanges->min(), $bladeChanges->max()), + $this->formatVarianceRange($blazeChanges->min(), $blazeChanges->max()), + $this->formatVarianceRange($improvementChanges->min(), $improvementChanges->max()), + ]; + } + + $this->newLine(2); + $this->table($headers, $rows); + + $this->newLine(); + $this->comment( + "{$this->iterations} iterations x {$this->rounds} rounds per benchmark, " + .count($allRuns).' runs' + .($this->option('filter-outliers') ? ' (outliers excluded)' : '') + ); + } + + protected function formatVarianceRange(float $min, float $max): string + { + $fmt = function (float $v): string { + $rounded = round($v, 1); + if ($rounded == 0) { + return '0%'; + } + $sign = $rounded > 0 ? '+' : ''; + + return $sign.$rounded.'%'; + }; + + return $fmt($min).' / '.$fmt($max); + } + + protected function percentChange(float $old, float $new): float + { + return $old > 0 ? round(($new - $old) / $old * 100, 1) : 0; + } +} From 48af588e0799af57bb2acbe64d987529a75116a9 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 00:52:38 +0100 Subject: [PATCH 14/38] Test variance --- .github/workflows/ci.yml | 26 +-- .../app/Console/Commands/BenchmarkCommand.php | 60 +++--- .../Commands/BenchmarkVarianceCommand.php | 204 +++++++++++++++++- 3 files changed, 242 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5e78e07..a880f8d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,8 +62,8 @@ jobs: steps: - name: Checkout base branch uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.base.sha }} + # with: + # ref: ${{ github.event.pull_request.base.sha }} - name: Setup PHP uses: shivammathur/setup-php@v2 @@ -76,26 +76,26 @@ jobs: - name: Install base dependencies run: composer install --prefer-dist --no-progress --no-interaction - - name: Generate baseline snapshot - run: vendor/bin/testbench benchmark --snapshot --ci --iterations=5000 --rounds=15 --warmup=2 + # - name: Generate baseline snapshot + # run: vendor/bin/testbench benchmark --snapshot --ci --iterations=5000 --rounds=15 --warmup=2 - - name: Save baseline snapshot - run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json + # - name: Save baseline snapshot + # run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json - - name: Checkout PR - uses: actions/checkout@v4 + # - name: Checkout PR + # uses: actions/checkout@v4 - - name: Install PR dependencies - run: composer install --prefer-dist --no-progress --no-interaction + # - name: Install PR dependencies + # run: composer install --prefer-dist --no-progress --no-interaction - - name: Restore baseline snapshot - run: cp ${{ runner.temp }}/benchmark-snapshot.json benchmark-snapshot.json + # - name: Restore baseline snapshot + # run: cp ${{ runner.temp }}/benchmark-snapshot.json benchmark-snapshot.json - name: Save PR number run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md - name: Run benchmark - run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=15 --warmup=2 >> benchmark-result.md + run: vendor/bin/testbench benchmark:variance --ci --iterations=2500 --rounds=100 --filter-outliers >> benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index ffaa462a..314397f1 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -27,19 +27,23 @@ class BenchmarkCommand extends Command protected int $warmupRounds; + protected int $filteredRounds = 0; + public function handle(): int { $this->iterations = (int) $this->option('iterations'); $this->rounds = (int) $this->option('rounds'); $this->warmupRounds = (int) $this->option('warmup'); + $commandStart = microtime(true); Artisan::call('view:clear'); $results = $this->runBenchmarks(); + $totalDuration = round(microtime(true) - $commandStart, 2); $this->option('ci') - ? $this->outputMarkdown($results) - : $this->displayResults($results); + ? $this->outputMarkdown($results, $totalDuration) + : $this->displayResults($results, $totalDuration); if ($this->option('snapshot')) { $this->saveSnapshot($results); @@ -94,24 +98,26 @@ protected function runBenchmarks(): array $this->newLine(2); } - $filterOutliers = $this->option('filter-outliers'); + if ($this->option('filter-outliers')) { + $roundTotals = collect(range(0, $this->rounds - 1))->map( + fn ($r) => collect($names)->sum(fn ($name) => $bladeTimes[$name][$r] + $blazeTimes[$name][$r]) + ); - return collect($names)->mapWithKeys(function ($name) use ($bladeTimes, $blazeTimes, $filterOutliers) { - $blade = collect($bladeTimes[$name]); - $blaze = collect($blazeTimes[$name]); + $keptRounds = $this->nonOutlierIndices($roundTotals); + $this->filteredRounds = $this->rounds - $keptRounds->count(); - if ($filterOutliers) { - $blade = $this->filterOutliers($blade); - $blaze = $this->filterOutliers($blaze); + foreach ($names as $name) { + $bladeTimes[$name] = $keptRounds->map(fn ($r) => $bladeTimes[$name][$r])->all(); + $blazeTimes[$name] = $keptRounds->map(fn ($r) => $blazeTimes[$name][$r])->all(); } + } - return [ - $name => [ - 'blade_ms' => round($blade->median(), 2), - 'blaze_ms' => round($blaze->median(), 2), - ], - ]; - })->all(); + return collect($names)->mapWithKeys(fn ($name) => [ + $name => [ + 'blade_ms' => round(collect($bladeTimes[$name])->median(), 2), + 'blaze_ms' => round(collect($blazeTimes[$name])->median(), 2), + ], + ])->all(); } protected function buildTable(array $results): array @@ -136,7 +142,7 @@ protected function buildTable(array $results): array return [$headers, $rows, $snapshot]; } - protected function displayResults(array $results): void + protected function displayResults(array $results, float $totalDuration): void { [$headers, $rows, $snapshot] = $this->buildTable($results); @@ -144,10 +150,10 @@ protected function displayResults(array $results): void $this->table($headers, $rows); $this->newLine(); - $this->info("{$this->iterations} iterations x {$this->rounds} rounds per benchmark"); + $this->info("{$this->iterations} iterations x {$this->rounds} rounds per benchmark, {$totalDuration}s total"); if ($this->option('filter-outliers')) { - $this->comment('Outlier rounds excluded (IQR method)'); + $this->comment("{$this->filteredRounds} outlier rounds excluded (IQR method)"); } if ($snapshot) { @@ -156,7 +162,7 @@ protected function displayResults(array $results): void } } - protected function outputMarkdown(array $results): void + protected function outputMarkdown(array $results, float $totalDuration): void { [$headers, $rows, $snapshot] = $this->buildTable($results); @@ -178,8 +184,8 @@ protected function outputMarkdown(array $results): void $separator, ...collect($rows)->map($formatRow), '', - '' . "{$this->iterations} iterations x {$this->rounds} rounds per benchmark" - . ($this->option('filter-outliers') ? ' — outliers excluded (IQR)' : '') + '' . "{$this->iterations} iterations x {$this->rounds} rounds per benchmark, {$totalDuration}s total" + . ($this->option('filter-outliers') ? " — {$this->filteredRounds} outlier rounds excluded (IQR)" : '') . ($snapshot ? ' — compared against baseline snapshot' : '') . '', ])->implode("\n"); @@ -292,14 +298,14 @@ protected function getBenchmarks(): array } /** - * Remove outliers using the Interquartile Range (IQR) method. + * Return the indices of non-outlier values using the IQR method. * - * Values below Q1 - 1.5*IQR or above Q3 + 1.5*IQR are excluded. + * Values below Q1 - 1.5*IQR or above Q3 + 1.5*IQR are considered outliers. */ - protected function filterOutliers(\Illuminate\Support\Collection $values): \Illuminate\Support\Collection + protected function nonOutlierIndices(\Illuminate\Support\Collection $values): \Illuminate\Support\Collection { if ($values->count() < 4) { - return $values; + return $values->keys(); } $sorted = $values->sort()->values(); @@ -312,7 +318,7 @@ protected function filterOutliers(\Illuminate\Support\Collection $values): \Illu $lower = $q1 - 1.5 * $iqr; $upper = $q3 + 1.5 * $iqr; - return $values->filter(fn ($v) => $v >= $lower && $v <= $upper)->values(); + return $values->keys()->filter(fn ($i) => $values[$i] >= $lower && $values[$i] <= $upper)->values(); } protected function renderView(string $view): string diff --git a/workbench/app/Console/Commands/BenchmarkVarianceCommand.php b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php index 61874274..d7aa434a 100644 --- a/workbench/app/Console/Commands/BenchmarkVarianceCommand.php +++ b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php @@ -4,6 +4,7 @@ use Illuminate\Console\Command; use Illuminate\Support\Facades\Artisan; +use Illuminate\Support\Str; class BenchmarkVarianceCommand extends BenchmarkCommand { @@ -13,7 +14,8 @@ class BenchmarkVarianceCommand extends BenchmarkCommand {--rounds=100 : Number of timed rounds per benchmark} {--warmup=2 : Number of untimed warmup rounds} {--filter-outliers : Exclude outlier rounds using the IQR method} - {--ci : Suppress progress bars}'; + {--json : Output results as JSON} + {--ci : Output a markdown table with no progress (for CI)}'; protected $description = 'Run benchmarks multiple times and report variance (min/max/avg) with change deltas'; @@ -31,32 +33,79 @@ public function handle(): int } $totalRuns = $runs + 1; + $quiet = $this->option('json') || $this->option('ci'); + $commandStart = microtime(true); + + // Always suppress inner progress bars. + $this->input->setOption('ci', true); + + $runDurations = []; + + if (! $quiet) { + $bar = $this->output->createProgressBar($totalRuns); + $bar->setFormat(' %current%/%max% [%bar%] %message%'); + $bar->setMessage('Snapshot...'); + $bar->start(); + } Artisan::call('view:clear'); // Step 1: Snapshot run - $this->info("Run 1/{$totalRuns}: Creating baseline snapshot..."); + $t = microtime(true); $snapshotResults = $this->runBenchmarks(); + $runDurations[] = microtime(true) - $t; $this->saveSnapshot($snapshotResults); + if (! $quiet) { + $bar->advance(); + $bar->setMessage('Benchmarking...'); + } + // Step 2: Benchmark runs $allRuns = []; for ($i = 0; $i < $runs; $i++) { - $this->newLine(); - $this->info('Run '.($i + 2)."/{$totalRuns}: Benchmarking..."); Artisan::call('view:clear'); + $t = microtime(true); $allRuns[] = $this->runBenchmarks(); + $runDurations[] = microtime(true) - $t; + + if (! $quiet) { + $bar->advance(); + } } + if (! $quiet) { + $bar->setMessage('Done!'); + $bar->finish(); + $this->newLine(); + } + + $avgRunDuration = round(array_sum($runDurations) / count($runDurations), 2); + $totalDuration = round(microtime(true) - $commandStart, 2); + // Step 3: Display variance report - $this->displayVarianceResults($snapshotResults, $allRuns); + if ($this->option('json')) { + $this->outputJson($snapshotResults, $allRuns, $avgRunDuration, $totalDuration); + } elseif ($this->option('ci')) { + $this->outputVarianceMarkdown($snapshotResults, $allRuns, $avgRunDuration, $totalDuration); + } else { + $this->displayVarianceResults($snapshotResults, $allRuns, $avgRunDuration, $totalDuration); + } return Command::SUCCESS; } - protected function displayVarianceResults(array $snapshot, array $allRuns): void + protected function displayVarianceResults(array $snapshot, array $allRuns, float $avgRunDuration, float $totalDuration): void { + $stddev = function ($values) { + $count = $values->count(); + if ($count < 2) return 0.0; + $mean = $values->avg(); + $sumSquares = $values->reduce(fn ($carry, $v) => $carry + ($v - $mean) ** 2, 0); + return round(sqrt($sumSquares / ($count - 1)), 2); + }; + $benchmarkNames = array_keys($snapshot); $headers = ['', 'Blade', 'Blaze', 'Improvement']; $rows = []; @@ -81,6 +130,13 @@ protected function displayVarianceResults(array $snapshot, array $allRuns): void $this->formatVarianceRange($blazeChanges->min(), $blazeChanges->max()), $this->formatVarianceRange($improvementChanges->min(), $improvementChanges->max()), ]; + + $rows[] = [ + 'Std Dev', + '±'.$stddev($bladeChanges).'%', + '±'.$stddev($blazeChanges).'%', + '±'.$stddev($improvementChanges).'%', + ]; } $this->newLine(2); @@ -88,12 +144,144 @@ protected function displayVarianceResults(array $snapshot, array $allRuns): void $this->newLine(); $this->comment( - "{$this->iterations} iterations x {$this->rounds} rounds per benchmark, " - .count($allRuns).' runs' + count($allRuns)." runs x {$this->rounds} rounds x {$this->iterations} iterations" .($this->option('filter-outliers') ? ' (outliers excluded)' : '') + .", ~{$avgRunDuration}s/run, {$totalDuration}s total" + ); + } + + protected function outputVarianceMarkdown(array $snapshot, array $allRuns, float $avgRunDuration, float $totalDuration): void + { + $stddev = function ($values) { + $count = $values->count(); + if ($count < 2) return 0.0; + $mean = $values->avg(); + $sumSquares = $values->reduce(fn ($carry, $v) => $carry + ($v - $mean) ** 2, 0); + return round(sqrt($sumSquares / ($count - 1)), 2); + }; + + $benchmarkNames = array_keys($snapshot); + $headers = ['', 'Blade', 'Blaze', 'Improvement']; + $rows = []; + + foreach ($benchmarkNames as $name) { + $snapshotImprovement = $this->improvement($snapshot[$name]); + + $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot[$name]['blade_ms'], $run[$name]['blade_ms'])); + $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot[$name]['blaze_ms'], $run[$name]['blaze_ms'])); + $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run[$name]) - $snapshotImprovement, 1)); + + $rows[] = [ + 'Snapshot', + $this->formatTime($snapshot[$name]['blade_ms']), + $this->formatTime($snapshot[$name]['blaze_ms']), + $snapshotImprovement . '%', + ]; + + $rows[] = [ + 'Variance', + $this->formatVarianceRange($bladeChanges->min(), $bladeChanges->max()), + $this->formatVarianceRange($blazeChanges->min(), $blazeChanges->max()), + $this->formatVarianceRange($improvementChanges->min(), $improvementChanges->max()), + ]; + + $rows[] = [ + 'Std Dev', + '±' . $stddev($bladeChanges) . '%', + '±' . $stddev($blazeChanges) . '%', + '±' . $stddev($improvementChanges) . '%', + ]; + } + + $allRows = collect([$headers, ...$rows]); + $widths = collect($headers)->keys()->map( + fn ($i) => $allRows->max(fn ($row) => mb_strlen($row[$i])) + ); + + $formatRow = fn ($cells) => '| ' . collect($cells) + ->map(fn ($cell, $i) => Str::padRight($cell, $widths[$i])) + ->implode(' | ') . ' |'; + + $separator = '| ' . $widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ') . ' |'; + + $md = collect([ + '## Benchmark Variance Results', + '', + $formatRow($headers), + $separator, + ...collect($rows)->map($formatRow), + '', + '' . count($allRuns) . " runs x {$this->rounds} rounds x {$this->iterations} iterations" + . ($this->option('filter-outliers') ? " — outliers excluded" : '') + . ", ~{$avgRunDuration}s/run, {$totalDuration}s total" + . '', + ])->implode("\n"); + + $this->output->writeln($md); + } + + protected function saveSnapshot(array $results): void + { + $snapshot = [ + 'iterations' => $this->iterations, + 'rounds' => $this->rounds, + 'benchmarks' => collect($results)->map(fn ($result) => [ + 'blade_ms' => $result['blade_ms'], + 'blaze_ms' => $result['blaze_ms'], + 'improvement' => $this->improvement($result), + ])->all(), + ]; + + \Illuminate\Support\Facades\File::put( + $this->snapshotPath(), + json_encode($snapshot, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)."\n" ); } + protected function outputJson(array $snapshot, array $allRuns, float $avgRunDuration, float $totalDuration): void + { + $benchmarks = []; + + foreach (array_keys($snapshot) as $name) { + $snapshotImprovement = $this->improvement($snapshot[$name]); + + $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot[$name]['blade_ms'], $run[$name]['blade_ms'])); + $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot[$name]['blaze_ms'], $run[$name]['blaze_ms'])); + $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run[$name]) - $snapshotImprovement, 1)); + + $stddev = function ($values) { + $count = $values->count(); + if ($count < 2) return 0.0; + $mean = $values->avg(); + $sumSquares = $values->reduce(fn ($carry, $v) => $carry + ($v - $mean) ** 2, 0); + return round(sqrt($sumSquares / ($count - 1)), 2); + }; + + $benchmarks[$name] = [ + 'snapshot' => [ + 'blade_ms' => $snapshot[$name]['blade_ms'], + 'blaze_ms' => $snapshot[$name]['blaze_ms'], + 'improvement' => $snapshotImprovement, + ], + 'variance' => [ + 'blade' => ['min' => $bladeChanges->min(), 'max' => $bladeChanges->max(), 'stddev' => $stddev($bladeChanges)], + 'blaze' => ['min' => $blazeChanges->min(), 'max' => $blazeChanges->max(), 'stddev' => $stddev($blazeChanges)], + 'improvement' => ['min' => $improvementChanges->min(), 'max' => $improvementChanges->max(), 'stddev' => $stddev($improvementChanges)], + ], + ]; + } + + $this->output->writeln(json_encode([ + 'iterations' => $this->iterations, + 'rounds' => $this->rounds, + 'runs' => count($allRuns), + 'avg_run_duration_s' => $avgRunDuration, + 'total_duration_s' => $totalDuration, + 'filter_outliers' => (bool) $this->option('filter-outliers'), + 'benchmarks' => $benchmarks, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); + } + protected function formatVarianceRange(float $min, float $max): string { $fmt = function (float $v): string { From e214fe8a61354aa062fbbdd4eec2fa0b275b0bd3 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 01:01:15 +0100 Subject: [PATCH 15/38] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a880f8d5..0283d308 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,7 @@ jobs: run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md - name: Run benchmark - run: vendor/bin/testbench benchmark:variance --ci --iterations=2500 --rounds=100 --filter-outliers >> benchmark-result.md + run: vendor/bin/testbench benchmark:variance --ci --iterations=10000 --rounds=100 --filter-outliers >> benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 From 2b4e9c8e71d68ed20308428deecf4bb805aa010a Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 01:16:37 +0100 Subject: [PATCH 16/38] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0283d308..0ad459cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,7 @@ jobs: run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md - name: Run benchmark - run: vendor/bin/testbench benchmark:variance --ci --iterations=10000 --rounds=100 --filter-outliers >> benchmark-result.md + run: vendor/bin/testbench benchmark:variance --ci --runs=25 --iterations=5000 --rounds=100 --filter-outliers >> benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 From ade220d42f26f8f635f22d114b7e3e48d576dfb7 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 01:46:40 +0100 Subject: [PATCH 17/38] Update ci.yml --- .github/workflows/ci.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ad459cd..75dd518e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,8 +62,8 @@ jobs: steps: - name: Checkout base branch uses: actions/checkout@v4 - # with: - # ref: ${{ github.event.pull_request.base.sha }} + with: + ref: ${{ github.event.pull_request.base.sha }} - name: Setup PHP uses: shivammathur/setup-php@v2 @@ -76,26 +76,26 @@ jobs: - name: Install base dependencies run: composer install --prefer-dist --no-progress --no-interaction - # - name: Generate baseline snapshot - # run: vendor/bin/testbench benchmark --snapshot --ci --iterations=5000 --rounds=15 --warmup=2 + - name: Generate baseline snapshot + run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=100 --snapshot - # - name: Save baseline snapshot - # run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json + - name: Save baseline snapshot + run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json - # - name: Checkout PR - # uses: actions/checkout@v4 + - name: Checkout PR + uses: actions/checkout@v4 - # - name: Install PR dependencies - # run: composer install --prefer-dist --no-progress --no-interaction + - name: Install PR dependencies + run: composer install --prefer-dist --no-progress --no-interaction - # - name: Restore baseline snapshot - # run: cp ${{ runner.temp }}/benchmark-snapshot.json benchmark-snapshot.json + - name: Restore baseline snapshot + run: cp ${{ runner.temp }}/benchmark-snapshot.json benchmark-snapshot.json - name: Save PR number run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md - name: Run benchmark - run: vendor/bin/testbench benchmark:variance --ci --runs=25 --iterations=5000 --rounds=100 --filter-outliers >> benchmark-result.md + run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=100 --filter-outliers >> benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 From fe366d1cb9cceac0dd474b8e5c48ce867b82dc0d Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 01:51:23 +0100 Subject: [PATCH 18/38] Update commands --- .../app/Console/Commands/BenchmarkCommand.php | 46 +++++++++++-------- .../Commands/BenchmarkVarianceCommand.php | 7 +-- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index 314397f1..ad4ce27f 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -12,10 +12,9 @@ class BenchmarkCommand extends Command { protected $signature = 'benchmark - {--iterations=2500 : Number of component renders per benchmark} + {--iterations=5000 : Number of component renders per benchmark} {--rounds=100 : Number of timed rounds per benchmark} {--warmup=2 : Number of untimed warmup rounds} - {--filter-outliers : Exclude outlier rounds using the IQR method} {--snapshot : Save results as the baseline snapshot} {--ci : Output a markdown table with no progress (for CI)}'; @@ -98,18 +97,16 @@ protected function runBenchmarks(): array $this->newLine(2); } - if ($this->option('filter-outliers')) { - $roundTotals = collect(range(0, $this->rounds - 1))->map( - fn ($r) => collect($names)->sum(fn ($name) => $bladeTimes[$name][$r] + $blazeTimes[$name][$r]) - ); + $roundTotals = collect(range(0, $this->rounds - 1))->map( + fn ($r) => collect($names)->sum(fn ($name) => $bladeTimes[$name][$r] + $blazeTimes[$name][$r]) + ); - $keptRounds = $this->nonOutlierIndices($roundTotals); - $this->filteredRounds = $this->rounds - $keptRounds->count(); + $keptRounds = $this->nonOutlierIndices($roundTotals); + $this->filteredRounds = $this->rounds - $keptRounds->count(); - foreach ($names as $name) { - $bladeTimes[$name] = $keptRounds->map(fn ($r) => $bladeTimes[$name][$r])->all(); - $blazeTimes[$name] = $keptRounds->map(fn ($r) => $blazeTimes[$name][$r])->all(); - } + foreach ($names as $name) { + $bladeTimes[$name] = $keptRounds->map(fn ($r) => $bladeTimes[$name][$r])->all(); + $blazeTimes[$name] = $keptRounds->map(fn ($r) => $blazeTimes[$name][$r])->all(); } return collect($names)->mapWithKeys(fn ($name) => [ @@ -132,8 +129,9 @@ protected function buildTable(array $results): array $improvement = $this->improvement($result) . '%'; if ($prev = $snapshot['benchmarks'][$name] ?? null) { - $blade .= ' ' . $this->formatChange($prev['blade_ms'], $result['blade_ms']); - $blaze .= ' ' . $this->formatChange($prev['blaze_ms'], $result['blaze_ms']); + $blade .= ' ' . $this->formatChange($prev['blade_ms'], $result['blade_ms'], 1); + $blaze .= ' ' . $this->formatChange($prev['blaze_ms'], $result['blaze_ms'], 1); + $improvement .= ' ' . $this->formatImprovementChange($prev['improvement'], $this->improvement($result)); } return [$name, $blade, $blaze, $improvement]; @@ -151,10 +149,7 @@ protected function displayResults(array $results, float $totalDuration): void $this->newLine(); $this->info("{$this->iterations} iterations x {$this->rounds} rounds per benchmark, {$totalDuration}s total"); - - if ($this->option('filter-outliers')) { - $this->comment("{$this->filteredRounds} outlier rounds excluded (IQR method)"); - } + $this->comment("{$this->filteredRounds} outlier rounds excluded (IQR method)"); if ($snapshot) { $rounds = $snapshot['rounds'] ?? 1; @@ -185,7 +180,7 @@ protected function outputMarkdown(array $results, float $totalDuration): void ...collect($rows)->map($formatRow), '', '' . "{$this->iterations} iterations x {$this->rounds} rounds per benchmark, {$totalDuration}s total" - . ($this->option('filter-outliers') ? " — {$this->filteredRounds} outlier rounds excluded (IQR)" : '') + . " — {$this->filteredRounds} outlier rounds excluded (IQR)" . ($snapshot ? ' — compared against baseline snapshot' : '') . '', ])->implode("\n"); @@ -255,6 +250,19 @@ protected function formatChange(float $old, float $new, float $threshold = 0.1): return "({$sign}" . round($change, 1) . '%)'; } + protected function formatImprovementChange(float $old, float $new, float $threshold = 0.1): string + { + $delta = round($new - $old, 1); + + if (abs($delta) < $threshold) { + return '(~)'; + } + + $sign = $delta > 0 ? '+' : ''; + + return "({$sign}{$delta}%)"; + } + protected function getBenchmarks(): array { return [ diff --git a/workbench/app/Console/Commands/BenchmarkVarianceCommand.php b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php index d7aa434a..af1976f2 100644 --- a/workbench/app/Console/Commands/BenchmarkVarianceCommand.php +++ b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php @@ -10,10 +10,9 @@ class BenchmarkVarianceCommand extends BenchmarkCommand { protected $signature = 'benchmark:variance {--runs=5 : Number of benchmark runs after the initial snapshot run} - {--iterations=2500 : Number of component renders per benchmark} + {--iterations=5000 : Number of component renders per benchmark} {--rounds=100 : Number of timed rounds per benchmark} {--warmup=2 : Number of untimed warmup rounds} - {--filter-outliers : Exclude outlier rounds using the IQR method} {--json : Output results as JSON} {--ci : Output a markdown table with no progress (for CI)}'; @@ -145,7 +144,6 @@ protected function displayVarianceResults(array $snapshot, array $allRuns, float $this->newLine(); $this->comment( count($allRuns)." runs x {$this->rounds} rounds x {$this->iterations} iterations" - .($this->option('filter-outliers') ? ' (outliers excluded)' : '') .", ~{$avgRunDuration}s/run, {$totalDuration}s total" ); } @@ -212,7 +210,6 @@ protected function outputVarianceMarkdown(array $snapshot, array $allRuns, float ...collect($rows)->map($formatRow), '', '' . count($allRuns) . " runs x {$this->rounds} rounds x {$this->iterations} iterations" - . ($this->option('filter-outliers') ? " — outliers excluded" : '') . ", ~{$avgRunDuration}s/run, {$totalDuration}s total" . '', ])->implode("\n"); @@ -277,7 +274,7 @@ protected function outputJson(array $snapshot, array $allRuns, float $avgRunDura 'runs' => count($allRuns), 'avg_run_duration_s' => $avgRunDuration, 'total_duration_s' => $totalDuration, - 'filter_outliers' => (bool) $this->option('filter-outliers'), + 'filter_outliers' => true, 'benchmarks' => $benchmarks, ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); } From 64742353c2e7ec18619d55763b147815109eee2e Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 01:59:11 +0100 Subject: [PATCH 19/38] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75dd518e..6fb5708b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,7 @@ jobs: run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md - name: Run benchmark - run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=100 --filter-outliers >> benchmark-result.md + run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=100 >> benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 From 5e6a63eeb516f8e2335fbb7fbb23a139c2cd2c8f Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 02:04:45 +0100 Subject: [PATCH 20/38] Update ci.yml --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fb5708b..a483280a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,6 +79,11 @@ jobs: - name: Generate baseline snapshot run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=100 --snapshot + - name: Checkout BenchmarkCommand from PR + run: | + git fetch origin --depth=1 ${{ github.event.pull_request.head.sha }} + git checkout ${{ github.event.pull_request.head.sha }} -- workbench/app/Console/Commands/BenchmarkCommand.php + - name: Save baseline snapshot run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json From 4fe082b30f5823fb55d3a28bcbc4864f0c4d38c5 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 02:06:48 +0100 Subject: [PATCH 21/38] Update ci.yml --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a483280a..ee6e2f04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,11 @@ jobs: with: ref: ${{ github.event.pull_request.base.sha }} + - name: Checkout BenchmarkCommand from PR (for testing on this branch) + run: | + git fetch origin --depth=1 ${{ github.event.pull_request.head.sha }} + git checkout ${{ github.event.pull_request.head.sha }} -- workbench/app/Console/Commands/BenchmarkCommand.php + - name: Setup PHP uses: shivammathur/setup-php@v2 with: @@ -79,11 +84,6 @@ jobs: - name: Generate baseline snapshot run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=100 --snapshot - - name: Checkout BenchmarkCommand from PR - run: | - git fetch origin --depth=1 ${{ github.event.pull_request.head.sha }} - git checkout ${{ github.event.pull_request.head.sha }} -- workbench/app/Console/Commands/BenchmarkCommand.php - - name: Save baseline snapshot run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json From d196f10c21d8a56b2221b09dacbc229a1e618356 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 02:37:43 +0100 Subject: [PATCH 22/38] Run benchmarks in separate processes --- .github/workflows/ci.yml | 38 +++++----- .../app/Console/Commands/BenchmarkCommand.php | 42 +++++++++-- .../Commands/BenchmarkVarianceCommand.php | 74 ++++++++++++++----- 3 files changed, 113 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee6e2f04..ad4d93d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,13 +62,13 @@ jobs: steps: - name: Checkout base branch uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.base.sha }} + # with: + # ref: ${{ github.event.pull_request.base.sha }} - - name: Checkout BenchmarkCommand from PR (for testing on this branch) - run: | - git fetch origin --depth=1 ${{ github.event.pull_request.head.sha }} - git checkout ${{ github.event.pull_request.head.sha }} -- workbench/app/Console/Commands/BenchmarkCommand.php + # - name: Checkout BenchmarkCommand from PR (for testing on this branch) + # run: | + # git fetch origin --depth=1 ${{ github.event.pull_request.head.sha }} + # git checkout ${{ github.event.pull_request.head.sha }} -- workbench/app/Console/Commands/BenchmarkCommand.php - name: Setup PHP uses: shivammathur/setup-php@v2 @@ -81,26 +81,26 @@ jobs: - name: Install base dependencies run: composer install --prefer-dist --no-progress --no-interaction - - name: Generate baseline snapshot - run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=100 --snapshot + # - name: Generate baseline snapshot + # run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=100 --snapshot - - name: Save baseline snapshot - run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json + # - name: Save baseline snapshot + # run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json - - name: Checkout PR - uses: actions/checkout@v4 + # - name: Checkout PR + # uses: actions/checkout@v4 - - name: Install PR dependencies - run: composer install --prefer-dist --no-progress --no-interaction + # - name: Install PR dependencies + # run: composer install --prefer-dist --no-progress --no-interaction - - name: Restore baseline snapshot - run: cp ${{ runner.temp }}/benchmark-snapshot.json benchmark-snapshot.json + # - name: Restore baseline snapshot + # run: cp ${{ runner.temp }}/benchmark-snapshot.json benchmark-snapshot.json - - name: Save PR number - run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md + # - name: Save PR number + # run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md - name: Run benchmark - run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=100 >> benchmark-result.md + run: vendor/bin/testbench benchmark:variance --ci --runs=10 --iterations=5000 --rounds=100 >> benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index ad4ce27f..8efb331d 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -16,6 +16,8 @@ class BenchmarkCommand extends Command {--rounds=100 : Number of timed rounds per benchmark} {--warmup=2 : Number of untimed warmup rounds} {--snapshot : Save results as the baseline snapshot} + {--json : Output results as JSON} + {--only= : Run only the named benchmark} {--ci : Output a markdown table with no progress (for CI)}'; protected $description = 'Run Blaze performance benchmarks'; @@ -40,9 +42,13 @@ public function handle(): int $results = $this->runBenchmarks(); $totalDuration = round(microtime(true) - $commandStart, 2); - $this->option('ci') - ? $this->outputMarkdown($results, $totalDuration) - : $this->displayResults($results, $totalDuration); + if ($this->option('json')) { + $this->outputJsonResults($results, $totalDuration); + } elseif ($this->option('ci')) { + $this->outputMarkdown($results, $totalDuration); + } else { + $this->displayResults($results, $totalDuration); + } if ($this->option('snapshot')) { $this->saveSnapshot($results); @@ -53,8 +59,8 @@ public function handle(): int protected function runBenchmarks(): array { - $showProgress = ! $this->option('ci'); - $benchmarks = $this->getBenchmarks(); + $showProgress = ! $this->option('ci') && ! $this->option('json'); + $benchmarks = $this->getFilteredBenchmarks(); $names = array_keys($benchmarks); if ($showProgress) { @@ -263,6 +269,32 @@ protected function formatImprovementChange(float $old, float $new, float $thresh return "({$sign}{$delta}%)"; } + protected function getFilteredBenchmarks(): array + { + $benchmarks = $this->getBenchmarks(); + + if ($this->input->hasOption('only') && ($only = $this->option('only'))) { + if (! isset($benchmarks[$only])) { + throw new \InvalidArgumentException("Unknown benchmark: {$only}"); + } + + return [$only => $benchmarks[$only]]; + } + + return $benchmarks; + } + + protected function outputJsonResults(array $results, float $totalDuration): void + { + $this->output->writeln(json_encode([ + 'iterations' => $this->iterations, + 'rounds' => $this->rounds, + 'filtered_rounds' => $this->filteredRounds, + 'total_duration_s' => $totalDuration, + 'benchmarks' => $results, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); + } + protected function getBenchmarks(): array { return [ diff --git a/workbench/app/Console/Commands/BenchmarkVarianceCommand.php b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php index af1976f2..06be704b 100644 --- a/workbench/app/Console/Commands/BenchmarkVarianceCommand.php +++ b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php @@ -3,7 +3,7 @@ namespace Workbench\App\Console\Commands; use Illuminate\Console\Command; -use Illuminate\Support\Facades\Artisan; +use Illuminate\Support\Facades\Process; use Illuminate\Support\Str; class BenchmarkVarianceCommand extends BenchmarkCommand @@ -35,43 +35,53 @@ public function handle(): int $quiet = $this->option('json') || $this->option('ci'); $commandStart = microtime(true); - // Always suppress inner progress bars. - $this->input->setOption('ci', true); - + $benchmarkNames = array_keys($this->getBenchmarks()); + $totalSteps = $totalRuns * count($benchmarkNames); $runDurations = []; if (! $quiet) { - $bar = $this->output->createProgressBar($totalRuns); + $bar = $this->output->createProgressBar($totalSteps); $bar->setFormat(' %current%/%max% [%bar%] %message%'); $bar->setMessage('Snapshot...'); $bar->start(); } - Artisan::call('view:clear'); - - // Step 1: Snapshot run + // Step 1: Snapshot run (each benchmark in its own process) $t = microtime(true); - $snapshotResults = $this->runBenchmarks(); + $snapshotResults = []; + + foreach ($benchmarkNames as $name) { + $snapshotResults[$name] = $this->runBenchmarkInProcess($name); + + if (! $quiet) { + $bar->advance(); + } + } + $runDurations[] = microtime(true) - $t; $this->saveSnapshot($snapshotResults); if (! $quiet) { - $bar->advance(); $bar->setMessage('Benchmarking...'); } - // Step 2: Benchmark runs + // Step 2: Benchmark runs (each benchmark in its own process) $allRuns = []; for ($i = 0; $i < $runs; $i++) { - Artisan::call('view:clear'); $t = microtime(true); - $allRuns[] = $this->runBenchmarks(); - $runDurations[] = microtime(true) - $t; + $runResults = []; - if (! $quiet) { - $bar->advance(); + foreach ($benchmarkNames as $name) { + $runResults[$name] = $this->runBenchmarkInProcess($name); + + if (! $quiet) { + $bar->advance(); + } } + + $allRuns[] = $runResults; + $runDurations[] = microtime(true) - $t; } if (! $quiet) { @@ -95,6 +105,36 @@ public function handle(): int return Command::SUCCESS; } + protected function runBenchmarkInProcess(string $name): array + { + $result = Process::path(base_path()) + ->timeout(300) + ->run([ + PHP_BINARY, 'artisan', 'benchmark', + '--only='.$name, + '--json', + '--iterations='.$this->iterations, + '--rounds='.$this->rounds, + '--warmup='.$this->warmupRounds, + ]); + + if (! $result->successful()) { + throw new \RuntimeException( + "Benchmark process failed for '{$name}': ".$result->errorOutput() + ); + } + + $data = json_decode($result->output(), true); + + if (! $data || ! isset($data['benchmarks'][$name])) { + throw new \RuntimeException( + "Invalid benchmark output for '{$name}': ".$result->output() + ); + } + + return $data['benchmarks'][$name]; + } + protected function displayVarianceResults(array $snapshot, array $allRuns, float $avgRunDuration, float $totalDuration): void { $stddev = function ($values) { @@ -203,7 +243,7 @@ protected function outputVarianceMarkdown(array $snapshot, array $allRuns, float $separator = '| ' . $widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ') . ' |'; $md = collect([ - '## Benchmark Variance Results', + '## Benchmark Results', '', $formatRow($headers), $separator, From c12c63c3d1740a9297a3c0d42eab2f6d67564e9a Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 09:27:11 +0100 Subject: [PATCH 23/38] Update ci.yml --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad4d93d4..555ea625 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,11 +96,11 @@ jobs: # - name: Restore baseline snapshot # run: cp ${{ runner.temp }}/benchmark-snapshot.json benchmark-snapshot.json - # - name: Save PR number - # run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md + - name: Save PR number + run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md - name: Run benchmark - run: vendor/bin/testbench benchmark:variance --ci --runs=10 --iterations=5000 --rounds=100 >> benchmark-result.md + run: vendor/bin/testbench benchmark:variance --ci --runs=25 --iterations=2500 --rounds=200 >> benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 From 513661d3555a6def2e1467dd3982471387bd4bd1 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 10:16:21 +0100 Subject: [PATCH 24/38] Update ci.yml --- .github/workflows/ci.yml | 92 +++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 555ea625..b6041fea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,17 +58,20 @@ jobs: needs: pest if: github.event_name == 'pull_request' runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - rounds: 50 + iterations: 10000 + - rounds: 200 + iterations: 2500 + - rounds: 1000 + iterations: 500 steps: - - name: Checkout base branch + - name: Checkout uses: actions/checkout@v4 - # with: - # ref: ${{ github.event.pull_request.base.sha }} - - # - name: Checkout BenchmarkCommand from PR (for testing on this branch) - # run: | - # git fetch origin --depth=1 ${{ github.event.pull_request.head.sha }} - # git checkout ${{ github.event.pull_request.head.sha }} -- workbench/app/Console/Commands/BenchmarkCommand.php - name: Setup PHP uses: shivammathur/setup-php@v2 @@ -78,32 +81,61 @@ jobs: coverage: none extensions: mbstring, dom, curl, json, libxml, xml, xmlwriter, simplexml, tokenizer - - name: Install base dependencies + - name: Install dependencies run: composer install --prefer-dist --no-progress --no-interaction - # - name: Generate baseline snapshot - # run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=100 --snapshot - - # - name: Save baseline snapshot - # run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json - - # - name: Checkout PR - # uses: actions/checkout@v4 - - # - name: Install PR dependencies - # run: composer install --prefer-dist --no-progress --no-interaction - - # - name: Restore baseline snapshot - # run: cp ${{ runner.temp }}/benchmark-snapshot.json benchmark-snapshot.json - - - name: Save PR number - run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md - - - name: Run benchmark - run: vendor/bin/testbench benchmark:variance --ci --runs=25 --iterations=2500 --rounds=200 >> benchmark-result.md + - name: Run benchmark (${{ matrix.rounds }} rounds x ${{ matrix.iterations }} iterations) + run: vendor/bin/testbench benchmark:variance --ci --runs=25 --iterations=${{ matrix.iterations }} --rounds=${{ matrix.rounds }} > benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 with: - name: benchmark-result + name: benchmark-result-${{ matrix.rounds }}-${{ matrix.iterations }} path: benchmark-result.md + + benchmark-comment: + needs: benchmark + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + actions: read + pull-requests: write + + steps: + - name: Download all benchmark results + uses: actions/download-artifact@v4 + with: + pattern: benchmark-result-* + + - name: Assemble comment + run: | + echo "## Benchmark Variance Results" > comment.md + echo "" >> comment.md + + for config in "50-10000" "200-2500" "1000-500"; do + rounds="${config%%-*}" + iterations="${config##*-}" + dir="benchmark-result-${config}" + + echo "### ${rounds} rounds x ${iterations} iterations" >> comment.md + echo "" >> comment.md + # Strip the "## Benchmark Results" heading line from each result + sed '/^## Benchmark Results$/d' "$dir/benchmark-result.md" >> comment.md + echo "" >> comment.md + done + + - name: Find existing comment + uses: peter-evans/find-comment@v3 + id: find + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: '## Benchmark Variance Results' + + - name: Post or update comment + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-id: ${{ steps.find.outputs.comment-id }} + edit-mode: replace + body-path: comment.md From 51765f427bff970c43694e95cee6642aa14792e2 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 10:23:50 +0100 Subject: [PATCH 25/38] Update ci.yml --- .github/workflows/ci.yml | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6041fea..fc7d0e1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,13 +61,8 @@ jobs: strategy: fail-fast: false matrix: - include: - - rounds: 50 - iterations: 10000 - - rounds: 200 - iterations: 2500 - - rounds: 1000 - iterations: 500 + config: ["50-10000", "200-2500", "1000-500"] + repeat: [1, 2, 3] steps: - name: Checkout @@ -84,13 +79,18 @@ jobs: - name: Install dependencies run: composer install --prefer-dist --no-progress --no-interaction - - name: Run benchmark (${{ matrix.rounds }} rounds x ${{ matrix.iterations }} iterations) - run: vendor/bin/testbench benchmark:variance --ci --runs=25 --iterations=${{ matrix.iterations }} --rounds=${{ matrix.rounds }} > benchmark-result.md + - name: Run benchmark (${{ matrix.config }}, run ${{ matrix.repeat }}/3) + run: | + config="${{ matrix.config }}" + rounds="${config%%-*}" + iterations="${config##*-}" + + vendor/bin/testbench benchmark:variance --ci --runs=25 --iterations="$iterations" --rounds="$rounds" > benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 with: - name: benchmark-result-${{ matrix.rounds }}-${{ matrix.iterations }} + name: benchmark-result-${{ matrix.config }}-${{ matrix.repeat }} path: benchmark-result.md benchmark-comment: @@ -111,17 +111,25 @@ jobs: run: | echo "## Benchmark Variance Results" > comment.md echo "" >> comment.md + echo "Each configuration is executed 3 times." >> comment.md + echo "" >> comment.md for config in "50-10000" "200-2500" "1000-500"; do rounds="${config%%-*}" iterations="${config##*-}" - dir="benchmark-result-${config}" echo "### ${rounds} rounds x ${iterations} iterations" >> comment.md echo "" >> comment.md - # Strip the "## Benchmark Results" heading line from each result - sed '/^## Benchmark Results$/d' "$dir/benchmark-result.md" >> comment.md - echo "" >> comment.md + + for repeat in 1 2 3; do + dir="benchmark-result-${config}-${repeat}" + + echo "#### Run ${repeat}/3" >> comment.md + echo "" >> comment.md + # Strip the "## Benchmark Results" heading line from each result + sed '/^## Benchmark Results$/d' "$dir/benchmark-result.md" >> comment.md + echo "" >> comment.md + done done - name: Find existing comment From f0791661f4024e10f41e13e0da15be11aa25e4a5 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 12:29:24 +0100 Subject: [PATCH 26/38] Make rounds parallel --- composer.json | 5 +- .../app/Console/Commands/BenchmarkCommand.php | 132 +++++++++++++++--- 2 files changed, 117 insertions(+), 20 deletions(-) diff --git a/composer.json b/composer.json index 887719cc..72084a85 100644 --- a/composer.json +++ b/composer.json @@ -18,7 +18,8 @@ "livewire/flux": "dev-main", "orchestra/testbench": "^8.0|^9.0|^10.0", "pestphp/pest": "^2.0|^3.0", - "pestphp/pest-plugin-laravel": "^2.0|^3.0" + "pestphp/pest-plugin-laravel": "^2.0|^3.0", + "spatie/fork": "^1.2" }, "conflict": { "livewire/flux": "<2.12.1" @@ -54,4 +55,4 @@ "benchmark": "vendor/bin/testbench benchmark", "benchmark:snapshot": "vendor/bin/testbench benchmark --snapshot" } -} \ No newline at end of file +} diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index 8efb331d..51d9c26b 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -5,6 +5,7 @@ use Illuminate\Console\Command; use Illuminate\Support\Benchmark; use Illuminate\Support\Facades\Artisan; +use Illuminate\Support\Facades\Concurrency; use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\View; use Illuminate\Support\Str; @@ -18,6 +19,7 @@ class BenchmarkCommand extends Command {--snapshot : Save results as the baseline snapshot} {--json : Output results as JSON} {--only= : Run only the named benchmark} + {--processes= : Number of parallel processes (auto-detected if omitted)} {--ci : Output a markdown table with no progress (for CI)}'; protected $description = 'Run Blaze performance benchmarks'; @@ -30,11 +32,14 @@ class BenchmarkCommand extends Command protected int $filteredRounds = 0; + protected int $processes = 1; + public function handle(): int { $this->iterations = (int) $this->option('iterations'); $this->rounds = (int) $this->option('rounds'); $this->warmupRounds = (int) $this->option('warmup'); + $this->processes = $this->detectProcessCount(); $commandStart = microtime(true); Artisan::call('view:clear'); @@ -64,12 +69,14 @@ protected function runBenchmarks(): array $names = array_keys($benchmarks); if ($showProgress) { - $this->info("Running benchmarks ({$this->iterations} iterations x {$this->rounds} rounds)..."); + $parallel = $this->processes > 1 ? " across {$this->processes} processes" : ''; + $this->info("Running benchmarks ({$this->iterations} iterations x {$this->rounds} rounds{$parallel})..."); $this->newLine(); } - $totalSteps = (count($benchmarks) * $this->warmupRounds) + ($this->rounds * count($benchmarks)); - $bar = $showProgress ? $this->output->createProgressBar($totalSteps) : null; + $warmupSteps = count($benchmarks) * $this->warmupRounds; + $benchmarkSteps = $this->processes > 1 ? 0 : ($this->rounds * count($benchmarks)); + $bar = $showProgress ? $this->output->createProgressBar($warmupSteps + $benchmarkSteps) : null; $bar?->setFormat(' %current%/%max% [%bar%] %message%'); $bar?->setMessage('Warming up...'); $bar?->start(); @@ -83,24 +90,40 @@ protected function runBenchmarks(): array } } - $bladeTimes = array_fill_keys($names, []); - $blazeTimes = array_fill_keys($names, []); + if ($this->processes > 1) { + $bar?->setMessage('Done!'); + $bar?->finish(); - $bar?->setMessage('Benchmarking...'); + if ($showProgress) { + $this->newLine(2); + $this->comment("Forking {$this->processes} worker processes..."); + } - for ($r = 0; $r < $this->rounds; $r++) { - foreach ($benchmarks as $name => $benchmark) { - $bladeTimes[$name][] = $this->measureView($benchmark['blade']); - $blazeTimes[$name][] = $this->measureView($benchmark['blaze']); - $bar?->advance(); + [$bladeTimes, $blazeTimes] = $this->runRoundsParallel($benchmarks, $names); + + if ($showProgress) { + $this->newLine(); } - } + } else { + $bladeTimes = array_fill_keys($names, []); + $blazeTimes = array_fill_keys($names, []); - $bar?->setMessage('Done!'); - $bar?->finish(); + $bar?->setMessage('Benchmarking...'); - if ($showProgress) { - $this->newLine(2); + for ($r = 0; $r < $this->rounds; $r++) { + foreach ($benchmarks as $name => $benchmark) { + $bladeTimes[$name][] = $this->measureView($benchmark['blade']); + $blazeTimes[$name][] = $this->measureView($benchmark['blaze']); + $bar?->advance(); + } + } + + $bar?->setMessage('Done!'); + $bar?->finish(); + + if ($showProgress) { + $this->newLine(2); + } } $roundTotals = collect(range(0, $this->rounds - 1))->map( @@ -123,6 +146,75 @@ protected function runBenchmarks(): array ])->all(); } + protected function runRoundsParallel(array $benchmarks, array $names): array + { + $roundsPerProcess = intdiv($this->rounds, $this->processes); + $remainder = $this->rounds % $this->processes; + $iterations = $this->iterations; + + $tasks = []; + + for ($p = 0; $p < $this->processes; $p++) { + $workerRounds = $roundsPerProcess + ($p < $remainder ? 1 : 0); + + if ($workerRounds === 0) { + continue; + } + + $tasks[] = function () use ($benchmarks, $names, $workerRounds, $iterations) { + $bladeTimes = array_fill_keys($names, []); + $blazeTimes = array_fill_keys($names, []); + + for ($r = 0; $r < $workerRounds; $r++) { + foreach ($benchmarks as $name => $benchmark) { + $bladeTimes[$name][] = Benchmark::measure( + fn () => View::make($benchmark['blade'], ['iterations' => $iterations])->render() + ); + $blazeTimes[$name][] = Benchmark::measure( + fn () => View::make($benchmark['blaze'], ['iterations' => $iterations])->render() + ); + } + } + + return compact('bladeTimes', 'blazeTimes'); + }; + } + + $workerResults = Concurrency::driver('fork')->run($tasks); + + // Merge timing data from all workers. + $bladeTimes = array_fill_keys($names, []); + $blazeTimes = array_fill_keys($names, []); + + foreach ($workerResults as $result) { + foreach ($names as $name) { + array_push($bladeTimes[$name], ...$result['bladeTimes'][$name]); + array_push($blazeTimes[$name], ...$result['blazeTimes'][$name]); + } + } + + return [$bladeTimes, $blazeTimes]; + } + + protected function detectProcessCount(): int + { + if ($this->option('processes')) { + return max(1, (int) $this->option('processes')); + } + + if (! function_exists('pcntl_fork')) { + return 1; + } + + $cores = match (PHP_OS_FAMILY) { + 'Darwin' => (int) trim((string) shell_exec('sysctl -n hw.ncpu')), + 'Linux' => (int) trim((string) shell_exec('nproc')), + default => 1, + }; + + return max(1, min($cores, $this->rounds)); + } + protected function buildTable(array $results): array { $snapshot = $this->option('snapshot') ? null : $this->loadSnapshot(); @@ -154,7 +246,8 @@ protected function displayResults(array $results, float $totalDuration): void $this->table($headers, $rows); $this->newLine(); - $this->info("{$this->iterations} iterations x {$this->rounds} rounds per benchmark, {$totalDuration}s total"); + $parallel = $this->processes > 1 ? " across {$this->processes} processes" : ''; + $this->info("{$this->iterations} iterations x {$this->rounds} rounds per benchmark{$parallel}, {$totalDuration}s total"); $this->comment("{$this->filteredRounds} outlier rounds excluded (IQR method)"); if ($snapshot) { @@ -185,7 +278,9 @@ protected function outputMarkdown(array $results, float $totalDuration): void $separator, ...collect($rows)->map($formatRow), '', - '' . "{$this->iterations} iterations x {$this->rounds} rounds per benchmark, {$totalDuration}s total" + '' . "{$this->iterations} iterations x {$this->rounds} rounds per benchmark" + . ($this->processes > 1 ? " across {$this->processes} processes" : '') + . ", {$totalDuration}s total" . " — {$this->filteredRounds} outlier rounds excluded (IQR)" . ($snapshot ? ' — compared against baseline snapshot' : '') . '', @@ -289,6 +384,7 @@ protected function outputJsonResults(array $results, float $totalDuration): void $this->output->writeln(json_encode([ 'iterations' => $this->iterations, 'rounds' => $this->rounds, + 'processes' => $this->processes, 'filtered_rounds' => $this->filteredRounds, 'total_duration_s' => $totalDuration, 'benchmarks' => $results, From 3ba16de4a28bff96740263f9ca867a3e62bd6e83 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 12:42:28 +0100 Subject: [PATCH 27/38] Revert "Make rounds parallel" This reverts commit f0791661f4024e10f41e13e0da15be11aa25e4a5. --- composer.json | 5 +- .../app/Console/Commands/BenchmarkCommand.php | 132 +++--------------- 2 files changed, 20 insertions(+), 117 deletions(-) diff --git a/composer.json b/composer.json index 72084a85..887719cc 100644 --- a/composer.json +++ b/composer.json @@ -18,8 +18,7 @@ "livewire/flux": "dev-main", "orchestra/testbench": "^8.0|^9.0|^10.0", "pestphp/pest": "^2.0|^3.0", - "pestphp/pest-plugin-laravel": "^2.0|^3.0", - "spatie/fork": "^1.2" + "pestphp/pest-plugin-laravel": "^2.0|^3.0" }, "conflict": { "livewire/flux": "<2.12.1" @@ -55,4 +54,4 @@ "benchmark": "vendor/bin/testbench benchmark", "benchmark:snapshot": "vendor/bin/testbench benchmark --snapshot" } -} +} \ No newline at end of file diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index 51d9c26b..8efb331d 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -5,7 +5,6 @@ use Illuminate\Console\Command; use Illuminate\Support\Benchmark; use Illuminate\Support\Facades\Artisan; -use Illuminate\Support\Facades\Concurrency; use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\View; use Illuminate\Support\Str; @@ -19,7 +18,6 @@ class BenchmarkCommand extends Command {--snapshot : Save results as the baseline snapshot} {--json : Output results as JSON} {--only= : Run only the named benchmark} - {--processes= : Number of parallel processes (auto-detected if omitted)} {--ci : Output a markdown table with no progress (for CI)}'; protected $description = 'Run Blaze performance benchmarks'; @@ -32,14 +30,11 @@ class BenchmarkCommand extends Command protected int $filteredRounds = 0; - protected int $processes = 1; - public function handle(): int { $this->iterations = (int) $this->option('iterations'); $this->rounds = (int) $this->option('rounds'); $this->warmupRounds = (int) $this->option('warmup'); - $this->processes = $this->detectProcessCount(); $commandStart = microtime(true); Artisan::call('view:clear'); @@ -69,14 +64,12 @@ protected function runBenchmarks(): array $names = array_keys($benchmarks); if ($showProgress) { - $parallel = $this->processes > 1 ? " across {$this->processes} processes" : ''; - $this->info("Running benchmarks ({$this->iterations} iterations x {$this->rounds} rounds{$parallel})..."); + $this->info("Running benchmarks ({$this->iterations} iterations x {$this->rounds} rounds)..."); $this->newLine(); } - $warmupSteps = count($benchmarks) * $this->warmupRounds; - $benchmarkSteps = $this->processes > 1 ? 0 : ($this->rounds * count($benchmarks)); - $bar = $showProgress ? $this->output->createProgressBar($warmupSteps + $benchmarkSteps) : null; + $totalSteps = (count($benchmarks) * $this->warmupRounds) + ($this->rounds * count($benchmarks)); + $bar = $showProgress ? $this->output->createProgressBar($totalSteps) : null; $bar?->setFormat(' %current%/%max% [%bar%] %message%'); $bar?->setMessage('Warming up...'); $bar?->start(); @@ -90,40 +83,24 @@ protected function runBenchmarks(): array } } - if ($this->processes > 1) { - $bar?->setMessage('Done!'); - $bar?->finish(); - - if ($showProgress) { - $this->newLine(2); - $this->comment("Forking {$this->processes} worker processes..."); - } - - [$bladeTimes, $blazeTimes] = $this->runRoundsParallel($benchmarks, $names); - - if ($showProgress) { - $this->newLine(); - } - } else { - $bladeTimes = array_fill_keys($names, []); - $blazeTimes = array_fill_keys($names, []); + $bladeTimes = array_fill_keys($names, []); + $blazeTimes = array_fill_keys($names, []); - $bar?->setMessage('Benchmarking...'); + $bar?->setMessage('Benchmarking...'); - for ($r = 0; $r < $this->rounds; $r++) { - foreach ($benchmarks as $name => $benchmark) { - $bladeTimes[$name][] = $this->measureView($benchmark['blade']); - $blazeTimes[$name][] = $this->measureView($benchmark['blaze']); - $bar?->advance(); - } + for ($r = 0; $r < $this->rounds; $r++) { + foreach ($benchmarks as $name => $benchmark) { + $bladeTimes[$name][] = $this->measureView($benchmark['blade']); + $blazeTimes[$name][] = $this->measureView($benchmark['blaze']); + $bar?->advance(); } + } - $bar?->setMessage('Done!'); - $bar?->finish(); + $bar?->setMessage('Done!'); + $bar?->finish(); - if ($showProgress) { - $this->newLine(2); - } + if ($showProgress) { + $this->newLine(2); } $roundTotals = collect(range(0, $this->rounds - 1))->map( @@ -146,75 +123,6 @@ protected function runBenchmarks(): array ])->all(); } - protected function runRoundsParallel(array $benchmarks, array $names): array - { - $roundsPerProcess = intdiv($this->rounds, $this->processes); - $remainder = $this->rounds % $this->processes; - $iterations = $this->iterations; - - $tasks = []; - - for ($p = 0; $p < $this->processes; $p++) { - $workerRounds = $roundsPerProcess + ($p < $remainder ? 1 : 0); - - if ($workerRounds === 0) { - continue; - } - - $tasks[] = function () use ($benchmarks, $names, $workerRounds, $iterations) { - $bladeTimes = array_fill_keys($names, []); - $blazeTimes = array_fill_keys($names, []); - - for ($r = 0; $r < $workerRounds; $r++) { - foreach ($benchmarks as $name => $benchmark) { - $bladeTimes[$name][] = Benchmark::measure( - fn () => View::make($benchmark['blade'], ['iterations' => $iterations])->render() - ); - $blazeTimes[$name][] = Benchmark::measure( - fn () => View::make($benchmark['blaze'], ['iterations' => $iterations])->render() - ); - } - } - - return compact('bladeTimes', 'blazeTimes'); - }; - } - - $workerResults = Concurrency::driver('fork')->run($tasks); - - // Merge timing data from all workers. - $bladeTimes = array_fill_keys($names, []); - $blazeTimes = array_fill_keys($names, []); - - foreach ($workerResults as $result) { - foreach ($names as $name) { - array_push($bladeTimes[$name], ...$result['bladeTimes'][$name]); - array_push($blazeTimes[$name], ...$result['blazeTimes'][$name]); - } - } - - return [$bladeTimes, $blazeTimes]; - } - - protected function detectProcessCount(): int - { - if ($this->option('processes')) { - return max(1, (int) $this->option('processes')); - } - - if (! function_exists('pcntl_fork')) { - return 1; - } - - $cores = match (PHP_OS_FAMILY) { - 'Darwin' => (int) trim((string) shell_exec('sysctl -n hw.ncpu')), - 'Linux' => (int) trim((string) shell_exec('nproc')), - default => 1, - }; - - return max(1, min($cores, $this->rounds)); - } - protected function buildTable(array $results): array { $snapshot = $this->option('snapshot') ? null : $this->loadSnapshot(); @@ -246,8 +154,7 @@ protected function displayResults(array $results, float $totalDuration): void $this->table($headers, $rows); $this->newLine(); - $parallel = $this->processes > 1 ? " across {$this->processes} processes" : ''; - $this->info("{$this->iterations} iterations x {$this->rounds} rounds per benchmark{$parallel}, {$totalDuration}s total"); + $this->info("{$this->iterations} iterations x {$this->rounds} rounds per benchmark, {$totalDuration}s total"); $this->comment("{$this->filteredRounds} outlier rounds excluded (IQR method)"); if ($snapshot) { @@ -278,9 +185,7 @@ protected function outputMarkdown(array $results, float $totalDuration): void $separator, ...collect($rows)->map($formatRow), '', - '' . "{$this->iterations} iterations x {$this->rounds} rounds per benchmark" - . ($this->processes > 1 ? " across {$this->processes} processes" : '') - . ", {$totalDuration}s total" + '' . "{$this->iterations} iterations x {$this->rounds} rounds per benchmark, {$totalDuration}s total" . " — {$this->filteredRounds} outlier rounds excluded (IQR)" . ($snapshot ? ' — compared against baseline snapshot' : '') . '', @@ -384,7 +289,6 @@ protected function outputJsonResults(array $results, float $totalDuration): void $this->output->writeln(json_encode([ 'iterations' => $this->iterations, 'rounds' => $this->rounds, - 'processes' => $this->processes, 'filtered_rounds' => $this->filteredRounds, 'total_duration_s' => $totalDuration, 'benchmarks' => $results, From 4b425e647eae98aa5e7514470def5333782ef7e6 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 12:46:28 +0100 Subject: [PATCH 28/38] Update ci.yml --- .github/workflows/ci.yml | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc7d0e1d..7412e747 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,8 +61,7 @@ jobs: strategy: fail-fast: false matrix: - config: ["50-10000", "200-2500", "1000-500"] - repeat: [1, 2, 3] + repeat: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] steps: - name: Checkout @@ -79,18 +78,13 @@ jobs: - name: Install dependencies run: composer install --prefer-dist --no-progress --no-interaction - - name: Run benchmark (${{ matrix.config }}, run ${{ matrix.repeat }}/3) - run: | - config="${{ matrix.config }}" - rounds="${config%%-*}" - iterations="${config##*-}" - - vendor/bin/testbench benchmark:variance --ci --runs=25 --iterations="$iterations" --rounds="$rounds" > benchmark-result.md + - name: Run benchmark (run ${{ matrix.repeat }}/10) + run: vendor/bin/testbench benchmark:variance --ci --runs=25 --iterations=5000 --rounds=100 > benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 with: - name: benchmark-result-${{ matrix.config }}-${{ matrix.repeat }} + name: benchmark-result-${{ matrix.repeat }} path: benchmark-result.md benchmark-comment: @@ -111,25 +105,17 @@ jobs: run: | echo "## Benchmark Variance Results" > comment.md echo "" >> comment.md - echo "Each configuration is executed 3 times." >> comment.md + echo "100 rounds x 5000 iterations, repeated 10 times." >> comment.md echo "" >> comment.md - for config in "50-10000" "200-2500" "1000-500"; do - rounds="${config%%-*}" - iterations="${config##*-}" + for repeat in $(seq 1 10); do + dir="benchmark-result-${repeat}" - echo "### ${rounds} rounds x ${iterations} iterations" >> comment.md + echo "### Run ${repeat}/10" >> comment.md + echo "" >> comment.md + # Strip the "## Benchmark Results" heading line from each result + sed '/^## Benchmark Results$/d' "$dir/benchmark-result.md" >> comment.md echo "" >> comment.md - - for repeat in 1 2 3; do - dir="benchmark-result-${config}-${repeat}" - - echo "#### Run ${repeat}/3" >> comment.md - echo "" >> comment.md - # Strip the "## Benchmark Results" heading line from each result - sed '/^## Benchmark Results$/d' "$dir/benchmark-result.md" >> comment.md - echo "" >> comment.md - done done - name: Find existing comment From 01e122b2f44d5a22b2d598014b1ae1c7d6a923a4 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 12:48:20 +0100 Subject: [PATCH 29/38] Update ci.yml --- .github/workflows/ci.yml | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7412e747..b6831d3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: strategy: fail-fast: false matrix: - repeat: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + repeat: ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10"] steps: - name: Checkout @@ -108,7 +108,7 @@ jobs: echo "100 rounds x 5000 iterations, repeated 10 times." >> comment.md echo "" >> comment.md - for repeat in $(seq 1 10); do + for repeat in $(seq -w 1 10); do dir="benchmark-result-${repeat}" echo "### Run ${repeat}/10" >> comment.md @@ -118,18 +118,8 @@ jobs: echo "" >> comment.md done - - name: Find existing comment - uses: peter-evans/find-comment@v3 - id: find - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: '## Benchmark Variance Results' - - - name: Post or update comment + - name: Post comment uses: peter-evans/create-or-update-comment@v4 with: issue-number: ${{ github.event.pull_request.number }} - comment-id: ${{ steps.find.outputs.comment-id }} - edit-mode: replace body-path: comment.md From 344d9425e86724395142a32e2ed56879a653e0aa Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 15:47:02 +0100 Subject: [PATCH 30/38] Add attempts --- .github/workflows/ci.yml | 2 +- .../app/Console/Commands/BenchmarkCommand.php | 396 +++++++++++++----- .../Commands/BenchmarkVarianceCommand.php | 230 +++++----- 3 files changed, 376 insertions(+), 252 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6831d3d..83532513 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: run: composer install --prefer-dist --no-progress --no-interaction - name: Run benchmark (run ${{ matrix.repeat }}/10) - run: vendor/bin/testbench benchmark:variance --ci --runs=25 --iterations=5000 --rounds=100 > benchmark-result.md + run: vendor/bin/testbench benchmark:variance props --ci --runs=10 --iterations=5000 --rounds=50 --attempts=10 > benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index 8efb331d..d57a15b7 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -4,23 +4,26 @@ use Illuminate\Console\Command; use Illuminate\Support\Benchmark; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\Artisan; use Illuminate\Support\Facades\File; +use Illuminate\Support\Facades\Process; use Illuminate\Support\Facades\View; use Illuminate\Support\Str; class BenchmarkCommand extends Command { protected $signature = 'benchmark + {benchmark : Name of the benchmark to run} {--iterations=5000 : Number of component renders per benchmark} {--rounds=100 : Number of timed rounds per benchmark} {--warmup=2 : Number of untimed warmup rounds} + {--attempts=5 : Number of times to run the entire benchmark in separate processes} {--snapshot : Save results as the baseline snapshot} {--json : Output results as JSON} - {--only= : Run only the named benchmark} {--ci : Output a markdown table with no progress (for CI)}'; - protected $description = 'Run Blaze performance benchmarks'; + protected $description = 'Run a Blaze performance benchmark'; protected int $iterations; @@ -35,13 +38,28 @@ public function handle(): int $this->iterations = (int) $this->option('iterations'); $this->rounds = (int) $this->option('rounds'); $this->warmupRounds = (int) $this->option('warmup'); + $attempts = (int) $this->option('attempts'); + + if ($attempts < 1) { + $this->error('--attempts must be at least 1.'); + + return Command::FAILURE; + } + + if ($attempts > 1) { + return $this->runMultipleAttempts($attempts); + } + $commandStart = microtime(true); Artisan::call('view:clear'); - $results = $this->runBenchmarks(); + $result = $this->runBenchmark(); $totalDuration = round(microtime(true) - $commandStart, 2); + $benchmarkName = $this->argument('benchmark'); + $results = [$benchmarkName => $result]; + if ($this->option('json')) { $this->outputJsonResults($results, $totalDuration); } elseif ($this->option('ci')) { @@ -57,43 +75,40 @@ public function handle(): int return Command::SUCCESS; } - protected function runBenchmarks(): array + protected function runBenchmark(): array { + $benchmarkName = $this->argument('benchmark'); + $bladeView = "bench.blade.{$benchmarkName}"; + $blazeView = "bench.blaze.{$benchmarkName}"; $showProgress = ! $this->option('ci') && ! $this->option('json'); - $benchmarks = $this->getFilteredBenchmarks(); - $names = array_keys($benchmarks); if ($showProgress) { - $this->info("Running benchmarks ({$this->iterations} iterations x {$this->rounds} rounds)..."); + $this->info("Running '{$benchmarkName}' ({$this->iterations} iterations x {$this->rounds} rounds)..."); $this->newLine(); } - $totalSteps = (count($benchmarks) * $this->warmupRounds) + ($this->rounds * count($benchmarks)); + $totalSteps = $this->warmupRounds + $this->rounds; $bar = $showProgress ? $this->output->createProgressBar($totalSteps) : null; $bar?->setFormat(' %current%/%max% [%bar%] %message%'); $bar?->setMessage('Warming up...'); $bar?->start(); // Warmup: compile views and stabilize opcache. - foreach ($benchmarks as $benchmark) { - for ($w = 0; $w < $this->warmupRounds; $w++) { - $this->measureView($benchmark['blade']); - $this->measureView($benchmark['blaze']); - $bar?->advance(); - } + for ($w = 0; $w < $this->warmupRounds; $w++) { + $this->measureView($bladeView); + $this->measureView($blazeView); + $bar?->advance(); } - $bladeTimes = array_fill_keys($names, []); - $blazeTimes = array_fill_keys($names, []); + $bladeTimes = []; + $blazeTimes = []; $bar?->setMessage('Benchmarking...'); for ($r = 0; $r < $this->rounds; $r++) { - foreach ($benchmarks as $name => $benchmark) { - $bladeTimes[$name][] = $this->measureView($benchmark['blade']); - $blazeTimes[$name][] = $this->measureView($benchmark['blaze']); - $bar?->advance(); - } + $bladeTimes[] = $this->measureView($bladeView); + $blazeTimes[] = $this->measureView($blazeView); + $bar?->advance(); } $bar?->setMessage('Done!'); @@ -104,25 +119,123 @@ protected function runBenchmarks(): array } $roundTotals = collect(range(0, $this->rounds - 1))->map( - fn ($r) => collect($names)->sum(fn ($name) => $bladeTimes[$name][$r] + $blazeTimes[$name][$r]) + fn ($r) => $bladeTimes[$r] + $blazeTimes[$r] ); $keptRounds = $this->nonOutlierIndices($roundTotals); $this->filteredRounds = $this->rounds - $keptRounds->count(); - foreach ($names as $name) { - $bladeTimes[$name] = $keptRounds->map(fn ($r) => $bladeTimes[$name][$r])->all(); - $blazeTimes[$name] = $keptRounds->map(fn ($r) => $blazeTimes[$name][$r])->all(); + $bladeTimes = $keptRounds->map(fn ($r) => $bladeTimes[$r])->all(); + $blazeTimes = $keptRounds->map(fn ($r) => $blazeTimes[$r])->all(); + + return [ + 'blade_ms' => round(collect($bladeTimes)->median(), 2), + 'blaze_ms' => round(collect($blazeTimes)->median(), 2), + ]; + } + + protected function runMultipleAttempts(int $attempts): int + { + $benchmarkName = $this->argument('benchmark'); + $showProgress = ! $this->option('ci') && ! $this->option('json'); + $commandStart = microtime(true); + + $allAttempts = []; + + if ($showProgress) { + $this->info("Running '{$benchmarkName}' ({$attempts} attempts, {$this->iterations} iterations x {$this->rounds} rounds each)..."); + $this->newLine(); + $bar = $this->output->createProgressBar($attempts); + $bar->setFormat(' %current%/%max% [%bar%] %message%'); + } + + for ($i = 0; $i < $attempts; $i++) { + if ($showProgress) { + $bar->setMessage('Attempt '.($i + 1).'/'.$attempts.'...'); + $i === 0 ? $bar->start() : $bar->display(); + } + + $result = Process::path(base_path()) + ->timeout(300) + ->run([ + PHP_BINARY, 'artisan', 'benchmark', $benchmarkName, + '--attempts=1', + '--json', + '--iterations='.$this->iterations, + '--rounds='.$this->rounds, + '--warmup='.$this->warmupRounds, + ]); + + if (! $result->successful()) { + if ($showProgress) { + $this->newLine(); + } + + $this->error('Attempt '.($i + 1).' failed: '.$result->errorOutput()); + + return Command::FAILURE; + } + + $data = json_decode($result->output(), true); + + if (! $data || ! isset($data['benchmarks'][$benchmarkName])) { + if ($showProgress) { + $this->newLine(); + } + + $this->error('Invalid output from attempt '.($i + 1).': '.$result->output()); + + return Command::FAILURE; + } + + $allAttempts[] = $data['benchmarks'][$benchmarkName]; + + if ($showProgress) { + $bar->advance(); + } } - return collect($names)->mapWithKeys(fn ($name) => [ - $name => [ - 'blade_ms' => round(collect($bladeTimes[$name])->median(), 2), - 'blaze_ms' => round(collect($blazeTimes[$name])->median(), 2), - ], - ])->all(); + if ($showProgress) { + $bar->setMessage('Done!'); + $bar->finish(); + $this->newLine(2); + } + + $totalDuration = round(microtime(true) - $commandStart, 2); + + // Filter outlier attempts — if either blade or blaze is an outlier, drop the whole attempt. + $bladeValues = collect($allAttempts)->map(fn ($r) => $r['blade_ms']); + $blazeValues = collect($allAttempts)->map(fn ($r) => $r['blaze_ms']); + $keptIndices = $this->nonOutlierIndices($bladeValues) + ->intersect($this->nonOutlierIndices($blazeValues)) + ->values(); + + $medianResult = [ + 'blade_ms' => round($keptIndices->map(fn ($i) => $allAttempts[$i]['blade_ms'])->median(), 2), + 'blaze_ms' => round($keptIndices->map(fn ($i) => $allAttempts[$i]['blaze_ms'])->median(), 2), + ]; + + $results = [$benchmarkName => $medianResult]; + + if ($this->option('json')) { + $this->outputJsonAttemptsResults($benchmarkName, $allAttempts, $results, $keptIndices, $totalDuration); + } elseif ($this->option('ci')) { + $this->outputMarkdownAttempts($allAttempts, $results, $keptIndices, $totalDuration); + } else { + $this->displayAttemptsResults($allAttempts, $results, $keptIndices, $totalDuration); + } + + if ($this->option('snapshot')) { + $this->saveSnapshot($results); + } + + return Command::SUCCESS; } + // ────────────────────────────────────────────────────────────── + // Display — single attempt + // ────────────────────────────────────────────────────────────── + protected function buildTable(array $results): array { $snapshot = $this->option('snapshot') ? null : $this->loadSnapshot(); @@ -132,12 +245,12 @@ protected function buildTable(array $results): array $rows = collect($results)->map(function ($result, $name) use ($snapshot) { $blade = $this->formatTime($result['blade_ms']); $blaze = $this->formatTime($result['blaze_ms']); - $improvement = $this->improvement($result) . '%'; + $improvement = $this->improvement($result).'%'; if ($prev = $snapshot['benchmarks'][$name] ?? null) { - $blade .= ' ' . $this->formatChange($prev['blade_ms'], $result['blade_ms'], 1); - $blaze .= ' ' . $this->formatChange($prev['blaze_ms'], $result['blaze_ms'], 1); - $improvement .= ' ' . $this->formatImprovementChange($prev['improvement'], $this->improvement($result)); + $blade .= ' '.$this->formatChange($prev['blade_ms'], $result['blade_ms'], 1); + $blaze .= ' '.$this->formatChange($prev['blaze_ms'], $result['blaze_ms'], 1); + $improvement .= ' '.$this->formatImprovementChange($prev['improvement'], $this->improvement($result)); } return [$name, $blade, $blaze, $improvement]; @@ -154,7 +267,7 @@ protected function displayResults(array $results, float $totalDuration): void $this->table($headers, $rows); $this->newLine(); - $this->info("{$this->iterations} iterations x {$this->rounds} rounds per benchmark, {$totalDuration}s total"); + $this->info("{$this->iterations} iterations x {$this->rounds} rounds, {$totalDuration}s total"); $this->comment("{$this->filteredRounds} outlier rounds excluded (IQR method)"); if ($snapshot) { @@ -172,11 +285,11 @@ protected function outputMarkdown(array $results, float $totalDuration): void fn ($i) => $allRows->max(fn ($row) => mb_strlen($row[$i])) ); - $formatRow = fn ($cells) => '| ' . collect($cells) + $formatRow = fn ($cells) => '| '.collect($cells) ->map(fn ($cell, $i) => Str::padRight($cell, $widths[$i])) - ->implode(' | ') . ' |'; + ->implode(' | ').' |'; - $separator = '| ' . $widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ') . ' |'; + $separator = '| '.$widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ').' |'; $md = collect([ '## Benchmark Results', @@ -185,15 +298,129 @@ protected function outputMarkdown(array $results, float $totalDuration): void $separator, ...collect($rows)->map($formatRow), '', - '' . "{$this->iterations} iterations x {$this->rounds} rounds per benchmark, {$totalDuration}s total" - . " — {$this->filteredRounds} outlier rounds excluded (IQR)" - . ($snapshot ? ' — compared against baseline snapshot' : '') - . '', + ''."{$this->iterations} iterations x {$this->rounds} rounds, {$totalDuration}s total" + ." — {$this->filteredRounds} outlier rounds excluded (IQR)" + .($snapshot ? ' — compared against baseline snapshot' : '') + .'', ])->implode("\n"); $this->output->writeln($md); } + protected function outputJsonResults(array $results, float $totalDuration): void + { + $this->output->writeln(json_encode([ + 'iterations' => $this->iterations, + 'rounds' => $this->rounds, + 'filtered_rounds' => $this->filteredRounds, + 'total_duration_s' => $totalDuration, + 'benchmarks' => $results, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); + } + + // ────────────────────────────────────────────────────────────── + // Display — multiple attempts + // ────────────────────────────────────────────────────────────── + + protected function displayAttemptsResults(array $allAttempts, array $results, Collection $keptIndices, float $totalDuration): void + { + $attempts = count($allAttempts); + $filteredAttempts = $attempts - $keptIndices->count(); + + foreach ($allAttempts as $i => $attempt) { + $isOutlier = ! $keptIndices->contains($i); + $improvement = $this->improvement($attempt); + $line = sprintf( + ' Attempt %d: Blade %s Blaze %s (%s%%)', + $i + 1, + $this->formatTime($attempt['blade_ms']), + $this->formatTime($attempt['blaze_ms']), + $improvement + ); + + $isOutlier + ? $this->line($line.' ← outlier') + : $this->line($line); + } + + [$headers, $rows, $snapshot] = $this->buildTable($results); + + $this->newLine(); + $this->table($headers, $rows); + + $this->newLine(); + $this->info( + "Median of {$attempts} attempts" + .($filteredAttempts ? " ({$filteredAttempts} outlier(s) excluded)" : '') + .", {$this->iterations} iterations x {$this->rounds} rounds, {$totalDuration}s total" + ); + + if ($snapshot) { + $rounds = $snapshot['rounds'] ?? 1; + $this->comment("Compared against baseline snapshot ({$snapshot['iterations']} iterations x {$rounds} rounds)"); + } + } + + protected function outputMarkdownAttempts(array $allAttempts, array $results, Collection $keptIndices, float $totalDuration): void + { + $attempts = count($allAttempts); + $filteredAttempts = $attempts - $keptIndices->count(); + + [$headers, $rows, $snapshot] = $this->buildTable($results); + + $allRows = collect([$headers, ...$rows]); + $widths = collect($headers)->keys()->map( + fn ($i) => $allRows->max(fn ($row) => mb_strlen($row[$i])) + ); + + $formatRow = fn ($cells) => '| '.collect($cells) + ->map(fn ($cell, $i) => Str::padRight($cell, $widths[$i])) + ->implode(' | ').' |'; + + $separator = '| '.$widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ').' |'; + + $md = collect([ + '## Benchmark Results', + '', + $formatRow($headers), + $separator, + ...collect($rows)->map($formatRow), + '', + '' + ."Median of {$attempts} attempts" + .($filteredAttempts ? " ({$filteredAttempts} outlier(s) excluded)" : '') + .", {$this->iterations} iterations x {$this->rounds} rounds, {$totalDuration}s total" + .($snapshot ? ' — compared against baseline snapshot' : '') + .'', + ])->implode("\n"); + + $this->output->writeln($md); + } + + protected function outputJsonAttemptsResults(string $benchmarkName, array $allAttempts, array $results, Collection $keptIndices, float $totalDuration): void + { + $attempts = count($allAttempts); + + $this->output->writeln(json_encode([ + 'iterations' => $this->iterations, + 'rounds' => $this->rounds, + 'attempts' => $attempts, + 'filtered_attempts' => $attempts - $keptIndices->count(), + 'total_duration_s' => $totalDuration, + 'attempts_detail' => collect($allAttempts)->map(fn ($attempt, $i) => [ + 'blade_ms' => $attempt['blade_ms'], + 'blaze_ms' => $attempt['blaze_ms'], + 'improvement' => $this->improvement($attempt), + 'outlier' => ! $keptIndices->contains($i), + ])->values()->all(), + 'benchmarks' => $results, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); + } + + // ────────────────────────────────────────────────────────────── + // Snapshot + // ────────────────────────────────────────────────────────────── + protected function saveSnapshot(array $results): void { $snapshot = [ @@ -208,7 +435,7 @@ protected function saveSnapshot(array $results): void $path = $this->snapshotPath(); - File::put($path, json_encode($snapshot, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n"); + File::put($path, json_encode($snapshot, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)."\n"); $this->newLine(); $this->info("Snapshot saved to {$path}"); @@ -229,9 +456,13 @@ protected function loadSnapshot(): ?array protected function snapshotPath(): string { - return dirname(__DIR__, 4) . '/benchmark-snapshot.json'; + return dirname(__DIR__, 4).'/benchmark-snapshot.json'; } + // ────────────────────────────────────────────────────────────── + // Helpers + // ────────────────────────────────────────────────────────────── + protected function improvement(array $result): float { return $result['blade_ms'] > 0 @@ -253,7 +484,7 @@ protected function formatChange(float $old, float $new, float $threshold = 0.1): $sign = $change > 0 ? '+' : ''; - return "({$sign}" . round($change, 1) . '%)'; + return "({$sign}".round($change, 1).'%)'; } protected function formatImprovementChange(float $old, float $new, float $threshold = 0.1): string @@ -269,80 +500,12 @@ protected function formatImprovementChange(float $old, float $new, float $thresh return "({$sign}{$delta}%)"; } - protected function getFilteredBenchmarks(): array - { - $benchmarks = $this->getBenchmarks(); - - if ($this->input->hasOption('only') && ($only = $this->option('only'))) { - if (! isset($benchmarks[$only])) { - throw new \InvalidArgumentException("Unknown benchmark: {$only}"); - } - - return [$only => $benchmarks[$only]]; - } - - return $benchmarks; - } - - protected function outputJsonResults(array $results, float $totalDuration): void - { - $this->output->writeln(json_encode([ - 'iterations' => $this->iterations, - 'rounds' => $this->rounds, - 'filtered_rounds' => $this->filteredRounds, - 'total_duration_s' => $totalDuration, - 'benchmarks' => $results, - ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); - } - - protected function getBenchmarks(): array - { - return [ - // 'No attributes' => [ - // 'blade' => 'bench.blade.no-attributes', - // 'blaze' => 'bench.blaze.no-attributes', - // ], - // 'Attributes only' => [ - // 'blade' => 'bench.blade.attributes', - // 'blaze' => 'bench.blaze.attributes', - // ], - // 'Attributes + merge()' => [ - // 'blade' => 'bench.blade.merge', - // 'blaze' => 'bench.blaze.merge', - // ], - // 'Attributes + class()' => [ - // 'blade' => 'bench.blade.class', - // 'blaze' => 'bench.blaze.class', - // ], - 'Props + attributes' => [ - 'blade' => 'bench.blade.props', - 'blaze' => 'bench.blaze.props', - ], - // 'Default slot' => [ - // 'blade' => 'bench.blade.slot', - // 'blaze' => 'bench.blaze.slot', - // ], - // 'Named slots' => [ - // 'blade' => 'bench.blade.named-slots', - // 'blaze' => 'bench.blaze.named-slots', - // ], - // '`@aware` (nested)' => [ - // 'blade' => 'bench.blade.aware', - // 'blaze' => 'bench.blaze.aware', - // ], - // 'Attribute forwarding' => [ - // 'blade' => 'bench.blade.forwarding', - // 'blaze' => 'bench.blaze.forwarding', - // ], - ]; - } - /** * Return the indices of non-outlier values using the IQR method. * * Values below Q1 - 1.5*IQR or above Q3 + 1.5*IQR are considered outliers. */ - protected function nonOutlierIndices(\Illuminate\Support\Collection $values): \Illuminate\Support\Collection + protected function nonOutlierIndices(Collection $values): Collection { if ($values->count() < 4) { return $values->keys(); @@ -373,6 +536,7 @@ protected function measureView(string $view): float protected function formatTime(float $ms): string { - return number_format($ms, 2) . 'ms'; + return number_format($ms, 2).'ms'; } + } diff --git a/workbench/app/Console/Commands/BenchmarkVarianceCommand.php b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php index 06be704b..201572fb 100644 --- a/workbench/app/Console/Commands/BenchmarkVarianceCommand.php +++ b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php @@ -9,14 +9,16 @@ class BenchmarkVarianceCommand extends BenchmarkCommand { protected $signature = 'benchmark:variance + {benchmark : Name of the benchmark to run} {--runs=5 : Number of benchmark runs after the initial snapshot run} {--iterations=5000 : Number of component renders per benchmark} {--rounds=100 : Number of timed rounds per benchmark} {--warmup=2 : Number of untimed warmup rounds} + {--attempts=1 : Number of attempts per run (forwarded to benchmark command)} {--json : Output results as JSON} {--ci : Output a markdown table with no progress (for CI)}'; - protected $description = 'Run benchmarks multiple times and report variance (min/max/avg) with change deltas'; + protected $description = 'Run a benchmark multiple times and report variance (min/max/avg) with change deltas'; public function handle(): int { @@ -31,57 +33,42 @@ public function handle(): int return Command::FAILURE; } + $benchmarkName = $this->argument('benchmark'); $totalRuns = $runs + 1; $quiet = $this->option('json') || $this->option('ci'); $commandStart = microtime(true); - - $benchmarkNames = array_keys($this->getBenchmarks()); - $totalSteps = $totalRuns * count($benchmarkNames); $runDurations = []; if (! $quiet) { - $bar = $this->output->createProgressBar($totalSteps); + $bar = $this->output->createProgressBar($totalRuns); $bar->setFormat(' %current%/%max% [%bar%] %message%'); $bar->setMessage('Snapshot...'); $bar->start(); } - // Step 1: Snapshot run (each benchmark in its own process) + // Step 1: Snapshot run. $t = microtime(true); - $snapshotResults = []; - - foreach ($benchmarkNames as $name) { - $snapshotResults[$name] = $this->runBenchmarkInProcess($name); - - if (! $quiet) { - $bar->advance(); - } - } - + $snapshotResult = $this->runBenchmarkInProcess($benchmarkName); $runDurations[] = microtime(true) - $t; - $this->saveSnapshot($snapshotResults); + + $this->saveSnapshot([$benchmarkName => $snapshotResult]); if (! $quiet) { + $bar->advance(); $bar->setMessage('Benchmarking...'); } - // Step 2: Benchmark runs (each benchmark in its own process) + // Step 2: Benchmark runs. $allRuns = []; for ($i = 0; $i < $runs; $i++) { $t = microtime(true); - $runResults = []; - - foreach ($benchmarkNames as $name) { - $runResults[$name] = $this->runBenchmarkInProcess($name); + $allRuns[] = $this->runBenchmarkInProcess($benchmarkName); + $runDurations[] = microtime(true) - $t; - if (! $quiet) { - $bar->advance(); - } + if (! $quiet) { + $bar->advance(); } - - $allRuns[] = $runResults; - $runDurations[] = microtime(true) - $t; } if (! $quiet) { @@ -93,13 +80,13 @@ public function handle(): int $avgRunDuration = round(array_sum($runDurations) / count($runDurations), 2); $totalDuration = round(microtime(true) - $commandStart, 2); - // Step 3: Display variance report + // Step 3: Display variance report. if ($this->option('json')) { - $this->outputJson($snapshotResults, $allRuns, $avgRunDuration, $totalDuration); + $this->outputJson($snapshotResult, $allRuns, $avgRunDuration, $totalDuration); } elseif ($this->option('ci')) { - $this->outputVarianceMarkdown($snapshotResults, $allRuns, $avgRunDuration, $totalDuration); + $this->outputVarianceMarkdown($snapshotResult, $allRuns, $avgRunDuration, $totalDuration); } else { - $this->displayVarianceResults($snapshotResults, $allRuns, $avgRunDuration, $totalDuration); + $this->displayVarianceResults($snapshotResult, $allRuns, $avgRunDuration, $totalDuration); } return Command::SUCCESS; @@ -110,12 +97,12 @@ protected function runBenchmarkInProcess(string $name): array $result = Process::path(base_path()) ->timeout(300) ->run([ - PHP_BINARY, 'artisan', 'benchmark', - '--only='.$name, + PHP_BINARY, 'artisan', 'benchmark', $name, '--json', '--iterations='.$this->iterations, '--rounds='.$this->rounds, '--warmup='.$this->warmupRounds, + '--attempts='.$this->option('attempts'), ]); if (! $result->successful()) { @@ -137,46 +124,33 @@ protected function runBenchmarkInProcess(string $name): array protected function displayVarianceResults(array $snapshot, array $allRuns, float $avgRunDuration, float $totalDuration): void { - $stddev = function ($values) { - $count = $values->count(); - if ($count < 2) return 0.0; - $mean = $values->avg(); - $sumSquares = $values->reduce(fn ($carry, $v) => $carry + ($v - $mean) ** 2, 0); - return round(sqrt($sumSquares / ($count - 1)), 2); - }; - - $benchmarkNames = array_keys($snapshot); - $headers = ['', 'Blade', 'Blaze', 'Improvement']; - $rows = []; + $snapshotImprovement = $this->improvement($snapshot); - foreach ($benchmarkNames as $name) { - $snapshotImprovement = $this->improvement($snapshot[$name]); + $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blade_ms'], $run['blade_ms'])); + $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blaze_ms'], $run['blaze_ms'])); + $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run) - $snapshotImprovement, 1)); - $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot[$name]['blade_ms'], $run[$name]['blade_ms'])); - $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot[$name]['blaze_ms'], $run[$name]['blaze_ms'])); - $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run[$name]) - $snapshotImprovement, 1)); - - $rows[] = [ + $headers = ['', 'Blade', 'Blaze', 'Improvement']; + $rows = [ + [ 'Snapshot', - $this->formatTime($snapshot[$name]['blade_ms']), - $this->formatTime($snapshot[$name]['blaze_ms']), + $this->formatTime($snapshot['blade_ms']), + $this->formatTime($snapshot['blaze_ms']), $snapshotImprovement.'%', - ]; - - $rows[] = [ + ], + [ 'Variance', $this->formatVarianceRange($bladeChanges->min(), $bladeChanges->max()), $this->formatVarianceRange($blazeChanges->min(), $blazeChanges->max()), $this->formatVarianceRange($improvementChanges->min(), $improvementChanges->max()), - ]; - - $rows[] = [ + ], + [ 'Std Dev', - '±'.$stddev($bladeChanges).'%', - '±'.$stddev($blazeChanges).'%', - '±'.$stddev($improvementChanges).'%', - ]; - } + '±'.$this->stddev($bladeChanges).'%', + '±'.$this->stddev($blazeChanges).'%', + '±'.$this->stddev($improvementChanges).'%', + ], + ]; $this->newLine(2); $this->table($headers, $rows); @@ -190,57 +164,44 @@ protected function displayVarianceResults(array $snapshot, array $allRuns, float protected function outputVarianceMarkdown(array $snapshot, array $allRuns, float $avgRunDuration, float $totalDuration): void { - $stddev = function ($values) { - $count = $values->count(); - if ($count < 2) return 0.0; - $mean = $values->avg(); - $sumSquares = $values->reduce(fn ($carry, $v) => $carry + ($v - $mean) ** 2, 0); - return round(sqrt($sumSquares / ($count - 1)), 2); - }; - - $benchmarkNames = array_keys($snapshot); - $headers = ['', 'Blade', 'Blaze', 'Improvement']; - $rows = []; - - foreach ($benchmarkNames as $name) { - $snapshotImprovement = $this->improvement($snapshot[$name]); + $snapshotImprovement = $this->improvement($snapshot); - $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot[$name]['blade_ms'], $run[$name]['blade_ms'])); - $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot[$name]['blaze_ms'], $run[$name]['blaze_ms'])); - $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run[$name]) - $snapshotImprovement, 1)); + $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blade_ms'], $run['blade_ms'])); + $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blaze_ms'], $run['blaze_ms'])); + $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run) - $snapshotImprovement, 1)); - $rows[] = [ + $headers = ['', 'Blade', 'Blaze', 'Improvement']; + $rows = [ + [ 'Snapshot', - $this->formatTime($snapshot[$name]['blade_ms']), - $this->formatTime($snapshot[$name]['blaze_ms']), - $snapshotImprovement . '%', - ]; - - $rows[] = [ + $this->formatTime($snapshot['blade_ms']), + $this->formatTime($snapshot['blaze_ms']), + $snapshotImprovement.'%', + ], + [ 'Variance', $this->formatVarianceRange($bladeChanges->min(), $bladeChanges->max()), $this->formatVarianceRange($blazeChanges->min(), $blazeChanges->max()), $this->formatVarianceRange($improvementChanges->min(), $improvementChanges->max()), - ]; - - $rows[] = [ + ], + [ 'Std Dev', - '±' . $stddev($bladeChanges) . '%', - '±' . $stddev($blazeChanges) . '%', - '±' . $stddev($improvementChanges) . '%', - ]; - } + '±'.$this->stddev($bladeChanges).'%', + '±'.$this->stddev($blazeChanges).'%', + '±'.$this->stddev($improvementChanges).'%', + ], + ]; $allRows = collect([$headers, ...$rows]); $widths = collect($headers)->keys()->map( fn ($i) => $allRows->max(fn ($row) => mb_strlen($row[$i])) ); - $formatRow = fn ($cells) => '| ' . collect($cells) + $formatRow = fn ($cells) => '| '.collect($cells) ->map(fn ($cell, $i) => Str::padRight($cell, $widths[$i])) - ->implode(' | ') . ' |'; + ->implode(' | ').' |'; - $separator = '| ' . $widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ') . ' |'; + $separator = '| '.$widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ').' |'; $md = collect([ '## Benchmark Results', @@ -249,9 +210,9 @@ protected function outputVarianceMarkdown(array $snapshot, array $allRuns, float $separator, ...collect($rows)->map($formatRow), '', - '' . count($allRuns) . " runs x {$this->rounds} rounds x {$this->iterations} iterations" - . ", ~{$avgRunDuration}s/run, {$totalDuration}s total" - . '', + ''.count($allRuns)." runs x {$this->rounds} rounds x {$this->iterations} iterations" + .", ~{$avgRunDuration}s/run, {$totalDuration}s total" + .'', ])->implode("\n"); $this->output->writeln($md); @@ -277,36 +238,11 @@ protected function saveSnapshot(array $results): void protected function outputJson(array $snapshot, array $allRuns, float $avgRunDuration, float $totalDuration): void { - $benchmarks = []; - - foreach (array_keys($snapshot) as $name) { - $snapshotImprovement = $this->improvement($snapshot[$name]); - - $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot[$name]['blade_ms'], $run[$name]['blade_ms'])); - $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot[$name]['blaze_ms'], $run[$name]['blaze_ms'])); - $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run[$name]) - $snapshotImprovement, 1)); - - $stddev = function ($values) { - $count = $values->count(); - if ($count < 2) return 0.0; - $mean = $values->avg(); - $sumSquares = $values->reduce(fn ($carry, $v) => $carry + ($v - $mean) ** 2, 0); - return round(sqrt($sumSquares / ($count - 1)), 2); - }; - - $benchmarks[$name] = [ - 'snapshot' => [ - 'blade_ms' => $snapshot[$name]['blade_ms'], - 'blaze_ms' => $snapshot[$name]['blaze_ms'], - 'improvement' => $snapshotImprovement, - ], - 'variance' => [ - 'blade' => ['min' => $bladeChanges->min(), 'max' => $bladeChanges->max(), 'stddev' => $stddev($bladeChanges)], - 'blaze' => ['min' => $blazeChanges->min(), 'max' => $blazeChanges->max(), 'stddev' => $stddev($blazeChanges)], - 'improvement' => ['min' => $improvementChanges->min(), 'max' => $improvementChanges->max(), 'stddev' => $stddev($improvementChanges)], - ], - ]; - } + $snapshotImprovement = $this->improvement($snapshot); + + $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blade_ms'], $run['blade_ms'])); + $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blaze_ms'], $run['blaze_ms'])); + $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run) - $snapshotImprovement, 1)); $this->output->writeln(json_encode([ 'iterations' => $this->iterations, @@ -314,18 +250,42 @@ protected function outputJson(array $snapshot, array $allRuns, float $avgRunDura 'runs' => count($allRuns), 'avg_run_duration_s' => $avgRunDuration, 'total_duration_s' => $totalDuration, - 'filter_outliers' => true, - 'benchmarks' => $benchmarks, + 'snapshot' => [ + 'blade_ms' => $snapshot['blade_ms'], + 'blaze_ms' => $snapshot['blaze_ms'], + 'improvement' => $snapshotImprovement, + ], + 'variance' => [ + 'blade' => ['min' => $bladeChanges->min(), 'max' => $bladeChanges->max(), 'stddev' => $this->stddev($bladeChanges)], + 'blaze' => ['min' => $blazeChanges->min(), 'max' => $blazeChanges->max(), 'stddev' => $this->stddev($blazeChanges)], + 'improvement' => ['min' => $improvementChanges->min(), 'max' => $improvementChanges->max(), 'stddev' => $this->stddev($improvementChanges)], + ], ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); } + protected function stddev(\Illuminate\Support\Collection $values): float + { + $count = $values->count(); + + if ($count < 2) { + return 0.0; + } + + $mean = $values->avg(); + $sumSquares = $values->reduce(fn ($carry, $v) => $carry + ($v - $mean) ** 2, 0); + + return round(sqrt($sumSquares / ($count - 1)), 2); + } + protected function formatVarianceRange(float $min, float $max): string { $fmt = function (float $v): string { $rounded = round($v, 1); + if ($rounded == 0) { return '0%'; } + $sign = $rounded > 0 ? '+' : ''; return $sign.$rounded.'%'; From bc0625a28b460033ce0b231f0dadc3ffe1a949f4 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 16:38:45 +0100 Subject: [PATCH 31/38] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83532513..9aebdcb9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: run: composer install --prefer-dist --no-progress --no-interaction - name: Run benchmark (run ${{ matrix.repeat }}/10) - run: vendor/bin/testbench benchmark:variance props --ci --runs=10 --iterations=5000 --rounds=50 --attempts=10 > benchmark-result.md + run: vendor/bin/testbench benchmark:variance props --ci --runs=10 --iterations=5000 --rounds=10 --attempts=10 > benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 From 963f4cc8f1e76bb8c54073d7244221e65d9cafbe Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 17:25:02 +0100 Subject: [PATCH 32/38] Update ci.yml --- .github/workflows/ci.yml | 76 ++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9aebdcb9..4d6abf06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,14 +58,17 @@ jobs: needs: pest if: github.event_name == 'pull_request' runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - repeat: ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10"] steps: - - name: Checkout + - name: Checkout base branch uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + + - name: Checkout BenchmarkCommand from PR (for testing on this branch) + run: | + git fetch origin --depth=1 ${{ github.event.pull_request.head.sha }} + git checkout ${{ github.event.pull_request.head.sha }} -- workbench/app/Console/Commands/BenchmarkCommand.php - name: Setup PHP uses: shivammathur/setup-php@v2 @@ -75,51 +78,32 @@ jobs: coverage: none extensions: mbstring, dom, curl, json, libxml, xml, xmlwriter, simplexml, tokenizer - - name: Install dependencies + - name: Install base dependencies run: composer install --prefer-dist --no-progress --no-interaction - - name: Run benchmark (run ${{ matrix.repeat }}/10) - run: vendor/bin/testbench benchmark:variance props --ci --runs=10 --iterations=5000 --rounds=10 --attempts=10 > benchmark-result.md + - name: Generate baseline snapshot + run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=100 --snapshot - - name: Upload benchmark result - uses: actions/upload-artifact@v4 - with: - name: benchmark-result-${{ matrix.repeat }} - path: benchmark-result.md + - name: Save baseline snapshot + run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json - benchmark-comment: - needs: benchmark - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - permissions: - actions: read - pull-requests: write + - name: Checkout PR + uses: actions/checkout@v4 - steps: - - name: Download all benchmark results - uses: actions/download-artifact@v4 - with: - pattern: benchmark-result-* + - name: Install PR dependencies + run: composer install --prefer-dist --no-progress --no-interaction - - name: Assemble comment - run: | - echo "## Benchmark Variance Results" > comment.md - echo "" >> comment.md - echo "100 rounds x 5000 iterations, repeated 10 times." >> comment.md - echo "" >> comment.md - - for repeat in $(seq -w 1 10); do - dir="benchmark-result-${repeat}" - - echo "### Run ${repeat}/10" >> comment.md - echo "" >> comment.md - # Strip the "## Benchmark Results" heading line from each result - sed '/^## Benchmark Results$/d' "$dir/benchmark-result.md" >> comment.md - echo "" >> comment.md - done - - - name: Post comment - uses: peter-evans/create-or-update-comment@v4 + - name: Restore baseline snapshot + run: cp ${{ runner.temp }}/benchmark-snapshot.json benchmark-snapshot.json + + - name: Save PR number + run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md + + - name: Run benchmark + run: vendor/bin/testbench benchmark:variance --ci --runs=10 --iterations=5000 --rounds=100 >> benchmark-result.md + + - name: Upload benchmark result + uses: actions/upload-artifact@v4 with: - issue-number: ${{ github.event.pull_request.number }} - body-path: comment.md + name: benchmark-result + path: benchmark-result.md From 16e84a2a736f1897c12d7b28360230ff472821f1 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 17:26:46 +0100 Subject: [PATCH 33/38] Update ci.yml --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d6abf06..ac6c0436 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: run: composer install --prefer-dist --no-progress --no-interaction - name: Generate baseline snapshot - run: vendor/bin/testbench benchmark --ci --iterations=5000 --rounds=100 --snapshot + run: vendor/bin/testbench benchmark props --ci --iterations=5000 --rounds=10 --attempts=10 --snapshot - name: Save baseline snapshot run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json @@ -100,7 +100,7 @@ jobs: run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md - name: Run benchmark - run: vendor/bin/testbench benchmark:variance --ci --runs=10 --iterations=5000 --rounds=100 >> benchmark-result.md + run: vendor/bin/testbench benchmark props --ci --iterations=5000 --rounds=10 --attempts=10 >> benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 From 2969db2eabc5275de15ec4d7c3142ee1c821a67c Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 17:39:32 +0100 Subject: [PATCH 34/38] Test bench --- src/Runtime/BlazeRuntime.php | 2 +- workbench/app/Console/Commands/BenchmarkCommand.php | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Runtime/BlazeRuntime.php b/src/Runtime/BlazeRuntime.php index 9131ce75..cc82112a 100644 --- a/src/Runtime/BlazeRuntime.php +++ b/src/Runtime/BlazeRuntime.php @@ -45,7 +45,7 @@ public function __construct() */ public function ensureCompiled(string $path, string $compiledPath): void { - if (isset($this->compiled[$path]) && file_exists($compiledPath)) { + if (isset($this->compiled[$path])) { return; } diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index d57a15b7..614cc5dd 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -240,7 +240,7 @@ protected function buildTable(array $results): array { $snapshot = $this->option('snapshot') ? null : $this->loadSnapshot(); - $headers = ['Benchmark', 'Blade', 'Blaze', 'Improvement']; + $headers = ['Blade', 'Blaze', 'Improvement']; $rows = collect($results)->map(function ($result, $name) use ($snapshot) { $blade = $this->formatTime($result['blade_ms']); @@ -253,7 +253,7 @@ protected function buildTable(array $results): array $improvement .= ' '.$this->formatImprovementChange($prev['improvement'], $this->improvement($result)); } - return [$name, $blade, $blaze, $improvement]; + return [$blade, $blaze, $improvement]; })->values()->all(); return [$headers, $rows, $snapshot]; @@ -470,7 +470,7 @@ protected function improvement(array $result): float : 0; } - protected function formatChange(float $old, float $new, float $threshold = 0.1): string + protected function formatChange(float $old, float $new, float $threshold = 3): string { if ($old == 0) { return '(~)'; @@ -487,7 +487,7 @@ protected function formatChange(float $old, float $new, float $threshold = 0.1): return "({$sign}".round($change, 1).'%)'; } - protected function formatImprovementChange(float $old, float $new, float $threshold = 0.1): string + protected function formatImprovementChange(float $old, float $new, float $threshold = 0.2): string { $delta = round($new - $old, 1); From 3153b2782ded55da338262df2a4a87efdc7a0fb9 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 18:19:44 +0100 Subject: [PATCH 35/38] Update BenchmarkCommand.php --- workbench/app/Console/Commands/BenchmarkCommand.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index 614cc5dd..7224f1d5 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -478,7 +478,7 @@ protected function formatChange(float $old, float $new, float $threshold = 3): s $change = ($new - $old) / abs($old) * 100; - if (abs($change) < $threshold) { + if (abs($change) <= $threshold) { return '(~)'; } @@ -491,7 +491,7 @@ protected function formatImprovementChange(float $old, float $new, float $thresh { $delta = round($new - $old, 1); - if (abs($delta) < $threshold) { + if (abs($delta) <= $threshold) { return '(~)'; } From 977bc4ee4523eaae3bdf5c0dacdc5428f8fd71e2 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 19:28:04 +0100 Subject: [PATCH 36/38] Update BenchmarkCommand.php --- .../app/Console/Commands/BenchmarkCommand.php | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index 7224f1d5..b20f510a 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -365,8 +365,49 @@ protected function outputMarkdownAttempts(array $allAttempts, array $results, Co { $attempts = count($allAttempts); $filteredAttempts = $attempts - $keptIndices->count(); + $benchmarkName = array_key_first($results); + $medianResult = $results[$benchmarkName]; + $snapshot = $this->option('snapshot') ? null : $this->loadSnapshot(); + $snapshotData = $snapshot['benchmarks'][$benchmarkName] ?? null; - [$headers, $rows, $snapshot] = $this->buildTable($results); + $headers = ['Attempt', 'Blade', 'Blaze', 'Improvement']; + + $rows = []; + + // Individual attempt rows. + foreach ($allAttempts as $i => $attempt) { + $isOutlier = ! $keptIndices->contains($i); + + $rows[] = [ + '#'.($i + 1).($isOutlier ? ' \*' : ''), + $this->formatTime($attempt['blade_ms']), + $this->formatTime($attempt['blaze_ms']), + $this->improvement($attempt).'%', + ]; + } + + // Snapshot row. + if ($snapshotData) { + $rows[] = [ + 'Snapshot', + $this->formatTime($snapshotData['blade_ms']), + $this->formatTime($snapshotData['blaze_ms']), + $snapshotData['improvement'].'%', + ]; + } + + // Result row (median with comparison deltas when snapshot exists). + $blade = $this->formatTime($medianResult['blade_ms']); + $blaze = $this->formatTime($medianResult['blaze_ms']); + $improvement = $this->improvement($medianResult).'%'; + + if ($snapshotData) { + $blade .= ' '.$this->formatChange($snapshotData['blade_ms'], $medianResult['blade_ms'], 1); + $blaze .= ' '.$this->formatChange($snapshotData['blaze_ms'], $medianResult['blaze_ms'], 1); + $improvement .= ' '.$this->formatImprovementChange($snapshotData['improvement'], $this->improvement($medianResult)); + } + + $rows[] = ['**Result**', "**{$blade}**", "**{$blaze}**", "**{$improvement}**"]; $allRows = collect([$headers, ...$rows]); $widths = collect($headers)->keys()->map( @@ -388,9 +429,8 @@ protected function outputMarkdownAttempts(array $allAttempts, array $results, Co '', '' ."Median of {$attempts} attempts" - .($filteredAttempts ? " ({$filteredAttempts} outlier(s) excluded)" : '') + .($filteredAttempts ? ' (\* = outlier, excluded from result)' : '') .", {$this->iterations} iterations x {$this->rounds} rounds, {$totalDuration}s total" - .($snapshot ? ' — compared against baseline snapshot' : '') .'', ])->implode("\n"); From 06e440195a8ac33310f61910cd96bac0716f840c Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 19:35:55 +0100 Subject: [PATCH 37/38] Update BenchmarkCommand.php --- workbench/app/Console/Commands/BenchmarkCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index b20f510a..bcceb757 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -379,7 +379,7 @@ protected function outputMarkdownAttempts(array $allAttempts, array $results, Co $isOutlier = ! $keptIndices->contains($i); $rows[] = [ - '#'.($i + 1).($isOutlier ? ' \*' : ''), + '`#'.($i + 1).'`'.($isOutlier ? ' \*' : ''), $this->formatTime($attempt['blade_ms']), $this->formatTime($attempt['blaze_ms']), $this->improvement($attempt).'%', From d7011a14bc4089f4275b8a590a2268aedc5332b6 Mon Sep 17 00:00:00 2001 From: Filip Ganyicz Date: Tue, 3 Mar 2026 19:54:27 +0100 Subject: [PATCH 38/38] Finalize benchmarks --- .github/workflows/ci.yml | 5 ----- src/Runtime/BlazeRuntime.php | 2 +- .../app/Console/Commands/BenchmarkCommand.php | 18 +----------------- .../Commands/BenchmarkVarianceCommand.php | 6 +++++- 4 files changed, 7 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac6c0436..08270423 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,11 +65,6 @@ jobs: with: ref: ${{ github.event.pull_request.base.sha }} - - name: Checkout BenchmarkCommand from PR (for testing on this branch) - run: | - git fetch origin --depth=1 ${{ github.event.pull_request.head.sha }} - git checkout ${{ github.event.pull_request.head.sha }} -- workbench/app/Console/Commands/BenchmarkCommand.php - - name: Setup PHP uses: shivammathur/setup-php@v2 with: diff --git a/src/Runtime/BlazeRuntime.php b/src/Runtime/BlazeRuntime.php index cc82112a..9131ce75 100644 --- a/src/Runtime/BlazeRuntime.php +++ b/src/Runtime/BlazeRuntime.php @@ -45,7 +45,7 @@ public function __construct() */ public function ensureCompiled(string $path, string $compiledPath): void { - if (isset($this->compiled[$path])) { + if (isset($this->compiled[$path]) && file_exists($compiledPath)) { return; } diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index bcceb757..0adf9519 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -89,7 +89,7 @@ protected function runBenchmark(): array $totalSteps = $this->warmupRounds + $this->rounds; $bar = $showProgress ? $this->output->createProgressBar($totalSteps) : null; - $bar?->setFormat(' %current%/%max% [%bar%] %message%'); + $bar?->setFormat('[%bar%] %message%'); $bar?->setMessage('Warming up...'); $bar?->start(); @@ -232,10 +232,6 @@ protected function runMultipleAttempts(int $attempts): int return Command::SUCCESS; } - // ────────────────────────────────────────────────────────────── - // Display — single attempt - // ────────────────────────────────────────────────────────────── - protected function buildTable(array $results): array { $snapshot = $this->option('snapshot') ? null : $this->loadSnapshot(); @@ -318,10 +314,6 @@ protected function outputJsonResults(array $results, float $totalDuration): void ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); } - // ────────────────────────────────────────────────────────────── - // Display — multiple attempts - // ────────────────────────────────────────────────────────────── - protected function displayAttemptsResults(array $allAttempts, array $results, Collection $keptIndices, float $totalDuration): void { $attempts = count($allAttempts); @@ -457,10 +449,6 @@ protected function outputJsonAttemptsResults(string $benchmarkName, array $allAt ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); } - // ────────────────────────────────────────────────────────────── - // Snapshot - // ────────────────────────────────────────────────────────────── - protected function saveSnapshot(array $results): void { $snapshot = [ @@ -499,10 +487,6 @@ protected function snapshotPath(): string return dirname(__DIR__, 4).'/benchmark-snapshot.json'; } - // ────────────────────────────────────────────────────────────── - // Helpers - // ────────────────────────────────────────────────────────────── - protected function improvement(array $result): float { return $result['blade_ms'] > 0 diff --git a/workbench/app/Console/Commands/BenchmarkVarianceCommand.php b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php index 201572fb..31783e87 100644 --- a/workbench/app/Console/Commands/BenchmarkVarianceCommand.php +++ b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php @@ -6,6 +6,10 @@ use Illuminate\Support\Facades\Process; use Illuminate\Support\Str; +/** + * Used for measuring the reliability of the benchmark + * in the CI and its variance from the snapshot. + */ class BenchmarkVarianceCommand extends BenchmarkCommand { protected $signature = 'benchmark:variance @@ -14,7 +18,7 @@ class BenchmarkVarianceCommand extends BenchmarkCommand {--iterations=5000 : Number of component renders per benchmark} {--rounds=100 : Number of timed rounds per benchmark} {--warmup=2 : Number of untimed warmup rounds} - {--attempts=1 : Number of attempts per run (forwarded to benchmark command)} + {--attempts=5 : Number of attempts per run (forwarded to benchmark command)} {--json : Output results as JSON} {--ci : Output a markdown table with no progress (for CI)}';