diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1180fa16..08270423 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,7 @@ jobs: run: composer install --prefer-dist --no-progress --no-interaction - name: Generate baseline snapshot - run: vendor/bin/testbench benchmark --snapshot --ci + run: vendor/bin/testbench benchmark props --ci --iterations=5000 --rounds=10 --attempts=10 --snapshot - name: Save baseline snapshot run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json @@ -95,7 +95,7 @@ jobs: run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md - name: Run benchmark - run: vendor/bin/testbench benchmark --ci >> benchmark-result.md + run: vendor/bin/testbench benchmark props --ci --iterations=5000 --rounds=10 --attempts=10 >> benchmark-result.md - name: Upload benchmark result uses: actions/upload-artifact@v4 diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php index faad1577..0adf9519 100644 --- a/workbench/app/Console/Commands/BenchmarkCommand.php +++ b/workbench/app/Console/Commands/BenchmarkCommand.php @@ -4,21 +4,26 @@ use Illuminate\Console\Command; use Illuminate\Support\Benchmark; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\Artisan; use Illuminate\Support\Facades\File; +use Illuminate\Support\Facades\Process; use Illuminate\Support\Facades\View; use Illuminate\Support\Str; class BenchmarkCommand extends Command { protected $signature = 'benchmark - {--iterations=10000 : Number of component renders per benchmark} - {--rounds=5 : Number of timed rounds per benchmark} + {benchmark : Name of the benchmark to run} + {--iterations=5000 : Number of component renders per benchmark} + {--rounds=100 : Number of timed rounds per benchmark} {--warmup=2 : Number of untimed warmup rounds} + {--attempts=5 : Number of times to run the entire benchmark in separate processes} {--snapshot : Save results as the baseline snapshot} + {--json : Output results as JSON} {--ci : Output a markdown table with no progress (for CI)}'; - protected $description = 'Run Blaze performance benchmarks'; + protected $description = 'Run a Blaze performance benchmark'; protected int $iterations; @@ -26,19 +31,42 @@ class BenchmarkCommand extends Command protected int $warmupRounds; + protected int $filteredRounds = 0; + public function handle(): int { $this->iterations = (int) $this->option('iterations'); $this->rounds = (int) $this->option('rounds'); $this->warmupRounds = (int) $this->option('warmup'); + $attempts = (int) $this->option('attempts'); + + if ($attempts < 1) { + $this->error('--attempts must be at least 1.'); + + return Command::FAILURE; + } + + if ($attempts > 1) { + return $this->runMultipleAttempts($attempts); + } + + $commandStart = microtime(true); Artisan::call('view:clear'); - $results = $this->runBenchmarks(); + $result = $this->runBenchmark(); + $totalDuration = round(microtime(true) - $commandStart, 2); - $this->option('ci') - ? $this->outputMarkdown($results) - : $this->displayResults($results); + $benchmarkName = $this->argument('benchmark'); + $results = [$benchmarkName => $result]; + + if ($this->option('json')) { + $this->outputJsonResults($results, $totalDuration); + } elseif ($this->option('ci')) { + $this->outputMarkdown($results, $totalDuration); + } else { + $this->displayResults($results, $totalDuration); + } if ($this->option('snapshot')) { $this->saveSnapshot($results); @@ -47,43 +75,40 @@ public function handle(): int return Command::SUCCESS; } - protected function runBenchmarks(): array + protected function runBenchmark(): array { - $showProgress = ! $this->option('ci'); - $benchmarks = $this->getBenchmarks(); - $names = array_keys($benchmarks); + $benchmarkName = $this->argument('benchmark'); + $bladeView = "bench.blade.{$benchmarkName}"; + $blazeView = "bench.blaze.{$benchmarkName}"; + $showProgress = ! $this->option('ci') && ! $this->option('json'); if ($showProgress) { - $this->info("Running benchmarks ({$this->iterations} iterations x {$this->rounds} rounds)..."); + $this->info("Running '{$benchmarkName}' ({$this->iterations} iterations x {$this->rounds} rounds)..."); $this->newLine(); } - $totalSteps = (count($benchmarks) * $this->warmupRounds) + ($this->rounds * count($benchmarks)); + $totalSteps = $this->warmupRounds + $this->rounds; $bar = $showProgress ? $this->output->createProgressBar($totalSteps) : null; - $bar?->setFormat(' %current%/%max% [%bar%] %message%'); + $bar?->setFormat('[%bar%] %message%'); $bar?->setMessage('Warming up...'); $bar?->start(); // Warmup: compile views and stabilize opcache. - foreach ($benchmarks as $benchmark) { - for ($w = 0; $w < $this->warmupRounds; $w++) { - $this->measureView($benchmark['blade']); - $this->measureView($benchmark['blaze']); - $bar?->advance(); - } + for ($w = 0; $w < $this->warmupRounds; $w++) { + $this->measureView($bladeView); + $this->measureView($blazeView); + $bar?->advance(); } - $bladeTimes = array_fill_keys($names, []); - $blazeTimes = array_fill_keys($names, []); + $bladeTimes = []; + $blazeTimes = []; $bar?->setMessage('Benchmarking...'); for ($r = 0; $r < $this->rounds; $r++) { - foreach ($benchmarks as $name => $benchmark) { - $bladeTimes[$name][] = $this->measureView($benchmark['blade']); - $blazeTimes[$name][] = $this->measureView($benchmark['blaze']); - $bar?->advance(); - } + $bladeTimes[] = $this->measureView($bladeView); + $blazeTimes[] = $this->measureView($blazeView); + $bar?->advance(); } $bar?->setMessage('Done!'); @@ -93,38 +118,144 @@ protected function runBenchmarks(): array $this->newLine(2); } - return collect($names)->mapWithKeys(fn ($name) => [ - $name => [ - 'blade_ms' => round(collect($bladeTimes[$name])->median(), 2), - 'blaze_ms' => round(collect($blazeTimes[$name])->median(), 2), - ], - ])->all(); + $roundTotals = collect(range(0, $this->rounds - 1))->map( + fn ($r) => $bladeTimes[$r] + $blazeTimes[$r] + ); + + $keptRounds = $this->nonOutlierIndices($roundTotals); + $this->filteredRounds = $this->rounds - $keptRounds->count(); + + $bladeTimes = $keptRounds->map(fn ($r) => $bladeTimes[$r])->all(); + $blazeTimes = $keptRounds->map(fn ($r) => $blazeTimes[$r])->all(); + + return [ + 'blade_ms' => round(collect($bladeTimes)->median(), 2), + 'blaze_ms' => round(collect($blazeTimes)->median(), 2), + ]; + } + + protected function runMultipleAttempts(int $attempts): int + { + $benchmarkName = $this->argument('benchmark'); + $showProgress = ! $this->option('ci') && ! $this->option('json'); + $commandStart = microtime(true); + + $allAttempts = []; + + if ($showProgress) { + $this->info("Running '{$benchmarkName}' ({$attempts} attempts, {$this->iterations} iterations x {$this->rounds} rounds each)..."); + $this->newLine(); + $bar = $this->output->createProgressBar($attempts); + $bar->setFormat(' %current%/%max% [%bar%] %message%'); + } + + for ($i = 0; $i < $attempts; $i++) { + if ($showProgress) { + $bar->setMessage('Attempt '.($i + 1).'/'.$attempts.'...'); + $i === 0 ? $bar->start() : $bar->display(); + } + + $result = Process::path(base_path()) + ->timeout(300) + ->run([ + PHP_BINARY, 'artisan', 'benchmark', $benchmarkName, + '--attempts=1', + '--json', + '--iterations='.$this->iterations, + '--rounds='.$this->rounds, + '--warmup='.$this->warmupRounds, + ]); + + if (! $result->successful()) { + if ($showProgress) { + $this->newLine(); + } + + $this->error('Attempt '.($i + 1).' failed: '.$result->errorOutput()); + + return Command::FAILURE; + } + + $data = json_decode($result->output(), true); + + if (! $data || ! isset($data['benchmarks'][$benchmarkName])) { + if ($showProgress) { + $this->newLine(); + } + + $this->error('Invalid output from attempt '.($i + 1).': '.$result->output()); + + return Command::FAILURE; + } + + $allAttempts[] = $data['benchmarks'][$benchmarkName]; + + if ($showProgress) { + $bar->advance(); + } + } + + if ($showProgress) { + $bar->setMessage('Done!'); + $bar->finish(); + $this->newLine(2); + } + + $totalDuration = round(microtime(true) - $commandStart, 2); + + // Filter outlier attempts — if either blade or blaze is an outlier, drop the whole attempt. + $bladeValues = collect($allAttempts)->map(fn ($r) => $r['blade_ms']); + $blazeValues = collect($allAttempts)->map(fn ($r) => $r['blaze_ms']); + $keptIndices = $this->nonOutlierIndices($bladeValues) + ->intersect($this->nonOutlierIndices($blazeValues)) + ->values(); + + $medianResult = [ + 'blade_ms' => round($keptIndices->map(fn ($i) => $allAttempts[$i]['blade_ms'])->median(), 2), + 'blaze_ms' => round($keptIndices->map(fn ($i) => $allAttempts[$i]['blaze_ms'])->median(), 2), + ]; + + $results = [$benchmarkName => $medianResult]; + + if ($this->option('json')) { + $this->outputJsonAttemptsResults($benchmarkName, $allAttempts, $results, $keptIndices, $totalDuration); + } elseif ($this->option('ci')) { + $this->outputMarkdownAttempts($allAttempts, $results, $keptIndices, $totalDuration); + } else { + $this->displayAttemptsResults($allAttempts, $results, $keptIndices, $totalDuration); + } + + if ($this->option('snapshot')) { + $this->saveSnapshot($results); + } + + return Command::SUCCESS; } protected function buildTable(array $results): array { $snapshot = $this->option('snapshot') ? null : $this->loadSnapshot(); - $headers = ['Benchmark', 'Blade', 'Blaze', 'Improvement']; + $headers = ['Blade', 'Blaze', 'Improvement']; $rows = collect($results)->map(function ($result, $name) use ($snapshot) { $blade = $this->formatTime($result['blade_ms']); $blaze = $this->formatTime($result['blaze_ms']); - $improvement = $this->improvement($result) . '%'; + $improvement = $this->improvement($result).'%'; if ($prev = $snapshot['benchmarks'][$name] ?? null) { - $blade .= ' ' . $this->formatChange($prev['blade_ms'], $result['blade_ms'], 10.0); - $blaze .= ' ' . $this->formatChange($prev['blaze_ms'], $result['blaze_ms'], 5.0); - $improvement .= ' ' . $this->formatChange($prev['improvement'], $this->improvement($result), 1.0); + $blade .= ' '.$this->formatChange($prev['blade_ms'], $result['blade_ms'], 1); + $blaze .= ' '.$this->formatChange($prev['blaze_ms'], $result['blaze_ms'], 1); + $improvement .= ' '.$this->formatImprovementChange($prev['improvement'], $this->improvement($result)); } - return [$name, $blade, $blaze, $improvement]; + return [$blade, $blaze, $improvement]; })->values()->all(); return [$headers, $rows, $snapshot]; } - protected function displayResults(array $results): void + protected function displayResults(array $results, float $totalDuration): void { [$headers, $rows, $snapshot] = $this->buildTable($results); @@ -132,15 +263,16 @@ protected function displayResults(array $results): void $this->table($headers, $rows); $this->newLine(); - $this->info("{$this->iterations} iterations x {$this->rounds} rounds per benchmark"); + $this->info("{$this->iterations} iterations x {$this->rounds} rounds, {$totalDuration}s total"); + $this->comment("{$this->filteredRounds} outlier rounds excluded (IQR method)"); if ($snapshot) { $rounds = $snapshot['rounds'] ?? 1; - $this->comment("Compared against snapshot ({$snapshot['iterations']} iterations x {$rounds} rounds)"); + $this->comment("Compared against baseline snapshot ({$snapshot['iterations']} iterations x {$rounds} rounds)"); } } - protected function outputMarkdown(array $results): void + protected function outputMarkdown(array $results, float $totalDuration): void { [$headers, $rows, $snapshot] = $this->buildTable($results); @@ -149,11 +281,11 @@ protected function outputMarkdown(array $results): void fn ($i) => $allRows->max(fn ($row) => mb_strlen($row[$i])) ); - $formatRow = fn ($cells) => '| ' . collect($cells) + $formatRow = fn ($cells) => '| '.collect($cells) ->map(fn ($cell, $i) => Str::padRight($cell, $widths[$i])) - ->implode(' | ') . ' |'; + ->implode(' | ').' |'; - $separator = '| ' . $widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ') . ' |'; + $separator = '| '.$widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ').' |'; $md = collect([ '## Benchmark Results', @@ -162,25 +294,163 @@ protected function outputMarkdown(array $results): void $separator, ...collect($rows)->map($formatRow), '', - '' . "{$this->iterations} iterations x {$this->rounds} rounds per benchmark" - . ($snapshot ? ' — compared against committed snapshot' : '') - . '', + ''."{$this->iterations} iterations x {$this->rounds} rounds, {$totalDuration}s total" + ." — {$this->filteredRounds} outlier rounds excluded (IQR)" + .($snapshot ? ' — compared against baseline snapshot' : '') + .'', ])->implode("\n"); $this->output->writeln($md); } - protected function saveSnapshot(array $results): void + protected function outputJsonResults(array $results, float $totalDuration): void { - $existing = $this->loadSnapshot(); + $this->output->writeln(json_encode([ + 'iterations' => $this->iterations, + 'rounds' => $this->rounds, + 'filtered_rounds' => $this->filteredRounds, + 'total_duration_s' => $totalDuration, + 'benchmarks' => $results, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); + } - if ($existing && ! $this->hasSignificantChange($results, $existing)) { - $this->newLine(); - $this->comment('Snapshot unchanged (no significant difference).'); + protected function displayAttemptsResults(array $allAttempts, array $results, Collection $keptIndices, float $totalDuration): void + { + $attempts = count($allAttempts); + $filteredAttempts = $attempts - $keptIndices->count(); + + foreach ($allAttempts as $i => $attempt) { + $isOutlier = ! $keptIndices->contains($i); + $improvement = $this->improvement($attempt); + $line = sprintf( + ' Attempt %d: Blade %s Blaze %s (%s%%)', + $i + 1, + $this->formatTime($attempt['blade_ms']), + $this->formatTime($attempt['blaze_ms']), + $improvement + ); + + $isOutlier + ? $this->line($line.' ← outlier') + : $this->line($line); + } + + [$headers, $rows, $snapshot] = $this->buildTable($results); + + $this->newLine(); + $this->table($headers, $rows); + + $this->newLine(); + $this->info( + "Median of {$attempts} attempts" + .($filteredAttempts ? " ({$filteredAttempts} outlier(s) excluded)" : '') + .", {$this->iterations} iterations x {$this->rounds} rounds, {$totalDuration}s total" + ); - return; + if ($snapshot) { + $rounds = $snapshot['rounds'] ?? 1; + $this->comment("Compared against baseline snapshot ({$snapshot['iterations']} iterations x {$rounds} rounds)"); + } + } + + protected function outputMarkdownAttempts(array $allAttempts, array $results, Collection $keptIndices, float $totalDuration): void + { + $attempts = count($allAttempts); + $filteredAttempts = $attempts - $keptIndices->count(); + $benchmarkName = array_key_first($results); + $medianResult = $results[$benchmarkName]; + $snapshot = $this->option('snapshot') ? null : $this->loadSnapshot(); + $snapshotData = $snapshot['benchmarks'][$benchmarkName] ?? null; + + $headers = ['Attempt', 'Blade', 'Blaze', 'Improvement']; + + $rows = []; + + // Individual attempt rows. + foreach ($allAttempts as $i => $attempt) { + $isOutlier = ! $keptIndices->contains($i); + + $rows[] = [ + '`#'.($i + 1).'`'.($isOutlier ? ' \*' : ''), + $this->formatTime($attempt['blade_ms']), + $this->formatTime($attempt['blaze_ms']), + $this->improvement($attempt).'%', + ]; } + // Snapshot row. + if ($snapshotData) { + $rows[] = [ + 'Snapshot', + $this->formatTime($snapshotData['blade_ms']), + $this->formatTime($snapshotData['blaze_ms']), + $snapshotData['improvement'].'%', + ]; + } + + // Result row (median with comparison deltas when snapshot exists). + $blade = $this->formatTime($medianResult['blade_ms']); + $blaze = $this->formatTime($medianResult['blaze_ms']); + $improvement = $this->improvement($medianResult).'%'; + + if ($snapshotData) { + $blade .= ' '.$this->formatChange($snapshotData['blade_ms'], $medianResult['blade_ms'], 1); + $blaze .= ' '.$this->formatChange($snapshotData['blaze_ms'], $medianResult['blaze_ms'], 1); + $improvement .= ' '.$this->formatImprovementChange($snapshotData['improvement'], $this->improvement($medianResult)); + } + + $rows[] = ['**Result**', "**{$blade}**", "**{$blaze}**", "**{$improvement}**"]; + + $allRows = collect([$headers, ...$rows]); + $widths = collect($headers)->keys()->map( + fn ($i) => $allRows->max(fn ($row) => mb_strlen($row[$i])) + ); + + $formatRow = fn ($cells) => '| '.collect($cells) + ->map(fn ($cell, $i) => Str::padRight($cell, $widths[$i])) + ->implode(' | ').' |'; + + $separator = '| '.$widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ').' |'; + + $md = collect([ + '## Benchmark Results', + '', + $formatRow($headers), + $separator, + ...collect($rows)->map($formatRow), + '', + '' + ."Median of {$attempts} attempts" + .($filteredAttempts ? ' (\* = outlier, excluded from result)' : '') + .", {$this->iterations} iterations x {$this->rounds} rounds, {$totalDuration}s total" + .'', + ])->implode("\n"); + + $this->output->writeln($md); + } + + protected function outputJsonAttemptsResults(string $benchmarkName, array $allAttempts, array $results, Collection $keptIndices, float $totalDuration): void + { + $attempts = count($allAttempts); + + $this->output->writeln(json_encode([ + 'iterations' => $this->iterations, + 'rounds' => $this->rounds, + 'attempts' => $attempts, + 'filtered_attempts' => $attempts - $keptIndices->count(), + 'total_duration_s' => $totalDuration, + 'attempts_detail' => collect($allAttempts)->map(fn ($attempt, $i) => [ + 'blade_ms' => $attempt['blade_ms'], + 'blaze_ms' => $attempt['blaze_ms'], + 'improvement' => $this->improvement($attempt), + 'outlier' => ! $keptIndices->contains($i), + ])->values()->all(), + 'benchmarks' => $results, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); + } + + protected function saveSnapshot(array $results): void + { $snapshot = [ 'iterations' => $this->iterations, 'rounds' => $this->rounds, @@ -193,32 +463,12 @@ protected function saveSnapshot(array $results): void $path = $this->snapshotPath(); - File::put($path, json_encode($snapshot, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n"); + File::put($path, json_encode($snapshot, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)."\n"); $this->newLine(); $this->info("Snapshot saved to {$path}"); } - protected function hasSignificantChange(array $results, array $snapshot): bool - { - if (array_keys($results) !== array_keys($snapshot['benchmarks'])) { - return true; - } - - return collect($results)->contains(function ($result, $name) use ($snapshot) { - $prev = $snapshot['benchmarks'][$name]; - - return $this->exceedsThreshold($prev['blade_ms'], $result['blade_ms'], 10.0) - || $this->exceedsThreshold($prev['blaze_ms'], $result['blaze_ms'], 5.0) - || $this->exceedsThreshold($prev['improvement'], $this->improvement($result), 0.5); - }); - } - - protected function exceedsThreshold(float $old, float $new, float $threshold): bool - { - return $old > 0 && abs(($new - $old) / $old * 100) >= $threshold; - } - protected function loadSnapshot(): ?array { $path = $this->snapshotPath(); @@ -234,7 +484,7 @@ protected function loadSnapshot(): ?array protected function snapshotPath(): string { - return dirname(__DIR__, 4) . '/benchmark-snapshot.json'; + return dirname(__DIR__, 4).'/benchmark-snapshot.json'; } protected function improvement(array $result): float @@ -244,7 +494,7 @@ protected function improvement(array $result): float : 0; } - protected function formatChange(float $old, float $new, float $threshold): string + protected function formatChange(float $old, float $new, float $threshold = 3): string { if ($old == 0) { return '(~)'; @@ -252,55 +502,50 @@ protected function formatChange(float $old, float $new, float $threshold): strin $change = ($new - $old) / abs($old) * 100; - if (abs($change) < $threshold) { + if (abs($change) <= $threshold) { return '(~)'; } $sign = $change > 0 ? '+' : ''; - return "({$sign}" . round($change, 1) . '%)'; + return "({$sign}".round($change, 1).'%)'; } - protected function getBenchmarks(): array + protected function formatImprovementChange(float $old, float $new, float $threshold = 0.2): string { - return [ - 'No attributes' => [ - 'blade' => 'bench.blade.no-attributes', - 'blaze' => 'bench.blaze.no-attributes', - ], - 'Attributes only' => [ - 'blade' => 'bench.blade.attributes', - 'blaze' => 'bench.blaze.attributes', - ], - 'Attributes + merge()' => [ - 'blade' => 'bench.blade.merge', - 'blaze' => 'bench.blaze.merge', - ], - 'Attributes + class()' => [ - 'blade' => 'bench.blade.class', - 'blaze' => 'bench.blaze.class', - ], - 'Props + attributes' => [ - 'blade' => 'bench.blade.props', - 'blaze' => 'bench.blaze.props', - ], - 'Default slot' => [ - 'blade' => 'bench.blade.slot', - 'blaze' => 'bench.blaze.slot', - ], - 'Named slots' => [ - 'blade' => 'bench.blade.named-slots', - 'blaze' => 'bench.blaze.named-slots', - ], - '`@aware` (nested)' => [ - 'blade' => 'bench.blade.aware', - 'blaze' => 'bench.blaze.aware', - ], - 'Attribute forwarding' => [ - 'blade' => 'bench.blade.forwarding', - 'blaze' => 'bench.blaze.forwarding', - ], - ]; + $delta = round($new - $old, 1); + + if (abs($delta) <= $threshold) { + return '(~)'; + } + + $sign = $delta > 0 ? '+' : ''; + + return "({$sign}{$delta}%)"; + } + + /** + * Return the indices of non-outlier values using the IQR method. + * + * Values below Q1 - 1.5*IQR or above Q3 + 1.5*IQR are considered outliers. + */ + protected function nonOutlierIndices(Collection $values): Collection + { + if ($values->count() < 4) { + return $values->keys(); + } + + $sorted = $values->sort()->values(); + $count = $sorted->count(); + + $q1 = $sorted[intdiv($count, 4)]; + $q3 = $sorted[intdiv($count * 3, 4)]; + $iqr = $q3 - $q1; + + $lower = $q1 - 1.5 * $iqr; + $upper = $q3 + 1.5 * $iqr; + + return $values->keys()->filter(fn ($i) => $values[$i] >= $lower && $values[$i] <= $upper)->values(); } protected function renderView(string $view): string @@ -315,6 +560,7 @@ protected function measureView(string $view): float protected function formatTime(float $ms): string { - return number_format($ms, 2) . 'ms'; + return number_format($ms, 2).'ms'; } + } diff --git a/workbench/app/Console/Commands/BenchmarkVarianceCommand.php b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php new file mode 100644 index 00000000..31783e87 --- /dev/null +++ b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php @@ -0,0 +1,305 @@ +iterations = (int) $this->option('iterations'); + $this->rounds = (int) $this->option('rounds'); + $this->warmupRounds = (int) $this->option('warmup'); + $runs = (int) $this->option('runs'); + + if ($runs < 1) { + $this->error('--runs must be at least 1.'); + + return Command::FAILURE; + } + + $benchmarkName = $this->argument('benchmark'); + $totalRuns = $runs + 1; + $quiet = $this->option('json') || $this->option('ci'); + $commandStart = microtime(true); + $runDurations = []; + + if (! $quiet) { + $bar = $this->output->createProgressBar($totalRuns); + $bar->setFormat(' %current%/%max% [%bar%] %message%'); + $bar->setMessage('Snapshot...'); + $bar->start(); + } + + // Step 1: Snapshot run. + $t = microtime(true); + $snapshotResult = $this->runBenchmarkInProcess($benchmarkName); + $runDurations[] = microtime(true) - $t; + + $this->saveSnapshot([$benchmarkName => $snapshotResult]); + + if (! $quiet) { + $bar->advance(); + $bar->setMessage('Benchmarking...'); + } + + // Step 2: Benchmark runs. + $allRuns = []; + + for ($i = 0; $i < $runs; $i++) { + $t = microtime(true); + $allRuns[] = $this->runBenchmarkInProcess($benchmarkName); + $runDurations[] = microtime(true) - $t; + + if (! $quiet) { + $bar->advance(); + } + } + + if (! $quiet) { + $bar->setMessage('Done!'); + $bar->finish(); + $this->newLine(); + } + + $avgRunDuration = round(array_sum($runDurations) / count($runDurations), 2); + $totalDuration = round(microtime(true) - $commandStart, 2); + + // Step 3: Display variance report. + if ($this->option('json')) { + $this->outputJson($snapshotResult, $allRuns, $avgRunDuration, $totalDuration); + } elseif ($this->option('ci')) { + $this->outputVarianceMarkdown($snapshotResult, $allRuns, $avgRunDuration, $totalDuration); + } else { + $this->displayVarianceResults($snapshotResult, $allRuns, $avgRunDuration, $totalDuration); + } + + return Command::SUCCESS; + } + + protected function runBenchmarkInProcess(string $name): array + { + $result = Process::path(base_path()) + ->timeout(300) + ->run([ + PHP_BINARY, 'artisan', 'benchmark', $name, + '--json', + '--iterations='.$this->iterations, + '--rounds='.$this->rounds, + '--warmup='.$this->warmupRounds, + '--attempts='.$this->option('attempts'), + ]); + + if (! $result->successful()) { + throw new \RuntimeException( + "Benchmark process failed for '{$name}': ".$result->errorOutput() + ); + } + + $data = json_decode($result->output(), true); + + if (! $data || ! isset($data['benchmarks'][$name])) { + throw new \RuntimeException( + "Invalid benchmark output for '{$name}': ".$result->output() + ); + } + + return $data['benchmarks'][$name]; + } + + protected function displayVarianceResults(array $snapshot, array $allRuns, float $avgRunDuration, float $totalDuration): void + { + $snapshotImprovement = $this->improvement($snapshot); + + $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blade_ms'], $run['blade_ms'])); + $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blaze_ms'], $run['blaze_ms'])); + $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run) - $snapshotImprovement, 1)); + + $headers = ['', 'Blade', 'Blaze', 'Improvement']; + $rows = [ + [ + 'Snapshot', + $this->formatTime($snapshot['blade_ms']), + $this->formatTime($snapshot['blaze_ms']), + $snapshotImprovement.'%', + ], + [ + 'Variance', + $this->formatVarianceRange($bladeChanges->min(), $bladeChanges->max()), + $this->formatVarianceRange($blazeChanges->min(), $blazeChanges->max()), + $this->formatVarianceRange($improvementChanges->min(), $improvementChanges->max()), + ], + [ + 'Std Dev', + '±'.$this->stddev($bladeChanges).'%', + '±'.$this->stddev($blazeChanges).'%', + '±'.$this->stddev($improvementChanges).'%', + ], + ]; + + $this->newLine(2); + $this->table($headers, $rows); + + $this->newLine(); + $this->comment( + count($allRuns)." runs x {$this->rounds} rounds x {$this->iterations} iterations" + .", ~{$avgRunDuration}s/run, {$totalDuration}s total" + ); + } + + protected function outputVarianceMarkdown(array $snapshot, array $allRuns, float $avgRunDuration, float $totalDuration): void + { + $snapshotImprovement = $this->improvement($snapshot); + + $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blade_ms'], $run['blade_ms'])); + $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blaze_ms'], $run['blaze_ms'])); + $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run) - $snapshotImprovement, 1)); + + $headers = ['', 'Blade', 'Blaze', 'Improvement']; + $rows = [ + [ + 'Snapshot', + $this->formatTime($snapshot['blade_ms']), + $this->formatTime($snapshot['blaze_ms']), + $snapshotImprovement.'%', + ], + [ + 'Variance', + $this->formatVarianceRange($bladeChanges->min(), $bladeChanges->max()), + $this->formatVarianceRange($blazeChanges->min(), $blazeChanges->max()), + $this->formatVarianceRange($improvementChanges->min(), $improvementChanges->max()), + ], + [ + 'Std Dev', + '±'.$this->stddev($bladeChanges).'%', + '±'.$this->stddev($blazeChanges).'%', + '±'.$this->stddev($improvementChanges).'%', + ], + ]; + + $allRows = collect([$headers, ...$rows]); + $widths = collect($headers)->keys()->map( + fn ($i) => $allRows->max(fn ($row) => mb_strlen($row[$i])) + ); + + $formatRow = fn ($cells) => '| '.collect($cells) + ->map(fn ($cell, $i) => Str::padRight($cell, $widths[$i])) + ->implode(' | ').' |'; + + $separator = '| '.$widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ').' |'; + + $md = collect([ + '## Benchmark Results', + '', + $formatRow($headers), + $separator, + ...collect($rows)->map($formatRow), + '', + ''.count($allRuns)." runs x {$this->rounds} rounds x {$this->iterations} iterations" + .", ~{$avgRunDuration}s/run, {$totalDuration}s total" + .'', + ])->implode("\n"); + + $this->output->writeln($md); + } + + protected function saveSnapshot(array $results): void + { + $snapshot = [ + 'iterations' => $this->iterations, + 'rounds' => $this->rounds, + 'benchmarks' => collect($results)->map(fn ($result) => [ + 'blade_ms' => $result['blade_ms'], + 'blaze_ms' => $result['blaze_ms'], + 'improvement' => $this->improvement($result), + ])->all(), + ]; + + \Illuminate\Support\Facades\File::put( + $this->snapshotPath(), + json_encode($snapshot, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)."\n" + ); + } + + protected function outputJson(array $snapshot, array $allRuns, float $avgRunDuration, float $totalDuration): void + { + $snapshotImprovement = $this->improvement($snapshot); + + $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blade_ms'], $run['blade_ms'])); + $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blaze_ms'], $run['blaze_ms'])); + $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run) - $snapshotImprovement, 1)); + + $this->output->writeln(json_encode([ + 'iterations' => $this->iterations, + 'rounds' => $this->rounds, + 'runs' => count($allRuns), + 'avg_run_duration_s' => $avgRunDuration, + 'total_duration_s' => $totalDuration, + 'snapshot' => [ + 'blade_ms' => $snapshot['blade_ms'], + 'blaze_ms' => $snapshot['blaze_ms'], + 'improvement' => $snapshotImprovement, + ], + 'variance' => [ + 'blade' => ['min' => $bladeChanges->min(), 'max' => $bladeChanges->max(), 'stddev' => $this->stddev($bladeChanges)], + 'blaze' => ['min' => $blazeChanges->min(), 'max' => $blazeChanges->max(), 'stddev' => $this->stddev($blazeChanges)], + 'improvement' => ['min' => $improvementChanges->min(), 'max' => $improvementChanges->max(), 'stddev' => $this->stddev($improvementChanges)], + ], + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); + } + + protected function stddev(\Illuminate\Support\Collection $values): float + { + $count = $values->count(); + + if ($count < 2) { + return 0.0; + } + + $mean = $values->avg(); + $sumSquares = $values->reduce(fn ($carry, $v) => $carry + ($v - $mean) ** 2, 0); + + return round(sqrt($sumSquares / ($count - 1)), 2); + } + + protected function formatVarianceRange(float $min, float $max): string + { + $fmt = function (float $v): string { + $rounded = round($v, 1); + + if ($rounded == 0) { + return '0%'; + } + + $sign = $rounded > 0 ? '+' : ''; + + return $sign.$rounded.'%'; + }; + + return $fmt($min).' / '.$fmt($max); + } + + protected function percentChange(float $old, float $new): float + { + return $old > 0 ? round(($new - $old) / $old * 100, 1) : 0; + } +}