diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1180fa16..08270423 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -77,7 +77,7 @@ jobs:
run: composer install --prefer-dist --no-progress --no-interaction
- name: Generate baseline snapshot
- run: vendor/bin/testbench benchmark --snapshot --ci
+ run: vendor/bin/testbench benchmark props --ci --iterations=5000 --rounds=10 --attempts=10 --snapshot
- name: Save baseline snapshot
run: cp benchmark-snapshot.json ${{ runner.temp }}/benchmark-snapshot.json
@@ -95,7 +95,7 @@ jobs:
run: echo "${{ github.event.pull_request.number }}" > benchmark-result.md
- name: Run benchmark
- run: vendor/bin/testbench benchmark --ci >> benchmark-result.md
+ run: vendor/bin/testbench benchmark props --ci --iterations=5000 --rounds=10 --attempts=10 >> benchmark-result.md
- name: Upload benchmark result
uses: actions/upload-artifact@v4
diff --git a/workbench/app/Console/Commands/BenchmarkCommand.php b/workbench/app/Console/Commands/BenchmarkCommand.php
index faad1577..0adf9519 100644
--- a/workbench/app/Console/Commands/BenchmarkCommand.php
+++ b/workbench/app/Console/Commands/BenchmarkCommand.php
@@ -4,21 +4,26 @@
use Illuminate\Console\Command;
use Illuminate\Support\Benchmark;
+use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Support\Facades\File;
+use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\View;
use Illuminate\Support\Str;
class BenchmarkCommand extends Command
{
protected $signature = 'benchmark
- {--iterations=10000 : Number of component renders per benchmark}
- {--rounds=5 : Number of timed rounds per benchmark}
+ {benchmark : Name of the benchmark to run}
+ {--iterations=5000 : Number of component renders per benchmark}
+ {--rounds=100 : Number of timed rounds per benchmark}
{--warmup=2 : Number of untimed warmup rounds}
+ {--attempts=5 : Number of times to run the entire benchmark in separate processes}
{--snapshot : Save results as the baseline snapshot}
+ {--json : Output results as JSON}
{--ci : Output a markdown table with no progress (for CI)}';
- protected $description = 'Run Blaze performance benchmarks';
+ protected $description = 'Run a Blaze performance benchmark';
protected int $iterations;
@@ -26,19 +31,42 @@ class BenchmarkCommand extends Command
protected int $warmupRounds;
+ protected int $filteredRounds = 0;
+
public function handle(): int
{
$this->iterations = (int) $this->option('iterations');
$this->rounds = (int) $this->option('rounds');
$this->warmupRounds = (int) $this->option('warmup');
+ $attempts = (int) $this->option('attempts');
+
+ if ($attempts < 1) {
+ $this->error('--attempts must be at least 1.');
+
+ return Command::FAILURE;
+ }
+
+ if ($attempts > 1) {
+ return $this->runMultipleAttempts($attempts);
+ }
+
+ $commandStart = microtime(true);
Artisan::call('view:clear');
- $results = $this->runBenchmarks();
+ $result = $this->runBenchmark();
+ $totalDuration = round(microtime(true) - $commandStart, 2);
- $this->option('ci')
- ? $this->outputMarkdown($results)
- : $this->displayResults($results);
+ $benchmarkName = $this->argument('benchmark');
+ $results = [$benchmarkName => $result];
+
+ if ($this->option('json')) {
+ $this->outputJsonResults($results, $totalDuration);
+ } elseif ($this->option('ci')) {
+ $this->outputMarkdown($results, $totalDuration);
+ } else {
+ $this->displayResults($results, $totalDuration);
+ }
if ($this->option('snapshot')) {
$this->saveSnapshot($results);
@@ -47,43 +75,40 @@ public function handle(): int
return Command::SUCCESS;
}
- protected function runBenchmarks(): array
+ protected function runBenchmark(): array
{
- $showProgress = ! $this->option('ci');
- $benchmarks = $this->getBenchmarks();
- $names = array_keys($benchmarks);
+ $benchmarkName = $this->argument('benchmark');
+ $bladeView = "bench.blade.{$benchmarkName}";
+ $blazeView = "bench.blaze.{$benchmarkName}";
+ $showProgress = ! $this->option('ci') && ! $this->option('json');
if ($showProgress) {
- $this->info("Running benchmarks ({$this->iterations} iterations x {$this->rounds} rounds)...");
+ $this->info("Running '{$benchmarkName}' ({$this->iterations} iterations x {$this->rounds} rounds)...");
$this->newLine();
}
- $totalSteps = (count($benchmarks) * $this->warmupRounds) + ($this->rounds * count($benchmarks));
+ $totalSteps = $this->warmupRounds + $this->rounds;
$bar = $showProgress ? $this->output->createProgressBar($totalSteps) : null;
- $bar?->setFormat(' %current%/%max% [%bar%] %message%');
+ $bar?->setFormat('[%bar%] %message%');
$bar?->setMessage('Warming up...');
$bar?->start();
// Warmup: compile views and stabilize opcache.
- foreach ($benchmarks as $benchmark) {
- for ($w = 0; $w < $this->warmupRounds; $w++) {
- $this->measureView($benchmark['blade']);
- $this->measureView($benchmark['blaze']);
- $bar?->advance();
- }
+ for ($w = 0; $w < $this->warmupRounds; $w++) {
+ $this->measureView($bladeView);
+ $this->measureView($blazeView);
+ $bar?->advance();
}
- $bladeTimes = array_fill_keys($names, []);
- $blazeTimes = array_fill_keys($names, []);
+ $bladeTimes = [];
+ $blazeTimes = [];
$bar?->setMessage('Benchmarking...');
for ($r = 0; $r < $this->rounds; $r++) {
- foreach ($benchmarks as $name => $benchmark) {
- $bladeTimes[$name][] = $this->measureView($benchmark['blade']);
- $blazeTimes[$name][] = $this->measureView($benchmark['blaze']);
- $bar?->advance();
- }
+ $bladeTimes[] = $this->measureView($bladeView);
+ $blazeTimes[] = $this->measureView($blazeView);
+ $bar?->advance();
}
$bar?->setMessage('Done!');
@@ -93,38 +118,144 @@ protected function runBenchmarks(): array
$this->newLine(2);
}
- return collect($names)->mapWithKeys(fn ($name) => [
- $name => [
- 'blade_ms' => round(collect($bladeTimes[$name])->median(), 2),
- 'blaze_ms' => round(collect($blazeTimes[$name])->median(), 2),
- ],
- ])->all();
+ $roundTotals = collect(range(0, $this->rounds - 1))->map(
+ fn ($r) => $bladeTimes[$r] + $blazeTimes[$r]
+ );
+
+ $keptRounds = $this->nonOutlierIndices($roundTotals);
+ $this->filteredRounds = $this->rounds - $keptRounds->count();
+
+ $bladeTimes = $keptRounds->map(fn ($r) => $bladeTimes[$r])->all();
+ $blazeTimes = $keptRounds->map(fn ($r) => $blazeTimes[$r])->all();
+
+ return [
+ 'blade_ms' => round(collect($bladeTimes)->median(), 2),
+ 'blaze_ms' => round(collect($blazeTimes)->median(), 2),
+ ];
+ }
+
+ protected function runMultipleAttempts(int $attempts): int
+ {
+ $benchmarkName = $this->argument('benchmark');
+ $showProgress = ! $this->option('ci') && ! $this->option('json');
+ $commandStart = microtime(true);
+
+ $allAttempts = [];
+
+ if ($showProgress) {
+ $this->info("Running '{$benchmarkName}' ({$attempts} attempts, {$this->iterations} iterations x {$this->rounds} rounds each)...");
+ $this->newLine();
+ $bar = $this->output->createProgressBar($attempts);
+ $bar->setFormat(' %current%/%max% [%bar%] %message%');
+ }
+
+ for ($i = 0; $i < $attempts; $i++) {
+ if ($showProgress) {
+ $bar->setMessage('Attempt '.($i + 1).'/'.$attempts.'...');
+ $i === 0 ? $bar->start() : $bar->display();
+ }
+
+ $result = Process::path(base_path())
+ ->timeout(300)
+ ->run([
+ PHP_BINARY, 'artisan', 'benchmark', $benchmarkName,
+ '--attempts=1',
+ '--json',
+ '--iterations='.$this->iterations,
+ '--rounds='.$this->rounds,
+ '--warmup='.$this->warmupRounds,
+ ]);
+
+ if (! $result->successful()) {
+ if ($showProgress) {
+ $this->newLine();
+ }
+
+ $this->error('Attempt '.($i + 1).' failed: '.$result->errorOutput());
+
+ return Command::FAILURE;
+ }
+
+ $data = json_decode($result->output(), true);
+
+ if (! $data || ! isset($data['benchmarks'][$benchmarkName])) {
+ if ($showProgress) {
+ $this->newLine();
+ }
+
+ $this->error('Invalid output from attempt '.($i + 1).': '.$result->output());
+
+ return Command::FAILURE;
+ }
+
+ $allAttempts[] = $data['benchmarks'][$benchmarkName];
+
+ if ($showProgress) {
+ $bar->advance();
+ }
+ }
+
+ if ($showProgress) {
+ $bar->setMessage('Done!');
+ $bar->finish();
+ $this->newLine(2);
+ }
+
+ $totalDuration = round(microtime(true) - $commandStart, 2);
+
+ // Filter outlier attempts — if either blade or blaze is an outlier, drop the whole attempt.
+ $bladeValues = collect($allAttempts)->map(fn ($r) => $r['blade_ms']);
+ $blazeValues = collect($allAttempts)->map(fn ($r) => $r['blaze_ms']);
+ $keptIndices = $this->nonOutlierIndices($bladeValues)
+ ->intersect($this->nonOutlierIndices($blazeValues))
+ ->values();
+
+ $medianResult = [
+ 'blade_ms' => round($keptIndices->map(fn ($i) => $allAttempts[$i]['blade_ms'])->median(), 2),
+ 'blaze_ms' => round($keptIndices->map(fn ($i) => $allAttempts[$i]['blaze_ms'])->median(), 2),
+ ];
+
+ $results = [$benchmarkName => $medianResult];
+
+ if ($this->option('json')) {
+ $this->outputJsonAttemptsResults($benchmarkName, $allAttempts, $results, $keptIndices, $totalDuration);
+ } elseif ($this->option('ci')) {
+ $this->outputMarkdownAttempts($allAttempts, $results, $keptIndices, $totalDuration);
+ } else {
+ $this->displayAttemptsResults($allAttempts, $results, $keptIndices, $totalDuration);
+ }
+
+ if ($this->option('snapshot')) {
+ $this->saveSnapshot($results);
+ }
+
+ return Command::SUCCESS;
}
protected function buildTable(array $results): array
{
$snapshot = $this->option('snapshot') ? null : $this->loadSnapshot();
- $headers = ['Benchmark', 'Blade', 'Blaze', 'Improvement'];
+ $headers = ['Blade', 'Blaze', 'Improvement'];
$rows = collect($results)->map(function ($result, $name) use ($snapshot) {
$blade = $this->formatTime($result['blade_ms']);
$blaze = $this->formatTime($result['blaze_ms']);
- $improvement = $this->improvement($result) . '%';
+ $improvement = $this->improvement($result).'%';
if ($prev = $snapshot['benchmarks'][$name] ?? null) {
- $blade .= ' ' . $this->formatChange($prev['blade_ms'], $result['blade_ms'], 10.0);
- $blaze .= ' ' . $this->formatChange($prev['blaze_ms'], $result['blaze_ms'], 5.0);
- $improvement .= ' ' . $this->formatChange($prev['improvement'], $this->improvement($result), 1.0);
+ $blade .= ' '.$this->formatChange($prev['blade_ms'], $result['blade_ms'], 1);
+ $blaze .= ' '.$this->formatChange($prev['blaze_ms'], $result['blaze_ms'], 1);
+ $improvement .= ' '.$this->formatImprovementChange($prev['improvement'], $this->improvement($result));
}
- return [$name, $blade, $blaze, $improvement];
+ return [$blade, $blaze, $improvement];
})->values()->all();
return [$headers, $rows, $snapshot];
}
- protected function displayResults(array $results): void
+ protected function displayResults(array $results, float $totalDuration): void
{
[$headers, $rows, $snapshot] = $this->buildTable($results);
@@ -132,15 +263,16 @@ protected function displayResults(array $results): void
$this->table($headers, $rows);
$this->newLine();
- $this->info("{$this->iterations} iterations x {$this->rounds} rounds per benchmark");
+ $this->info("{$this->iterations} iterations x {$this->rounds} rounds, {$totalDuration}s total");
+ $this->comment("{$this->filteredRounds} outlier rounds excluded (IQR method)");
if ($snapshot) {
$rounds = $snapshot['rounds'] ?? 1;
- $this->comment("Compared against snapshot ({$snapshot['iterations']} iterations x {$rounds} rounds)");
+ $this->comment("Compared against baseline snapshot ({$snapshot['iterations']} iterations x {$rounds} rounds)");
}
}
- protected function outputMarkdown(array $results): void
+ protected function outputMarkdown(array $results, float $totalDuration): void
{
[$headers, $rows, $snapshot] = $this->buildTable($results);
@@ -149,11 +281,11 @@ protected function outputMarkdown(array $results): void
fn ($i) => $allRows->max(fn ($row) => mb_strlen($row[$i]))
);
- $formatRow = fn ($cells) => '| ' . collect($cells)
+ $formatRow = fn ($cells) => '| '.collect($cells)
->map(fn ($cell, $i) => Str::padRight($cell, $widths[$i]))
- ->implode(' | ') . ' |';
+ ->implode(' | ').' |';
- $separator = '| ' . $widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ') . ' |';
+ $separator = '| '.$widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ').' |';
$md = collect([
'## Benchmark Results',
@@ -162,25 +294,163 @@ protected function outputMarkdown(array $results): void
$separator,
...collect($rows)->map($formatRow),
'',
- '' . "{$this->iterations} iterations x {$this->rounds} rounds per benchmark"
- . ($snapshot ? ' — compared against committed snapshot' : '')
- . '',
+ ''."{$this->iterations} iterations x {$this->rounds} rounds, {$totalDuration}s total"
+ ." — {$this->filteredRounds} outlier rounds excluded (IQR)"
+ .($snapshot ? ' — compared against baseline snapshot' : '')
+ .'',
])->implode("\n");
$this->output->writeln($md);
}
- protected function saveSnapshot(array $results): void
+ protected function outputJsonResults(array $results, float $totalDuration): void
{
- $existing = $this->loadSnapshot();
+ $this->output->writeln(json_encode([
+ 'iterations' => $this->iterations,
+ 'rounds' => $this->rounds,
+ 'filtered_rounds' => $this->filteredRounds,
+ 'total_duration_s' => $totalDuration,
+ 'benchmarks' => $results,
+ ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
+ }
- if ($existing && ! $this->hasSignificantChange($results, $existing)) {
- $this->newLine();
- $this->comment('Snapshot unchanged (no significant difference).');
+ protected function displayAttemptsResults(array $allAttempts, array $results, Collection $keptIndices, float $totalDuration): void
+ {
+ $attempts = count($allAttempts);
+ $filteredAttempts = $attempts - $keptIndices->count();
+
+ foreach ($allAttempts as $i => $attempt) {
+ $isOutlier = ! $keptIndices->contains($i);
+ $improvement = $this->improvement($attempt);
+ $line = sprintf(
+ ' Attempt %d: Blade %s Blaze %s (%s%%)',
+ $i + 1,
+ $this->formatTime($attempt['blade_ms']),
+ $this->formatTime($attempt['blaze_ms']),
+ $improvement
+ );
+
+ $isOutlier
+ ? $this->line($line.' ← outlier>')
+ : $this->line($line);
+ }
+
+ [$headers, $rows, $snapshot] = $this->buildTable($results);
+
+ $this->newLine();
+ $this->table($headers, $rows);
+
+ $this->newLine();
+ $this->info(
+ "Median of {$attempts} attempts"
+ .($filteredAttempts ? " ({$filteredAttempts} outlier(s) excluded)" : '')
+ .", {$this->iterations} iterations x {$this->rounds} rounds, {$totalDuration}s total"
+ );
- return;
+ if ($snapshot) {
+ $rounds = $snapshot['rounds'] ?? 1;
+ $this->comment("Compared against baseline snapshot ({$snapshot['iterations']} iterations x {$rounds} rounds)");
+ }
+ }
+
+ protected function outputMarkdownAttempts(array $allAttempts, array $results, Collection $keptIndices, float $totalDuration): void
+ {
+ $attempts = count($allAttempts);
+ $filteredAttempts = $attempts - $keptIndices->count();
+ $benchmarkName = array_key_first($results);
+ $medianResult = $results[$benchmarkName];
+ $snapshot = $this->option('snapshot') ? null : $this->loadSnapshot();
+ $snapshotData = $snapshot['benchmarks'][$benchmarkName] ?? null;
+
+ $headers = ['Attempt', 'Blade', 'Blaze', 'Improvement'];
+
+ $rows = [];
+
+ // Individual attempt rows.
+ foreach ($allAttempts as $i => $attempt) {
+ $isOutlier = ! $keptIndices->contains($i);
+
+ $rows[] = [
+ '`#'.($i + 1).'`'.($isOutlier ? ' \*' : ''),
+ $this->formatTime($attempt['blade_ms']),
+ $this->formatTime($attempt['blaze_ms']),
+ $this->improvement($attempt).'%',
+ ];
}
+ // Snapshot row.
+ if ($snapshotData) {
+ $rows[] = [
+ 'Snapshot',
+ $this->formatTime($snapshotData['blade_ms']),
+ $this->formatTime($snapshotData['blaze_ms']),
+ $snapshotData['improvement'].'%',
+ ];
+ }
+
+ // Result row (median with comparison deltas when snapshot exists).
+ $blade = $this->formatTime($medianResult['blade_ms']);
+ $blaze = $this->formatTime($medianResult['blaze_ms']);
+ $improvement = $this->improvement($medianResult).'%';
+
+ if ($snapshotData) {
+ $blade .= ' '.$this->formatChange($snapshotData['blade_ms'], $medianResult['blade_ms'], 1);
+ $blaze .= ' '.$this->formatChange($snapshotData['blaze_ms'], $medianResult['blaze_ms'], 1);
+ $improvement .= ' '.$this->formatImprovementChange($snapshotData['improvement'], $this->improvement($medianResult));
+ }
+
+ $rows[] = ['**Result**', "**{$blade}**", "**{$blaze}**", "**{$improvement}**"];
+
+ $allRows = collect([$headers, ...$rows]);
+ $widths = collect($headers)->keys()->map(
+ fn ($i) => $allRows->max(fn ($row) => mb_strlen($row[$i]))
+ );
+
+ $formatRow = fn ($cells) => '| '.collect($cells)
+ ->map(fn ($cell, $i) => Str::padRight($cell, $widths[$i]))
+ ->implode(' | ').' |';
+
+ $separator = '| '.$widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ').' |';
+
+ $md = collect([
+ '## Benchmark Results',
+ '',
+ $formatRow($headers),
+ $separator,
+ ...collect($rows)->map($formatRow),
+ '',
+ ''
+ ."Median of {$attempts} attempts"
+ .($filteredAttempts ? ' (\* = outlier, excluded from result)' : '')
+ .", {$this->iterations} iterations x {$this->rounds} rounds, {$totalDuration}s total"
+ .'',
+ ])->implode("\n");
+
+ $this->output->writeln($md);
+ }
+
+ protected function outputJsonAttemptsResults(string $benchmarkName, array $allAttempts, array $results, Collection $keptIndices, float $totalDuration): void
+ {
+ $attempts = count($allAttempts);
+
+ $this->output->writeln(json_encode([
+ 'iterations' => $this->iterations,
+ 'rounds' => $this->rounds,
+ 'attempts' => $attempts,
+ 'filtered_attempts' => $attempts - $keptIndices->count(),
+ 'total_duration_s' => $totalDuration,
+ 'attempts_detail' => collect($allAttempts)->map(fn ($attempt, $i) => [
+ 'blade_ms' => $attempt['blade_ms'],
+ 'blaze_ms' => $attempt['blaze_ms'],
+ 'improvement' => $this->improvement($attempt),
+ 'outlier' => ! $keptIndices->contains($i),
+ ])->values()->all(),
+ 'benchmarks' => $results,
+ ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
+ }
+
+ protected function saveSnapshot(array $results): void
+ {
$snapshot = [
'iterations' => $this->iterations,
'rounds' => $this->rounds,
@@ -193,32 +463,12 @@ protected function saveSnapshot(array $results): void
$path = $this->snapshotPath();
- File::put($path, json_encode($snapshot, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n");
+ File::put($path, json_encode($snapshot, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)."\n");
$this->newLine();
$this->info("Snapshot saved to {$path}");
}
- protected function hasSignificantChange(array $results, array $snapshot): bool
- {
- if (array_keys($results) !== array_keys($snapshot['benchmarks'])) {
- return true;
- }
-
- return collect($results)->contains(function ($result, $name) use ($snapshot) {
- $prev = $snapshot['benchmarks'][$name];
-
- return $this->exceedsThreshold($prev['blade_ms'], $result['blade_ms'], 10.0)
- || $this->exceedsThreshold($prev['blaze_ms'], $result['blaze_ms'], 5.0)
- || $this->exceedsThreshold($prev['improvement'], $this->improvement($result), 0.5);
- });
- }
-
- protected function exceedsThreshold(float $old, float $new, float $threshold): bool
- {
- return $old > 0 && abs(($new - $old) / $old * 100) >= $threshold;
- }
-
protected function loadSnapshot(): ?array
{
$path = $this->snapshotPath();
@@ -234,7 +484,7 @@ protected function loadSnapshot(): ?array
protected function snapshotPath(): string
{
- return dirname(__DIR__, 4) . '/benchmark-snapshot.json';
+ return dirname(__DIR__, 4).'/benchmark-snapshot.json';
}
protected function improvement(array $result): float
@@ -244,7 +494,7 @@ protected function improvement(array $result): float
: 0;
}
- protected function formatChange(float $old, float $new, float $threshold): string
+ protected function formatChange(float $old, float $new, float $threshold = 3): string
{
if ($old == 0) {
return '(~)';
@@ -252,55 +502,50 @@ protected function formatChange(float $old, float $new, float $threshold): strin
$change = ($new - $old) / abs($old) * 100;
- if (abs($change) < $threshold) {
+ if (abs($change) <= $threshold) {
return '(~)';
}
$sign = $change > 0 ? '+' : '';
- return "({$sign}" . round($change, 1) . '%)';
+ return "({$sign}".round($change, 1).'%)';
}
- protected function getBenchmarks(): array
+ protected function formatImprovementChange(float $old, float $new, float $threshold = 0.2): string
{
- return [
- 'No attributes' => [
- 'blade' => 'bench.blade.no-attributes',
- 'blaze' => 'bench.blaze.no-attributes',
- ],
- 'Attributes only' => [
- 'blade' => 'bench.blade.attributes',
- 'blaze' => 'bench.blaze.attributes',
- ],
- 'Attributes + merge()' => [
- 'blade' => 'bench.blade.merge',
- 'blaze' => 'bench.blaze.merge',
- ],
- 'Attributes + class()' => [
- 'blade' => 'bench.blade.class',
- 'blaze' => 'bench.blaze.class',
- ],
- 'Props + attributes' => [
- 'blade' => 'bench.blade.props',
- 'blaze' => 'bench.blaze.props',
- ],
- 'Default slot' => [
- 'blade' => 'bench.blade.slot',
- 'blaze' => 'bench.blaze.slot',
- ],
- 'Named slots' => [
- 'blade' => 'bench.blade.named-slots',
- 'blaze' => 'bench.blaze.named-slots',
- ],
- '`@aware` (nested)' => [
- 'blade' => 'bench.blade.aware',
- 'blaze' => 'bench.blaze.aware',
- ],
- 'Attribute forwarding' => [
- 'blade' => 'bench.blade.forwarding',
- 'blaze' => 'bench.blaze.forwarding',
- ],
- ];
+ $delta = round($new - $old, 1);
+
+ if (abs($delta) <= $threshold) {
+ return '(~)';
+ }
+
+ $sign = $delta > 0 ? '+' : '';
+
+ return "({$sign}{$delta}%)";
+ }
+
+ /**
+ * Return the indices of non-outlier values using the IQR method.
+ *
+ * Values below Q1 - 1.5*IQR or above Q3 + 1.5*IQR are considered outliers.
+ */
+ protected function nonOutlierIndices(Collection $values): Collection
+ {
+ if ($values->count() < 4) {
+ return $values->keys();
+ }
+
+ $sorted = $values->sort()->values();
+ $count = $sorted->count();
+
+ $q1 = $sorted[intdiv($count, 4)];
+ $q3 = $sorted[intdiv($count * 3, 4)];
+ $iqr = $q3 - $q1;
+
+ $lower = $q1 - 1.5 * $iqr;
+ $upper = $q3 + 1.5 * $iqr;
+
+ return $values->keys()->filter(fn ($i) => $values[$i] >= $lower && $values[$i] <= $upper)->values();
}
protected function renderView(string $view): string
@@ -315,6 +560,7 @@ protected function measureView(string $view): float
protected function formatTime(float $ms): string
{
- return number_format($ms, 2) . 'ms';
+ return number_format($ms, 2).'ms';
}
+
}
diff --git a/workbench/app/Console/Commands/BenchmarkVarianceCommand.php b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php
new file mode 100644
index 00000000..31783e87
--- /dev/null
+++ b/workbench/app/Console/Commands/BenchmarkVarianceCommand.php
@@ -0,0 +1,305 @@
+iterations = (int) $this->option('iterations');
+ $this->rounds = (int) $this->option('rounds');
+ $this->warmupRounds = (int) $this->option('warmup');
+ $runs = (int) $this->option('runs');
+
+ if ($runs < 1) {
+ $this->error('--runs must be at least 1.');
+
+ return Command::FAILURE;
+ }
+
+ $benchmarkName = $this->argument('benchmark');
+ $totalRuns = $runs + 1;
+ $quiet = $this->option('json') || $this->option('ci');
+ $commandStart = microtime(true);
+ $runDurations = [];
+
+ if (! $quiet) {
+ $bar = $this->output->createProgressBar($totalRuns);
+ $bar->setFormat(' %current%/%max% [%bar%] %message%');
+ $bar->setMessage('Snapshot...');
+ $bar->start();
+ }
+
+ // Step 1: Snapshot run.
+ $t = microtime(true);
+ $snapshotResult = $this->runBenchmarkInProcess($benchmarkName);
+ $runDurations[] = microtime(true) - $t;
+
+ $this->saveSnapshot([$benchmarkName => $snapshotResult]);
+
+ if (! $quiet) {
+ $bar->advance();
+ $bar->setMessage('Benchmarking...');
+ }
+
+ // Step 2: Benchmark runs.
+ $allRuns = [];
+
+ for ($i = 0; $i < $runs; $i++) {
+ $t = microtime(true);
+ $allRuns[] = $this->runBenchmarkInProcess($benchmarkName);
+ $runDurations[] = microtime(true) - $t;
+
+ if (! $quiet) {
+ $bar->advance();
+ }
+ }
+
+ if (! $quiet) {
+ $bar->setMessage('Done!');
+ $bar->finish();
+ $this->newLine();
+ }
+
+ $avgRunDuration = round(array_sum($runDurations) / count($runDurations), 2);
+ $totalDuration = round(microtime(true) - $commandStart, 2);
+
+ // Step 3: Display variance report.
+ if ($this->option('json')) {
+ $this->outputJson($snapshotResult, $allRuns, $avgRunDuration, $totalDuration);
+ } elseif ($this->option('ci')) {
+ $this->outputVarianceMarkdown($snapshotResult, $allRuns, $avgRunDuration, $totalDuration);
+ } else {
+ $this->displayVarianceResults($snapshotResult, $allRuns, $avgRunDuration, $totalDuration);
+ }
+
+ return Command::SUCCESS;
+ }
+
+ protected function runBenchmarkInProcess(string $name): array
+ {
+ $result = Process::path(base_path())
+ ->timeout(300)
+ ->run([
+ PHP_BINARY, 'artisan', 'benchmark', $name,
+ '--json',
+ '--iterations='.$this->iterations,
+ '--rounds='.$this->rounds,
+ '--warmup='.$this->warmupRounds,
+ '--attempts='.$this->option('attempts'),
+ ]);
+
+ if (! $result->successful()) {
+ throw new \RuntimeException(
+ "Benchmark process failed for '{$name}': ".$result->errorOutput()
+ );
+ }
+
+ $data = json_decode($result->output(), true);
+
+ if (! $data || ! isset($data['benchmarks'][$name])) {
+ throw new \RuntimeException(
+ "Invalid benchmark output for '{$name}': ".$result->output()
+ );
+ }
+
+ return $data['benchmarks'][$name];
+ }
+
+ protected function displayVarianceResults(array $snapshot, array $allRuns, float $avgRunDuration, float $totalDuration): void
+ {
+ $snapshotImprovement = $this->improvement($snapshot);
+
+ $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blade_ms'], $run['blade_ms']));
+ $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blaze_ms'], $run['blaze_ms']));
+ $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run) - $snapshotImprovement, 1));
+
+ $headers = ['', 'Blade', 'Blaze', 'Improvement'];
+ $rows = [
+ [
+ 'Snapshot',
+ $this->formatTime($snapshot['blade_ms']),
+ $this->formatTime($snapshot['blaze_ms']),
+ $snapshotImprovement.'%',
+ ],
+ [
+ 'Variance',
+ $this->formatVarianceRange($bladeChanges->min(), $bladeChanges->max()),
+ $this->formatVarianceRange($blazeChanges->min(), $blazeChanges->max()),
+ $this->formatVarianceRange($improvementChanges->min(), $improvementChanges->max()),
+ ],
+ [
+ 'Std Dev',
+ '±'.$this->stddev($bladeChanges).'%',
+ '±'.$this->stddev($blazeChanges).'%',
+ '±'.$this->stddev($improvementChanges).'%',
+ ],
+ ];
+
+ $this->newLine(2);
+ $this->table($headers, $rows);
+
+ $this->newLine();
+ $this->comment(
+ count($allRuns)." runs x {$this->rounds} rounds x {$this->iterations} iterations"
+ .", ~{$avgRunDuration}s/run, {$totalDuration}s total"
+ );
+ }
+
+ protected function outputVarianceMarkdown(array $snapshot, array $allRuns, float $avgRunDuration, float $totalDuration): void
+ {
+ $snapshotImprovement = $this->improvement($snapshot);
+
+ $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blade_ms'], $run['blade_ms']));
+ $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blaze_ms'], $run['blaze_ms']));
+ $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run) - $snapshotImprovement, 1));
+
+ $headers = ['', 'Blade', 'Blaze', 'Improvement'];
+ $rows = [
+ [
+ 'Snapshot',
+ $this->formatTime($snapshot['blade_ms']),
+ $this->formatTime($snapshot['blaze_ms']),
+ $snapshotImprovement.'%',
+ ],
+ [
+ 'Variance',
+ $this->formatVarianceRange($bladeChanges->min(), $bladeChanges->max()),
+ $this->formatVarianceRange($blazeChanges->min(), $blazeChanges->max()),
+ $this->formatVarianceRange($improvementChanges->min(), $improvementChanges->max()),
+ ],
+ [
+ 'Std Dev',
+ '±'.$this->stddev($bladeChanges).'%',
+ '±'.$this->stddev($blazeChanges).'%',
+ '±'.$this->stddev($improvementChanges).'%',
+ ],
+ ];
+
+ $allRows = collect([$headers, ...$rows]);
+ $widths = collect($headers)->keys()->map(
+ fn ($i) => $allRows->max(fn ($row) => mb_strlen($row[$i]))
+ );
+
+ $formatRow = fn ($cells) => '| '.collect($cells)
+ ->map(fn ($cell, $i) => Str::padRight($cell, $widths[$i]))
+ ->implode(' | ').' |';
+
+ $separator = '| '.$widths->map(fn ($w) => str_repeat('-', $w))->implode(' | ').' |';
+
+ $md = collect([
+ '## Benchmark Results',
+ '',
+ $formatRow($headers),
+ $separator,
+ ...collect($rows)->map($formatRow),
+ '',
+ ''.count($allRuns)." runs x {$this->rounds} rounds x {$this->iterations} iterations"
+ .", ~{$avgRunDuration}s/run, {$totalDuration}s total"
+ .'',
+ ])->implode("\n");
+
+ $this->output->writeln($md);
+ }
+
+ protected function saveSnapshot(array $results): void
+ {
+ $snapshot = [
+ 'iterations' => $this->iterations,
+ 'rounds' => $this->rounds,
+ 'benchmarks' => collect($results)->map(fn ($result) => [
+ 'blade_ms' => $result['blade_ms'],
+ 'blaze_ms' => $result['blaze_ms'],
+ 'improvement' => $this->improvement($result),
+ ])->all(),
+ ];
+
+ \Illuminate\Support\Facades\File::put(
+ $this->snapshotPath(),
+ json_encode($snapshot, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)."\n"
+ );
+ }
+
+ protected function outputJson(array $snapshot, array $allRuns, float $avgRunDuration, float $totalDuration): void
+ {
+ $snapshotImprovement = $this->improvement($snapshot);
+
+ $bladeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blade_ms'], $run['blade_ms']));
+ $blazeChanges = collect($allRuns)->map(fn ($run) => $this->percentChange($snapshot['blaze_ms'], $run['blaze_ms']));
+ $improvementChanges = collect($allRuns)->map(fn ($run) => round($this->improvement($run) - $snapshotImprovement, 1));
+
+ $this->output->writeln(json_encode([
+ 'iterations' => $this->iterations,
+ 'rounds' => $this->rounds,
+ 'runs' => count($allRuns),
+ 'avg_run_duration_s' => $avgRunDuration,
+ 'total_duration_s' => $totalDuration,
+ 'snapshot' => [
+ 'blade_ms' => $snapshot['blade_ms'],
+ 'blaze_ms' => $snapshot['blaze_ms'],
+ 'improvement' => $snapshotImprovement,
+ ],
+ 'variance' => [
+ 'blade' => ['min' => $bladeChanges->min(), 'max' => $bladeChanges->max(), 'stddev' => $this->stddev($bladeChanges)],
+ 'blaze' => ['min' => $blazeChanges->min(), 'max' => $blazeChanges->max(), 'stddev' => $this->stddev($blazeChanges)],
+ 'improvement' => ['min' => $improvementChanges->min(), 'max' => $improvementChanges->max(), 'stddev' => $this->stddev($improvementChanges)],
+ ],
+ ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
+ }
+
+ protected function stddev(\Illuminate\Support\Collection $values): float
+ {
+ $count = $values->count();
+
+ if ($count < 2) {
+ return 0.0;
+ }
+
+ $mean = $values->avg();
+ $sumSquares = $values->reduce(fn ($carry, $v) => $carry + ($v - $mean) ** 2, 0);
+
+ return round(sqrt($sumSquares / ($count - 1)), 2);
+ }
+
+ protected function formatVarianceRange(float $min, float $max): string
+ {
+ $fmt = function (float $v): string {
+ $rounded = round($v, 1);
+
+ if ($rounded == 0) {
+ return '0%';
+ }
+
+ $sign = $rounded > 0 ? '+' : '';
+
+ return $sign.$rounded.'%';
+ };
+
+ return $fmt($min).' / '.$fmt($max);
+ }
+
+ protected function percentChange(float $old, float $new): float
+ {
+ return $old > 0 ? round(($new - $old) / $old * 100, 1) : 0;
+ }
+}