diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..8bf8bfb --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,193 @@ +# Performance Benchmark CI Workflow +# +# This workflow automatically runs performance benchmarks on pull requests +# and posts a comparison comment showing speed differences between the PR +# and the base branch (master). +# +# The workflow: +# 1. Runs benchmarks on the PR branch +# 2. Runs benchmarks on the base branch (master) +# 3. Compares the results and identifies performance regressions/improvements +# 4. Posts a detailed comparison table as a PR comment +# +# Performance changes > 10% are flagged as significant to help maintainers +# catch performance regressions before merging. + +name: Performance Benchmark + +on: + pull_request: + branches: [master] + +permissions: + contents: read + pull-requests: write + +jobs: + benchmark: + runs-on: ubuntu-latest + env: + BENCHMARK_PATH: tests/benchmarks/ + steps: + - name: Checkout PR branch + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install -r requirements_dev.txt + + - name: Run benchmarks on PR branch + run: | + python3 -m pytest \ + --benchmark-only \ + --benchmark-json=pr-benchmark.json \ + --benchmark-columns=mean,stddev,iqr,ops,rounds \ + ${{ env.BENCHMARK_PATH }} + + - name: Store PR benchmark result + uses: actions/upload-artifact@v4 + with: + name: pr-benchmark + path: pr-benchmark.json + + - name: Checkout base branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.ref }} + + - name: Install dependencies for base branch + run: | + pip install -r requirements.txt + pip install -r requirements_dev.txt + + - name: Run benchmarks on base branch + run: | + python3 -m pytest \ + --benchmark-only \ + --benchmark-json=base-benchmark.json \ + --benchmark-columns=mean,stddev,iqr,ops,rounds \ + ${{ env.BENCHMARK_PATH }} + + - name: Store base benchmark result + uses: actions/upload-artifact@v4 + with: + name: base-benchmark + path: base-benchmark.json + + # Checkout PR branch again to ensure artifact downloads happen in correct context + - name: Checkout PR branch again + uses: actions/checkout@v4 + + - name: Download PR benchmark result + uses: actions/download-artifact@v4 + with: + name: pr-benchmark + + - name: Download base benchmark result + uses: actions/download-artifact@v4 + with: + name: base-benchmark + + - name: Compare benchmarks and comment + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + + // Read benchmark results + const prBenchmark = JSON.parse(fs.readFileSync('pr-benchmark.json', 'utf8')); + const baseBenchmark = JSON.parse(fs.readFileSync('base-benchmark.json', 'utf8')); + + // Create a map of base benchmarks for easy lookup + const baseMap = {}; + baseBenchmark.benchmarks.forEach(bench => { + baseMap[bench.name] = bench; + }); + + // Build comparison table + let tableRows = []; + let significantChanges = []; + + prBenchmark.benchmarks.forEach(prBench => { + const baseBench = baseMap[prBench.name]; + if (!baseBench) { + tableRows.push({ + name: prBench.name, + status: '🆕 NEW', + prMean: prBench.stats.mean, + baseMean: '-', + change: '-' + }); + return; + } + + const prMean = prBench.stats.mean; + const baseMean = baseBench.stats.mean; + const changePercent = ((prMean - baseMean) / baseMean) * 100; + + let status = '✅'; + let changeStr = changePercent.toFixed(2) + '%'; + + // Flag significant changes (> 10% regression) + if (changePercent > 10) { + status = '⚠️ SLOWER'; + significantChanges.push(`${prBench.name}: ${changePercent.toFixed(2)}% slower`); + } else if (changePercent < -10) { + status = '🚀 FASTER'; + } + + tableRows.push({ + name: prBench.name, + status: status, + prMean: prMean, + baseMean: baseMean, + change: changeStr + }); + }); + + // Format table + let comment = '## 📊 Performance Benchmark Results\n\n'; + + if (significantChanges.length > 0) { + comment += '### ⚠️ Significant Performance Changes Detected\n\n'; + significantChanges.forEach(change => { + comment += `- ${change}\n`; + }); + comment += '\n'; + } + + comment += '### Detailed Comparison\n\n'; + comment += '| Benchmark | Status | PR Mean (s) | Base Mean (s) | Change |\n'; + comment += '|-----------|--------|-------------|---------------|--------|\n'; + + tableRows.forEach(row => { + const prMeanStr = typeof row.prMean === 'number' ? row.prMean.toExponential(4) : row.prMean; + const baseMeanStr = typeof row.baseMean === 'number' ? row.baseMean.toExponential(4) : row.baseMean; + comment += `| ${row.name} | ${row.status} | ${prMeanStr} | ${baseMeanStr} | ${row.change} |\n`; + }); + + comment += '\n---\n'; + comment += `**Base Branch:** \`${context.payload.pull_request.base.ref}\` (${context.payload.pull_request.base.sha.substring(0, 7)})\n`; + comment += `**PR Branch:** \`${context.payload.pull_request.head.ref}\` (${context.payload.pull_request.head.sha.substring(0, 7)})\n`; + if (prBenchmark.machine_info) { + if (prBenchmark.machine_info.python_version) { + comment += `**Python Version:** ${prBenchmark.machine_info.python_version}\n`; + } + if (prBenchmark.machine_info.system && prBenchmark.machine_info.release) { + comment += `**Platform:** ${prBenchmark.machine_info.system} ${prBenchmark.machine_info.release}\n`; + } + } + + // Post comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: comment + });