ci: add automatic performance benchmark CI for pull requests

.github/workflows/benchmark.yml

@@ -0,0 +1,193 @@
+# Performance Benchmark CI Workflow
+#
+# This workflow automatically runs performance benchmarks on pull requests
+# and posts a comparison comment showing speed differences between the PR
+# and the base branch (master).
+#
+# The workflow:
+# 1. Runs benchmarks on the PR branch
+# 2. Runs benchmarks on the base branch (master)
+# 3. Compares the results and identifies performance regressions/improvements
+# 4. Posts a detailed comparison table as a PR comment
+#
+# Performance changes > 10% are flagged as significant to help maintainers
+# catch performance regressions before merging.
+
+name: Performance Benchmark
+
+on:
+  pull_request:
+    branches: [master]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    env:
+      BENCHMARK_PATH: tests/benchmarks/
+    steps:
+      - name: Checkout PR branch
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install -r requirements_dev.txt
+
+      - name: Run benchmarks on PR branch
+        run: |
+          python3 -m pytest \
+            --benchmark-only \
+            --benchmark-json=pr-benchmark.json \
+            --benchmark-columns=mean,stddev,iqr,ops,rounds \
+            ${{ env.BENCHMARK_PATH }}
+
+      - name: Store PR benchmark result
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr-benchmark
+          path: pr-benchmark.json
+
+      - name: Checkout base branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.ref }}
+
+      - name: Install dependencies for base branch
+        run: |
+          pip install -r requirements.txt
+          pip install -r requirements_dev.txt
+
+      - name: Run benchmarks on base branch
+        run: |
+          python3 -m pytest \
+            --benchmark-only \
+            --benchmark-json=base-benchmark.json \
+            --benchmark-columns=mean,stddev,iqr,ops,rounds \
+            ${{ env.BENCHMARK_PATH }}
+
+      - name: Store base benchmark result
+        uses: actions/upload-artifact@v4
+        with:
+          name: base-benchmark
+          path: base-benchmark.json
+
+      # Checkout PR branch again to ensure artifact downloads happen in correct context
+      - name: Checkout PR branch again
+        uses: actions/checkout@v4
+
+      - name: Download PR benchmark result
+        uses: actions/download-artifact@v4
+        with:
+          name: pr-benchmark
+
+      - name: Download base benchmark result
+        uses: actions/download-artifact@v4
+        with:
+          name: base-benchmark
+
+      - name: Compare benchmarks and comment
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+
+            // Read benchmark results
+            const prBenchmark = JSON.parse(fs.readFileSync('pr-benchmark.json', 'utf8'));
+            const baseBenchmark = JSON.parse(fs.readFileSync('base-benchmark.json', 'utf8'));
+
+            // Create a map of base benchmarks for easy lookup
+            const baseMap = {};
+            baseBenchmark.benchmarks.forEach(bench => {
+              baseMap[bench.name] = bench;
+            });
+
+            // Build comparison table
+            let tableRows = [];
+            let significantChanges = [];
+
+            prBenchmark.benchmarks.forEach(prBench => {
+              const baseBench = baseMap[prBench.name];
+              if (!baseBench) {
+                tableRows.push({
+                  name: prBench.name,
+                  status: '🆕 NEW',
+                  prMean: prBench.stats.mean,
+                  baseMean: '-',
+                  change: '-'
+                });
+                return;
+              }
+
+              const prMean = prBench.stats.mean;
+              const baseMean = baseBench.stats.mean;
+              const changePercent = ((prMean - baseMean) / baseMean) * 100;
+
+              let status = '✅';
+              let changeStr = changePercent.toFixed(2) + '%';
+
+              // Flag significant changes (> 10% regression)
+              if (changePercent > 10) {
+                status = '⚠️ SLOWER';
+                significantChanges.push(`${prBench.name}: ${changePercent.toFixed(2)}% slower`);
+              } else if (changePercent < -10) {
+                status = '🚀 FASTER';
+              }
+
+              tableRows.push({
+                name: prBench.name,
+                status: status,
+                prMean: prMean,
+                baseMean: baseMean,
+                change: changeStr
+              });
+            });
+
+            // Format table
+            let comment = '## 📊 Performance Benchmark Results\n\n';
+
+            if (significantChanges.length > 0) {
+              comment += '### ⚠️ Significant Performance Changes Detected\n\n';
+              significantChanges.forEach(change => {
+                comment += `- ${change}\n`;
+              });
+              comment += '\n';
+            }
+
+            comment += '### Detailed Comparison\n\n';
+            comment += '| Benchmark | Status | PR Mean (s) | Base Mean (s) | Change |\n';
+            comment += '|-----------|--------|-------------|---------------|--------|\n';
+
+            tableRows.forEach(row => {
+              const prMeanStr = typeof row.prMean === 'number' ? row.prMean.toExponential(4) : row.prMean;
+              const baseMeanStr = typeof row.baseMean === 'number' ? row.baseMean.toExponential(4) : row.baseMean;
+              comment += `| ${row.name} | ${row.status} | ${prMeanStr} | ${baseMeanStr} | ${row.change} |\n`;
+            });
+
+            comment += '\n---\n';
+            comment += `**Base Branch:** \`${context.payload.pull_request.base.ref}\` (${context.payload.pull_request.base.sha.substring(0, 7)})\n`;
+            comment += `**PR Branch:** \`${context.payload.pull_request.head.ref}\` (${context.payload.pull_request.head.sha.substring(0, 7)})\n`;
+            if (prBenchmark.machine_info) {
+              if (prBenchmark.machine_info.python_version) {
+                comment += `**Python Version:** ${prBenchmark.machine_info.python_version}\n`;
+              }
+              if (prBenchmark.machine_info.system && prBenchmark.machine_info.release) {
+                comment += `**Platform:** ${prBenchmark.machine_info.system} ${prBenchmark.machine_info.release}\n`;
+              }
+            }
+
+            // Post comment
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: comment
+            });