From 7783aad2bd4e9b589f653296226b66ef5b243f8f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 8 Jan 2026 06:03:51 +0000 Subject: [PATCH 1/5] Initial plan From 74842fe816a3c973ff135e328f14af6b513d9ce9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 8 Jan 2026 06:06:58 +0000 Subject: [PATCH 2/5] Add performance benchmark CI workflow for pull requests Co-authored-by: hsluoyz <3787410+hsluoyz@users.noreply.github.com> --- .github/workflows/benchmark.yml | 164 ++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..262c0dc --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,164 @@ +name: Performance Benchmark + +on: + pull_request: + branches: [master] + +permissions: + contents: read + pull-requests: write + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - name: Checkout PR branch + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install -r requirements_dev.txt + + - name: Run benchmarks on PR branch + run: | + python3 -m pytest \ + --benchmark-only \ + --benchmark-json=pr-benchmark.json \ + --benchmark-columns=mean,stddev,iqr,ops,rounds \ + tests/benchmarks/ + + - name: Store PR benchmark result + uses: actions/upload-artifact@v4 + with: + name: pr-benchmark + path: pr-benchmark.json + + - name: Checkout base branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.ref }} + + - name: Run benchmarks on base branch + run: | + python3 -m pytest \ + --benchmark-only \ + --benchmark-json=base-benchmark.json \ + --benchmark-columns=mean,stddev,iqr,ops,rounds \ + tests/benchmarks/ + + - name: Store base benchmark result + uses: actions/upload-artifact@v4 + with: + name: base-benchmark + path: base-benchmark.json + + - name: Checkout PR branch again + uses: actions/checkout@v4 + + - name: Download PR benchmark result + uses: actions/download-artifact@v4 + with: + name: pr-benchmark + + - name: Download base benchmark result + uses: actions/download-artifact@v4 + with: + name: base-benchmark + + - name: Compare benchmarks and comment + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + + // Read benchmark results + const prBenchmark = JSON.parse(fs.readFileSync('pr-benchmark.json', 'utf8')); + const baseBenchmark = JSON.parse(fs.readFileSync('base-benchmark.json', 'utf8')); + + // Create a map of base benchmarks for easy lookup + const baseMap = {}; + baseBenchmark.benchmarks.forEach(bench => { + baseMap[bench.name] = bench; + }); + + // Build comparison table + let tableRows = []; + let significantChanges = []; + + prBenchmark.benchmarks.forEach(prBench => { + const baseBench = baseMap[prBench.name]; + if (!baseBench) { + tableRows.push({ + name: prBench.name, + status: '🆕 NEW', + prMean: prBench.stats.mean, + baseMean: '-', + change: '-' + }); + return; + } + + const prMean = prBench.stats.mean; + const baseMean = baseBench.stats.mean; + const changePercent = ((prMean - baseMean) / baseMean) * 100; + + let status = '✅'; + let changeStr = changePercent.toFixed(2) + '%'; + + // Flag significant changes (> 10% regression) + if (changePercent > 10) { + status = '⚠️ SLOWER'; + significantChanges.push(`${prBench.name}: ${changePercent.toFixed(2)}% slower`); + } else if (changePercent < -10) { + status = '🚀 FASTER'; + } + + tableRows.push({ + name: prBench.name, + status: status, + prMean: prMean, + baseMean: baseMean, + change: changeStr + }); + }); + + // Format table + let comment = '## 📊 Performance Benchmark Results\n\n'; + + if (significantChanges.length > 0) { + comment += '### ⚠️ Significant Performance Changes Detected\n\n'; + significantChanges.forEach(change => { + comment += `- ${change}\n`; + }); + comment += '\n'; + } + + comment += '### Detailed Comparison\n\n'; + comment += '| Benchmark | Status | PR Mean (s) | Base Mean (s) | Change |\n'; + comment += '|-----------|--------|-------------|---------------|--------|\n'; + + tableRows.forEach(row => { + const prMeanStr = typeof row.prMean === 'number' ? row.prMean.toExponential(4) : row.prMean; + const baseMeanStr = typeof row.baseMean === 'number' ? row.baseMean.toExponential(4) : row.baseMean; + comment += `| ${row.name} | ${row.status} | ${prMeanStr} | ${baseMeanStr} | ${row.change} |\n`; + }); + + comment += '\n---\n'; + comment += `**Base Branch:** \`${context.payload.pull_request.base.ref}\` (${context.payload.pull_request.base.sha.substring(0, 7)})\n`; + comment += `**PR Branch:** \`${context.payload.pull_request.head.ref}\` (${context.payload.pull_request.head.sha.substring(0, 7)})\n`; + comment += `**Python Version:** ${prBenchmark.machine_info.python_version}\n`; + comment += `**Platform:** ${prBenchmark.machine_info.system} ${prBenchmark.machine_info.release}\n`; + + // Post comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: comment + }); From e262e6414256b83d3dab89e1a25e50029f17bbd2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 8 Jan 2026 06:08:29 +0000 Subject: [PATCH 3/5] Add dependency reinstall for base branch benchmarks Co-authored-by: hsluoyz <3787410+hsluoyz@users.noreply.github.com> --- .github/workflows/benchmark.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 262c0dc..10cac7f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -44,6 +44,11 @@ jobs: with: ref: ${{ github.event.pull_request.base.ref }} + - name: Install dependencies for base branch + run: | + pip install -r requirements.txt + pip install -r requirements_dev.txt + - name: Run benchmarks on base branch run: | python3 -m pytest \ From e3de4ae58f2cc43ed7587bf1f669b81b734422cc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 8 Jan 2026 06:09:08 +0000 Subject: [PATCH 4/5] Add documentation comments to benchmark workflow Co-authored-by: hsluoyz <3787410+hsluoyz@users.noreply.github.com> --- .github/workflows/benchmark.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 10cac7f..aa62394 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,3 +1,18 @@ +# Performance Benchmark CI Workflow +# +# This workflow automatically runs performance benchmarks on pull requests +# and posts a comparison comment showing speed differences between the PR +# and the base branch (master). +# +# The workflow: +# 1. Runs benchmarks on the PR branch +# 2. Runs benchmarks on the base branch (master) +# 3. Compares the results and identifies performance regressions/improvements +# 4. Posts a detailed comparison table as a PR comment +# +# Performance changes > 10% are flagged as significant to help maintainers +# catch performance regressions before merging. + name: Performance Benchmark on: From ec7ba818341d328351c84d090ed5198a161c0480 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 8 Jan 2026 06:10:54 +0000 Subject: [PATCH 5/5] Address code review feedback - add env var, comments, and null checks Co-authored-by: hsluoyz <3787410+hsluoyz@users.noreply.github.com> --- .github/workflows/benchmark.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index aa62394..8bf8bfb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -26,6 +26,8 @@ permissions: jobs: benchmark: runs-on: ubuntu-latest + env: + BENCHMARK_PATH: tests/benchmarks/ steps: - name: Checkout PR branch uses: actions/checkout@v4 @@ -46,7 +48,7 @@ jobs: --benchmark-only \ --benchmark-json=pr-benchmark.json \ --benchmark-columns=mean,stddev,iqr,ops,rounds \ - tests/benchmarks/ + ${{ env.BENCHMARK_PATH }} - name: Store PR benchmark result uses: actions/upload-artifact@v4 @@ -70,7 +72,7 @@ jobs: --benchmark-only \ --benchmark-json=base-benchmark.json \ --benchmark-columns=mean,stddev,iqr,ops,rounds \ - tests/benchmarks/ + ${{ env.BENCHMARK_PATH }} - name: Store base benchmark result uses: actions/upload-artifact@v4 @@ -78,6 +80,7 @@ jobs: name: base-benchmark path: base-benchmark.json + # Checkout PR branch again to ensure artifact downloads happen in correct context - name: Checkout PR branch again uses: actions/checkout@v4 @@ -172,8 +175,14 @@ jobs: comment += '\n---\n'; comment += `**Base Branch:** \`${context.payload.pull_request.base.ref}\` (${context.payload.pull_request.base.sha.substring(0, 7)})\n`; comment += `**PR Branch:** \`${context.payload.pull_request.head.ref}\` (${context.payload.pull_request.head.sha.substring(0, 7)})\n`; - comment += `**Python Version:** ${prBenchmark.machine_info.python_version}\n`; - comment += `**Platform:** ${prBenchmark.machine_info.system} ${prBenchmark.machine_info.release}\n`; + if (prBenchmark.machine_info) { + if (prBenchmark.machine_info.python_version) { + comment += `**Python Version:** ${prBenchmark.machine_info.python_version}\n`; + } + if (prBenchmark.machine_info.system && prBenchmark.machine_info.release) { + comment += `**Platform:** ${prBenchmark.machine_info.system} ${prBenchmark.machine_info.release}\n`; + } + } // Post comment await github.rest.issues.createComment({