From 9d88e777d3bbb514216555388e40c0028fab020c Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Mon, 19 Oct 2026 15:57:07 -0500 Subject: [PATCH 1/5] ci(ai-evals): Run each label on several models in parallel Each label runs once per agent model the harness's plan-evals action lists, one job and artifact per model. A report job per label and side downloads every model's artifact and uploads them as one run with the harness's report-evals action, so evals.sentry.dev's PR comment, which shows only a dataset's newest run, has every model. --- .github/workflows/ai-evals.yml | 76 ++++++++++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index beabe876..290c21b6 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -3,7 +3,9 @@ # Label a PR `ai-evals:` to run a suite, or `ai-evals::` to run # only its tests with that pytest marker, e.g. `ai-evals:sentry-get-started:products`. # Each label runs against skills built at both the PR's merge-base and head, using the -# same eval harness commit. A new push reruns every eval label still on the PR. +# same eval harness commit, and on each agent model the harness's plan-evals action lists, +# one job per model. A report job per label and side uploads all the models' results as +# one run. A new push reruns every eval label still on the PR. # # sentry-for-ai-evals is internal, so the jobs check it out with the readonly bot's # token and use its actions from that path. Fork PRs get no secrets, so they're skipped. @@ -33,6 +35,8 @@ jobs: outputs: runs: ${{ steps.plan.outputs.runs }} count: ${{ steps.plan.outputs.count }} + models: ${{ steps.plan.outputs.models }} + model-count: ${{ steps.plan.outputs.model-count }} evals-sha: ${{ steps.evals.outputs.sha }} base-sha: ${{ steps.base.outputs.sha }} steps: @@ -55,14 +59,16 @@ jobs: - id: plan uses: ./sentry-for-ai-evals/.github/actions/plan-evals - # An older action ignores the revision and base-sha inputs, so it attributes both runs - # to head or pairs them by a commit the base side didn't run. - - name: Check baseline support + # An older harness has no model list or report action, so the matrix below would have + # no models and nothing would be uploaded. + - name: Check model support if: steps.plan.outputs.count != '0' shell: bash + env: + MODEL_COUNT: ${{ steps.plan.outputs.model-count }} run: | - if ! grep -q '^ base-sha:' sentry-for-ai-evals/.github/actions/run-evals/action.yml; then - echo "::error::Merge the base/head support in sentry-for-ai-evals before running this workflow" + if [[ -z "$MODEL_COUNT" || ! -f sentry-for-ai-evals/.github/actions/report-evals/action.yml ]]; then + echo "::error::Merge the several-models support in sentry-for-ai-evals before running this workflow" exit 1 fi @@ -87,10 +93,10 @@ jobs: eval: needs: plan if: needs.plan.outputs.count != '0' - name: ${{ matrix.run.label }} (${{ matrix.revision }}) - # Per PR, label and revision, so base and head don't cancel each other. + name: ${{ matrix.run.label }} (${{ matrix.revision }}, ${{ matrix.model.name }}) + # Per PR, label, revision and model, so the jobs don't cancel each other. concurrency: - group: ai-evals-${{ github.event.pull_request.number }}-${{ matrix.run.label }}-${{ matrix.revision }} + group: ai-evals-${{ github.event.pull_request.number }}-${{ matrix.run.label }}-${{ matrix.revision }}-${{ matrix.model.name }} cancel-in-progress: true runs-on: ubuntu-latest timeout-minutes: 60 @@ -101,6 +107,7 @@ jobs: matrix: run: ${{ fromJSON(needs.plan.outputs.runs) }} revision: [base, head] + model: ${{ fromJSON(needs.plan.outputs.models) }} steps: - name: Get sentry-for-ai-evals token id: token @@ -126,12 +133,63 @@ jobs: path: sentry-for-ai persist-credentials: false + # No evals key here: the report job uploads every model's results as one run. - uses: ./sentry-for-ai-evals/.github/actions/run-evals with: suite: ${{ matrix.run.suite }} marker: ${{ matrix.run.marker }} + model: ${{ matrix.model.id }} revision: ${{ matrix.revision }} base-sha: ${{ needs.plan.outputs.base-sha }} sentry-for-ai-path: sentry-for-ai + artifact-name: ai-evals-${{ matrix.run.suite }}-${{ matrix.run.marker || 'all' }}-${{ matrix.revision }}-${{ matrix.model.name }} openrouter-api-key: ${{ secrets.AI_EVALS_OPENROUTER_API_KEY }} + + report: + needs: [plan, eval] + # After every model's job, whether it passed, so a model's crash still fails here + # with its name rather than quietly leaving it out of the run. + if: ${{ always() && !cancelled() && needs.plan.outputs.count != '0' }} + name: ${{ matrix.run.label }} (${{ matrix.revision }}) report + concurrency: + group: ai-evals-${{ github.event.pull_request.number }}-${{ matrix.run.label }}-${{ matrix.revision }}-report + cancel-in-progress: true + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + run: ${{ fromJSON(needs.plan.outputs.runs) }} + revision: [base, head] + steps: + - name: Get sentry-for-ai-evals token + id: token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.SENTRY_READONLY_BOT_CLIENT_ID }} + private-key: ${{ secrets.SENTRY_READONLYBOT_PRIVATE_KEY }} + repositories: sentry-for-ai-evals + permission-contents: read + + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + repository: getsentry/sentry-for-ai-evals + ref: ${{ needs.plan.outputs.evals-sha }} + token: ${{ steps.token.outputs.token }} + path: sentry-for-ai-evals + persist-credentials: false + + # The skills sha is the tested side's own commit, as the eval jobs recorded it. + - uses: ./sentry-for-ai-evals/.github/actions/report-evals + with: + artifact-pattern: ai-evals-${{ matrix.run.suite }}-${{ matrix.run.marker || 'all' }}-${{ matrix.revision }}-* + expected-count: ${{ needs.plan.outputs.model-count }} + suite: ${{ matrix.run.suite }} + marker: ${{ matrix.run.marker }} + skills-sha: ${{ matrix.revision == 'base' && needs.plan.outputs.base-sha || github.event.pull_request.head.sha }} + revision: ${{ matrix.revision }} + base-sha: ${{ needs.plan.outputs.base-sha }} + artifact-name: ai-evals-upload-${{ matrix.run.suite }}-${{ matrix.run.marker || 'all' }}-${{ matrix.revision }} evals-api-key: ${{ secrets.SENTRY_EVALS_API_KEY }} From 14eafc56d26dd5cbe4cecd5ef801087aefd7794a Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Mon, 19 Oct 2026 16:07:39 -0500 Subject: [PATCH 2/5] Make the axis a target: an agent harness with a model Matches the eval repo's plan-evals output, a list of targets rather than of models, so a Codex target later is one more entry there. --- .github/workflows/ai-evals.yml | 42 ++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index 290c21b6..c41f005a 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -3,9 +3,10 @@ # Label a PR `ai-evals:` to run a suite, or `ai-evals::` to run # only its tests with that pytest marker, e.g. `ai-evals:sentry-get-started:products`. # Each label runs against skills built at both the PR's merge-base and head, using the -# same eval harness commit, and on each agent model the harness's plan-evals action lists, -# one job per model. A report job per label and side uploads all the models' results as -# one run. A new push reruns every eval label still on the PR. +# same eval harness commit, and on each target (an agent harness with a model) the eval +# repo's plan-evals action lists, one job per target. A report job per label and side +# uploads all the targets' results as one run. A new push reruns every eval label still +# on the PR. # # sentry-for-ai-evals is internal, so the jobs check it out with the readonly bot's # token and use its actions from that path. Fork PRs get no secrets, so they're skipped. @@ -35,8 +36,8 @@ jobs: outputs: runs: ${{ steps.plan.outputs.runs }} count: ${{ steps.plan.outputs.count }} - models: ${{ steps.plan.outputs.models }} - model-count: ${{ steps.plan.outputs.model-count }} + targets: ${{ steps.plan.outputs.targets }} + target-count: ${{ steps.plan.outputs.target-count }} evals-sha: ${{ steps.evals.outputs.sha }} base-sha: ${{ steps.base.outputs.sha }} steps: @@ -59,16 +60,16 @@ jobs: - id: plan uses: ./sentry-for-ai-evals/.github/actions/plan-evals - # An older harness has no model list or report action, so the matrix below would have - # no models and nothing would be uploaded. - - name: Check model support + # An older harness has no target list or report action, so the matrix below would + # have no targets and nothing would be uploaded. + - name: Check target support if: steps.plan.outputs.count != '0' shell: bash env: - MODEL_COUNT: ${{ steps.plan.outputs.model-count }} + TARGET_COUNT: ${{ steps.plan.outputs.target-count }} run: | - if [[ -z "$MODEL_COUNT" || ! -f sentry-for-ai-evals/.github/actions/report-evals/action.yml ]]; then - echo "::error::Merge the several-models support in sentry-for-ai-evals before running this workflow" + if [[ -z "$TARGET_COUNT" || ! -f sentry-for-ai-evals/.github/actions/report-evals/action.yml ]]; then + echo "::error::Merge the several-targets support in sentry-for-ai-evals before running this workflow" exit 1 fi @@ -93,10 +94,10 @@ jobs: eval: needs: plan if: needs.plan.outputs.count != '0' - name: ${{ matrix.run.label }} (${{ matrix.revision }}, ${{ matrix.model.name }}) - # Per PR, label, revision and model, so the jobs don't cancel each other. + name: ${{ matrix.run.label }} (${{ matrix.revision }}, ${{ matrix.target.name }}) + # Per PR, label, revision and target, so the jobs don't cancel each other. concurrency: - group: ai-evals-${{ github.event.pull_request.number }}-${{ matrix.run.label }}-${{ matrix.revision }}-${{ matrix.model.name }} + group: ai-evals-${{ github.event.pull_request.number }}-${{ matrix.run.label }}-${{ matrix.revision }}-${{ matrix.target.name }} cancel-in-progress: true runs-on: ubuntu-latest timeout-minutes: 60 @@ -107,7 +108,7 @@ jobs: matrix: run: ${{ fromJSON(needs.plan.outputs.runs) }} revision: [base, head] - model: ${{ fromJSON(needs.plan.outputs.models) }} + target: ${{ fromJSON(needs.plan.outputs.targets) }} steps: - name: Get sentry-for-ai-evals token id: token @@ -133,21 +134,22 @@ jobs: path: sentry-for-ai persist-credentials: false - # No evals key here: the report job uploads every model's results as one run. + # No evals key here: the report job uploads every target's results as one run. - uses: ./sentry-for-ai-evals/.github/actions/run-evals with: suite: ${{ matrix.run.suite }} marker: ${{ matrix.run.marker }} - model: ${{ matrix.model.id }} + agent: ${{ matrix.target.agent }} + model: ${{ matrix.target.model }} revision: ${{ matrix.revision }} base-sha: ${{ needs.plan.outputs.base-sha }} sentry-for-ai-path: sentry-for-ai - artifact-name: ai-evals-${{ matrix.run.suite }}-${{ matrix.run.marker || 'all' }}-${{ matrix.revision }}-${{ matrix.model.name }} + artifact-name: ai-evals-${{ matrix.run.suite }}-${{ matrix.run.marker || 'all' }}-${{ matrix.revision }}-${{ matrix.target.name }} openrouter-api-key: ${{ secrets.AI_EVALS_OPENROUTER_API_KEY }} report: needs: [plan, eval] - # After every model's job, whether it passed, so a model's crash still fails here + # After every target's job, whether it passed, so a target's crash still fails here # with its name rather than quietly leaving it out of the run. if: ${{ always() && !cancelled() && needs.plan.outputs.count != '0' }} name: ${{ matrix.run.label }} (${{ matrix.revision }}) report @@ -185,7 +187,7 @@ jobs: - uses: ./sentry-for-ai-evals/.github/actions/report-evals with: artifact-pattern: ai-evals-${{ matrix.run.suite }}-${{ matrix.run.marker || 'all' }}-${{ matrix.revision }}-* - expected-count: ${{ needs.plan.outputs.model-count }} + expected-count: ${{ needs.plan.outputs.target-count }} suite: ${{ matrix.run.suite }} marker: ${{ matrix.run.marker }} skills-sha: ${{ matrix.revision == 'base' && needs.plan.outputs.base-sha || github.event.pull_request.head.sha }} From d6aca25b6be75cb7da228bd5dd8d302e42ce3cbc Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Mon, 19 Oct 2026 16:17:56 -0500 Subject: [PATCH 3/5] Call the agent's driver the harness, not the agent --- .github/workflows/ai-evals.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index c41f005a..b81e5791 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -3,10 +3,10 @@ # Label a PR `ai-evals:` to run a suite, or `ai-evals::` to run # only its tests with that pytest marker, e.g. `ai-evals:sentry-get-started:products`. # Each label runs against skills built at both the PR's merge-base and head, using the -# same eval harness commit, and on each target (an agent harness with a model) the eval -# repo's plan-evals action lists, one job per target. A report job per label and side -# uploads all the targets' results as one run. A new push reruns every eval label still -# on the PR. +# same sentry-for-ai-evals commit, and on each target (a harness, Claude Code or Codex +# later, with a model) that repo's plan-evals action lists, one job per target. A report +# job per label and side uploads all the targets' results as one run. A new push reruns +# every eval label still on the PR. # # sentry-for-ai-evals is internal, so the jobs check it out with the readonly bot's # token and use its actions from that path. Fork PRs get no secrets, so they're skipped. @@ -60,8 +60,8 @@ jobs: - id: plan uses: ./sentry-for-ai-evals/.github/actions/plan-evals - # An older harness has no target list or report action, so the matrix below would - # have no targets and nothing would be uploaded. + # An older sentry-for-ai-evals has no target list or report action, so the matrix + # below would have no targets and nothing would be uploaded. - name: Check target support if: steps.plan.outputs.count != '0' shell: bash @@ -139,7 +139,7 @@ jobs: with: suite: ${{ matrix.run.suite }} marker: ${{ matrix.run.marker }} - agent: ${{ matrix.target.agent }} + harness: ${{ matrix.target.harness }} model: ${{ matrix.target.model }} revision: ${{ matrix.revision }} base-sha: ${{ needs.plan.outputs.base-sha }} From fe3dcdacc1698cb91c3a1ef269bc831b61ee578e Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Tue, 20 Oct 2026 12:40:45 -0500 Subject: [PATCH 4/5] ci(ai-evals): Create each run up front instead of a report job The report job gathered each target's results through artifacts, which this public repo can't keep private. The plan job now creates one run per label and side on evals.sentry.dev with sentry-for-ai-evals' create-runs action, and each target's job sends its results into it. --- .github/workflows/ai-evals.yml | 88 +++++++++++----------------------- 1 file changed, 28 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index b81e5791..3cee1275 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -4,15 +4,17 @@ # only its tests with that pytest marker, e.g. `ai-evals:sentry-get-started:products`. # Each label runs against skills built at both the PR's merge-base and head, using the # same sentry-for-ai-evals commit, and on each target (a harness, Claude Code or Codex -# later, with a model) that repo's plan-evals action lists, one job per target. A report -# job per label and side uploads all the targets' results as one run. A new push reruns -# every eval label still on the PR. +# later, with a model) that repo's plan-evals action lists, one job per target. The plan +# job creates one run per label and side on evals.sentry.dev, and each target's job +# sends its results into it, so one run holds every target. A new push reruns every +# eval label still on the PR. # # sentry-for-ai-evals is internal, so the jobs check it out with the readonly bot's # token and use its actions from that path. Fork PRs get no secrets, so they're skipped. # This repo is public, so the eval outputs (transcripts, reports, the agent's changes) are -# not kept as artifacts. The products tests' results go to evals.sentry.dev, which is -# where to look at a run; the logs show only pytest's progress and failures. +# not kept as artifacts, and nothing passes between the jobs but the run's id. The +# products tests' results go to evals.sentry.dev, which is where to look at a run; the +# logs show only pytest's progress and failures. name: AI Evals @@ -37,9 +39,9 @@ jobs: runs: ${{ steps.plan.outputs.runs }} count: ${{ steps.plan.outputs.count }} targets: ${{ steps.plan.outputs.targets }} - target-count: ${{ steps.plan.outputs.target-count }} evals-sha: ${{ steps.evals.outputs.sha }} base-sha: ${{ steps.base.outputs.sha }} + eval-runs: ${{ steps.create.outputs.runs }} steps: - name: Get sentry-for-ai-evals token id: token @@ -60,15 +62,15 @@ jobs: - id: plan uses: ./sentry-for-ai-evals/.github/actions/plan-evals - # An older sentry-for-ai-evals has no target list or report action, so the matrix - # below would have no targets and nothing would be uploaded. + # An older sentry-for-ai-evals has no target list or create-runs action, so the + # matrix below would have no targets and nothing would be uploaded. - name: Check target support if: steps.plan.outputs.count != '0' shell: bash env: - TARGET_COUNT: ${{ steps.plan.outputs.target-count }} + TARGETS: ${{ steps.plan.outputs.targets }} run: | - if [[ -z "$TARGET_COUNT" || ! -f sentry-for-ai-evals/.github/actions/report-evals/action.yml ]]; then + if [[ -z "$TARGETS" || ! -f sentry-for-ai-evals/.github/actions/create-runs/action.yml ]]; then echo "::error::Merge the several-targets support in sentry-for-ai-evals before running this workflow" exit 1 fi @@ -91,6 +93,18 @@ jobs: sha="$(gh api "repos/$GITHUB_REPOSITORY/compare/$BASE_SHA...$HEAD_SHA" --jq .merge_base_commit.sha)" echo "sha=$sha" >> "$GITHUB_OUTPUT" + # Each side's run records its own skills commit, the merge-base or the head. + - id: create + if: steps.plan.outputs.count != '0' + uses: ./sentry-for-ai-evals/.github/actions/create-runs + with: + runs: ${{ steps.plan.outputs.runs }} + targets: ${{ steps.plan.outputs.targets }} + skills-sha: ${{ github.event.pull_request.head.sha }} + base-skills-sha: ${{ steps.base.outputs.sha }} + base-sha: ${{ steps.base.outputs.sha }} + evals-api-key: ${{ secrets.SENTRY_EVALS_API_KEY }} + eval: needs: plan if: needs.plan.outputs.count != '0' @@ -134,7 +148,8 @@ jobs: path: sentry-for-ai persist-credentials: false - # No evals key here: the report job uploads every target's results as one run. + # Sends this target's results into the run the plan job created for the label and + # side. No artifact-name: this repo is public. - uses: ./sentry-for-ai-evals/.github/actions/run-evals with: suite: ${{ matrix.run.suite }} @@ -144,54 +159,7 @@ jobs: revision: ${{ matrix.revision }} base-sha: ${{ needs.plan.outputs.base-sha }} sentry-for-ai-path: sentry-for-ai - artifact-name: ai-evals-${{ matrix.run.suite }}-${{ matrix.run.marker || 'all' }}-${{ matrix.revision }}-${{ matrix.target.name }} + run-id: ${{ fromJSON(needs.plan.outputs.eval-runs)[format('{0}-{1}', matrix.run.label, matrix.revision)].id }} openrouter-api-key: ${{ secrets.AI_EVALS_OPENROUTER_API_KEY }} - - report: - needs: [plan, eval] - # After every target's job, whether it passed, so a target's crash still fails here - # with its name rather than quietly leaving it out of the run. - if: ${{ always() && !cancelled() && needs.plan.outputs.count != '0' }} - name: ${{ matrix.run.label }} (${{ matrix.revision }}) report - concurrency: - group: ai-evals-${{ github.event.pull_request.number }}-${{ matrix.run.label }}-${{ matrix.revision }}-report - cancel-in-progress: true - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - strategy: - fail-fast: false - matrix: - run: ${{ fromJSON(needs.plan.outputs.runs) }} - revision: [base, head] - steps: - - name: Get sentry-for-ai-evals token - id: token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - client-id: ${{ vars.SENTRY_READONLY_BOT_CLIENT_ID }} - private-key: ${{ secrets.SENTRY_READONLYBOT_PRIVATE_KEY }} - repositories: sentry-for-ai-evals - permission-contents: read - - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - repository: getsentry/sentry-for-ai-evals - ref: ${{ needs.plan.outputs.evals-sha }} - token: ${{ steps.token.outputs.token }} - path: sentry-for-ai-evals - persist-credentials: false - - # The skills sha is the tested side's own commit, as the eval jobs recorded it. - - uses: ./sentry-for-ai-evals/.github/actions/report-evals - with: - artifact-pattern: ai-evals-${{ matrix.run.suite }}-${{ matrix.run.marker || 'all' }}-${{ matrix.revision }}-* - expected-count: ${{ needs.plan.outputs.target-count }} - suite: ${{ matrix.run.suite }} - marker: ${{ matrix.run.marker }} - skills-sha: ${{ matrix.revision == 'base' && needs.plan.outputs.base-sha || github.event.pull_request.head.sha }} - revision: ${{ matrix.revision }} - base-sha: ${{ needs.plan.outputs.base-sha }} - artifact-name: ai-evals-upload-${{ matrix.run.suite }}-${{ matrix.run.marker || 'all' }}-${{ matrix.revision }} evals-api-key: ${{ secrets.SENTRY_EVALS_API_KEY }} + From 1763229fbe61b46be7586d0d8a7ccfc37e737348 Mon Sep 17 00:00:00 2001 From: Jay Goss Date: Tue, 20 Oct 2026 15:10:41 -0500 Subject: [PATCH 5/5] style: Drop the trailing blank line in ai-evals.yml --- .github/workflows/ai-evals.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ai-evals.yml b/.github/workflows/ai-evals.yml index 3cee1275..4545440e 100644 --- a/.github/workflows/ai-evals.yml +++ b/.github/workflows/ai-evals.yml @@ -162,4 +162,3 @@ jobs: run-id: ${{ fromJSON(needs.plan.outputs.eval-runs)[format('{0}-{1}', matrix.run.label, matrix.revision)].id }} openrouter-api-key: ${{ secrets.AI_EVALS_OPENROUTER_API_KEY }} evals-api-key: ${{ secrets.SENTRY_EVALS_API_KEY }} -