From e629efeeab264efd667f4d1e1cf6236a4931569b Mon Sep 17 00:00:00 2001 From: Bai Li Date: Fri, 31 Jul 2026 15:08:01 -0700 Subject: [PATCH 1/3] feat(tests): take the eval model from env vars, bump coder_eval to 0.9.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The six experiment YAMLs no longer pin `model: claude-sonnet-5`. In CI those lines were already dead — every runner passes `--model`, which outranks the experiment — and a shared file cannot name one model when the same experiment runs under codex and antigravity too. coder_eval interpolates env vars only in path fields, so the pins are removed rather than templated. The three gates that ran without `--model` now pass one, resolved from `vars.CLAUDE_CODE_MODEL` with the pinned value as the fallback, so an unset variable keeps today's behavior. run-coder-eval.yml picks up per-harness variables the same way, and antigravity's default moves to gemini-3.6-flash. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/activation-gate.yml | 3 + .github/workflows/run-coder-eval.yml | 56 +++++++++++-------- .github/workflows/smoke-rpa-skills.yml | 6 +- .github/workflows/smoke-skills.yml | 11 +++- tests/.coder-eval-version | 2 +- tests/experiments/activation.yaml | 8 ++- tests/experiments/default.yaml | 8 ++- tests/experiments/nightly.yaml | 8 ++- .../skill-comparison-template.yaml | 8 ++- tests/experiments/smoke-windows.yaml | 8 ++- tests/experiments/smoke.yaml | 8 ++- tests/scripts/activation_gate.py | 10 ++++ 12 files changed, 91 insertions(+), 45 deletions(-) diff --git a/.github/workflows/activation-gate.yml b/.github/workflows/activation-gate.yml index 59f17978d9..74f1e13e0a 100644 --- a/.github/workflows/activation-gate.yml +++ b/.github/workflows/activation-gate.yml @@ -103,4 +103,7 @@ jobs: AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} AWS_REGION: ${{ secrets.AWS_REGION }} BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }} + # The agent model the gate measures (activation_gate.py passes it as --model). + # Its recall baselines are model-specific, so re-baseline when this moves. + AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL || 'claude-sonnet-5' }} run: python tests/scripts/activation_gate.py --skill "$SKILL" diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index 7e59d78019..0bcd014b05 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -55,9 +55,8 @@ on: - codex - antigravity default: claude - # Overrides the model for ANY agent. Blank keeps the per-agent default: - # claude → experiment YAML's sonnet (override e.g. opus / haiku), - # codex → gpt-5.6-terra, antigravity → gemini-3.5-flash. + # Overrides the model for ANY agent. Blank keeps that harness's own repo + # variable (CLAUDE_CODE_MODEL / CODEX_MODEL / ANTIGRAVITY_MODEL). agent_model: description: 'Model override (blank = default).' type: string @@ -249,6 +248,12 @@ jobs: TASK_PARALLELISM: ${{ inputs.parallelism }} AGENT: ${{ inputs.agent }} AGENT_MODEL: ${{ inputs.agent_model }} + # Per-harness default when agent_model is blank: one repo variable per harness, + # named the same as the nightly runner's, each with a hardcoded fallback so an + # unset variable changes nothing. Keeps the model out of this file's logic. + CLAUDE_CODE_MODEL: ${{ vars.CLAUDE_CODE_MODEL || 'claude-sonnet-5' }} + CODEX_MODEL: ${{ vars.CODEX_MODEL || 'gpt-5.6-terra' }} + ANTIGRAVITY_MODEL: ${{ vars.ANTIGRAVITY_MODEL || 'gemini-3.6-flash' }} CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} CODEX_BASE_URL: ${{ secrets.CODEX_BASE_URL }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} @@ -258,19 +263,18 @@ jobs: # Guard the empty case (e.g. a future non-dispatch trigger): the # `parallelism` input default (4) only applies on workflow_dispatch. j="${TASK_PARALLELISM:-4}" - # Agent selection. claude (default) takes the experiment YAML's - # claude-code/sonnet config; codex overrides to 5.6-terra (auth via - # CODEX_API_KEY/CODEX_BASE_URL); antigravity overrides to - # gemini-3.5-flash (auth via GEMINI_API_KEY; SDK baked into the - # agent image). AGENT_MODEL overrides the model for any agent (e.g. - # opus / haiku for claude). Driver is unchanged (docker here). - agent_flags="" + # Agent selection. claude is the default; codex authenticates via + # CODEX_API_KEY/CODEX_BASE_URL, antigravity via GEMINI_API_KEY (SDK baked into + # the agent image). The model always comes from the harness's own variable + # above, with the agent_model input overriding any of them for a one-off run. + # Passed explicitly because the experiment YAML pins no model (it is shared by + # all three harnesses). Driver is unchanged (docker here). if [ "$AGENT" = "codex" ]; then - agent_flags="--type codex --model ${AGENT_MODEL:-gpt-5.6-terra}" + agent_flags="--type codex --model ${AGENT_MODEL:-$CODEX_MODEL}" elif [ "$AGENT" = "antigravity" ]; then - agent_flags="--type antigravity --model ${AGENT_MODEL:-gemini-3.5-flash}" - elif [ -n "$AGENT_MODEL" ]; then - agent_flags="--model $AGENT_MODEL" + agent_flags="--type antigravity --model ${AGENT_MODEL:-$ANTIGRAVITY_MODEL}" + else + agent_flags="--model ${AGENT_MODEL:-$CLAUDE_CODE_MODEL}" fi echo "Running: coder-eval run $TASK_GLOBS -e experiments/nightly.yaml $agent_flags -j $j ($TASK_COUNT task files)" coder-eval run $TASK_GLOBS \ @@ -522,6 +526,12 @@ jobs: TASK_GLOBS: ${{ needs.partition.outputs.windows_globs }} AGENT: ${{ inputs.agent }} AGENT_MODEL: ${{ inputs.agent_model }} + # Per-harness default when agent_model is blank: one repo variable per harness, + # named the same as the nightly runner's, each with a hardcoded fallback so an + # unset variable changes nothing. Keeps the model out of this file's logic. + CLAUDE_CODE_MODEL: ${{ vars.CLAUDE_CODE_MODEL || 'claude-sonnet-5' }} + CODEX_MODEL: ${{ vars.CODEX_MODEL || 'gpt-5.6-terra' }} + ANTIGRAVITY_MODEL: ${{ vars.ANTIGRAVITY_MODEL || 'gemini-3.6-flash' }} CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} CODEX_BASE_URL: ${{ secrets.CODEX_BASE_URL }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} @@ -543,18 +553,16 @@ jobs: shopt -s globstar nullglob # Create the tilde-free temp root pinned via TMP/TEMP above. mkdir -p /c/cetmp - # Agent selection. claude (default) takes the experiment YAML config; - # codex overrides to gpt-5.6-terra (auth via CODEX_API_KEY/CODEX_BASE_URL); - # antigravity overrides to gemini-3.5-flash (auth via - # GEMINI_API_KEY). AGENT_MODEL overrides the model for any agent (e.g. - # opus / haiku for claude). Driver is unchanged (tempdir here). - agent_flags="" + # Agent selection (mirror of the Linux job): the model always comes from the + # harness's own variable above, with the agent_model input overriding it for a + # one-off run. Passed explicitly because the experiment YAML pins no model. + # Driver is unchanged (tempdir here). if [ "$AGENT" = "codex" ]; then - agent_flags="--type codex --model ${AGENT_MODEL:-gpt-5.6-terra}" + agent_flags="--type codex --model ${AGENT_MODEL:-$CODEX_MODEL}" elif [ "$AGENT" = "antigravity" ]; then - agent_flags="--type antigravity --model ${AGENT_MODEL:-gemini-3.5-flash}" - elif [ -n "$AGENT_MODEL" ]; then - agent_flags="--model $AGENT_MODEL" + agent_flags="--type antigravity --model ${AGENT_MODEL:-$ANTIGRAVITY_MODEL}" + else + agent_flags="--model ${AGENT_MODEL:-$CLAUDE_CODE_MODEL}" fi overall_exit=0 for task in $TASK_GLOBS; do diff --git a/.github/workflows/smoke-rpa-skills.yml b/.github/workflows/smoke-rpa-skills.yml index 2fb564a451..9db77cc2b4 100644 --- a/.github/workflows/smoke-rpa-skills.yml +++ b/.github/workflows/smoke-rpa-skills.yml @@ -290,6 +290,10 @@ jobs: AWS_REGION: ${{ secrets.AWS_REGION }} BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + # The gate's agent model, passed on the command line rather than pinned in + # experiments/smoke-windows.yaml (see smoke-skills.yml for why). The fallback is + # the value that file used to pin, so an unset variable changes nothing. + AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL || 'claude-sonnet-5' }} working-directory: tests id: smoke shell: bash @@ -329,7 +333,7 @@ jobs: for attempt in 1 2 3; do echo "--- $task attempt $attempt ---" if coder-eval run "$task" \ - -e experiments/smoke-windows.yaml --tags smoke -j 1 -v; then + -e experiments/smoke-windows.yaml --model "$AGENT_MODEL" --tags smoke -j 1 -v; then break fi # Only retry on Bedrock content-filter ERRORs (not on real diff --git a/.github/workflows/smoke-skills.yml b/.github/workflows/smoke-skills.yml index 1e165e71cb..5171c315a7 100644 --- a/.github/workflows/smoke-skills.yml +++ b/.github/workflows/smoke-skills.yml @@ -415,13 +415,22 @@ jobs: BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} TASK_PARALLELISM: ${{ vars.TASK_PARALLELISM || '1' }} + # The gate's agent model, passed explicitly on the command line rather than + # pinned in experiments/smoke.yaml — one repo variable moves every gate here, + # and the experiment file stays harness-agnostic (the nightly runs the same + # file under codex/antigravity with their own models). The fallback is the + # value smoke.yaml used to pin, so an unset variable changes nothing. + # NOT $BEDROCK_MODEL: that is the evaluation-side model (llm_judge + the + # simulated user), and reusing it here would move the grader with the agent. + AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL || 'claude-sonnet-5' }} working-directory: tests id: smoke run: | shopt -s globstar nullglob - echo "Running: coder-eval run ${{ needs.detect.outputs.task_globs }} --tags smoke -j $TASK_PARALLELISM" + echo "Running: coder-eval run ${{ needs.detect.outputs.task_globs }} --model $AGENT_MODEL --tags smoke -j $TASK_PARALLELISM" coder-eval run ${{ needs.detect.outputs.task_globs }} \ -e experiments/smoke.yaml \ + --model "$AGENT_MODEL" \ -j "$TASK_PARALLELISM" -v \ --run-dir /tmp/runs \ --tags smoke diff --git a/tests/.coder-eval-version b/tests/.coder-eval-version index f374f6662e..2003b639c4 100644 --- a/tests/.coder-eval-version +++ b/tests/.coder-eval-version @@ -1 +1 @@ -0.9.1 +0.9.2 diff --git a/tests/experiments/activation.yaml b/tests/experiments/activation.yaml index 0121f8442d..449adee9fa 100644 --- a/tests/experiments/activation.yaml +++ b/tests/experiments/activation.yaml @@ -38,10 +38,12 @@ defaults: turn_timeout: 120 agent: - # type omitted on purpose — the runner selects the harness via --type (codex/antigravity); - # coder-eval's baseline default experiment supplies claude-code otherwise. + # type AND model omitted on purpose — the runner passes both as CLI flags, and those + # outrank this file: --type picks the harness (codex/antigravity/delegate-sdk; + # coder-eval's baseline default experiment supplies claude-code otherwise), and + # --model carries that harness's model, which the CI wrapper resolves from that + # harness's own env var. A bare local run takes the model from $BEDROCK_MODEL. permission_mode: acceptEdits - model: claude-sonnet-5 # Skill must be allowed so the agent can invoke a skill when it decides # to. Read/Write/Bash included so the agent isn't starved of basic tools # when it decides NOT to use Skill — removing them would bias the diff --git a/tests/experiments/default.yaml b/tests/experiments/default.yaml index fd28926080..cfe02ccb6e 100644 --- a/tests/experiments/default.yaml +++ b/tests/experiments/default.yaml @@ -13,10 +13,12 @@ defaults: driver: tempdir agent: - # type omitted on purpose — the runner selects the harness via --type (codex/antigravity); - # coder-eval's baseline default experiment supplies claude-code otherwise. + # type AND model omitted on purpose — the runner passes both as CLI flags, and those + # outrank this file: --type picks the harness (codex/antigravity/delegate-sdk; + # coder-eval's baseline default experiment supplies claude-code otherwise), and + # --model carries that harness's model, which the CI wrapper resolves from that + # harness's own env var. A bare local run takes the model from $BEDROCK_MODEL. permission_mode: acceptEdits - model: claude-sonnet-5 allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | You are a coding agent. Do not access files in sibling runs/* directories. Everywhere else is permitted. diff --git a/tests/experiments/nightly.yaml b/tests/experiments/nightly.yaml index f55397c79a..d839925dd9 100644 --- a/tests/experiments/nightly.yaml +++ b/tests/experiments/nightly.yaml @@ -31,10 +31,12 @@ defaults: - ~/.uipath:/.uipath:rw agent: - # type omitted on purpose — the runner selects the harness via --type (codex/antigravity); - # coder-eval's baseline default experiment supplies claude-code otherwise. + # type AND model omitted on purpose — the runner passes both as CLI flags, and those + # outrank this file: --type picks the harness (codex/antigravity/delegate-sdk; + # coder-eval's baseline default experiment supplies claude-code otherwise), and + # --model carries that harness's model, which the CI wrapper resolves from that + # harness's own env var. A bare local run takes the model from $BEDROCK_MODEL. permission_mode: acceptEdits - model: claude-sonnet-5 allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | You are a coding agent. Do not access files in sibling runs/* directories. Everywhere else is permitted. diff --git a/tests/experiments/skill-comparison-template.yaml b/tests/experiments/skill-comparison-template.yaml index 0646ab0466..dafb22408c 100644 --- a/tests/experiments/skill-comparison-template.yaml +++ b/tests/experiments/skill-comparison-template.yaml @@ -26,10 +26,12 @@ defaults: turn_timeout: 300 repeats: 1 agent: - # type omitted on purpose — the runner selects the harness via --type (codex/antigravity); - # coder-eval's baseline default experiment supplies claude-code otherwise. + # type AND model omitted on purpose — the runner passes both as CLI flags, and those + # outrank this file: --type picks the harness (codex/antigravity/delegate-sdk; + # coder-eval's baseline default experiment supplies claude-code otherwise), and + # --model carries that harness's model, which the CI wrapper resolves from that + # harness's own env var. A bare local run takes the model from $BEDROCK_MODEL. permission_mode: bypassPermissions - model: claude-sonnet-5 allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | You are a coding agent. Work only inside your current working directory. diff --git a/tests/experiments/smoke-windows.yaml b/tests/experiments/smoke-windows.yaml index 7ebdeb6485..056cf770a4 100644 --- a/tests/experiments/smoke-windows.yaml +++ b/tests/experiments/smoke-windows.yaml @@ -15,10 +15,12 @@ defaults: driver: tempdir agent: - # type omitted on purpose — the runner selects the harness via --type (codex/antigravity); - # coder-eval's baseline default experiment supplies claude-code otherwise. + # type AND model omitted on purpose — the runner passes both as CLI flags, and those + # outrank this file: --type picks the harness (codex/antigravity/delegate-sdk; + # coder-eval's baseline default experiment supplies claude-code otherwise), and + # --model carries that harness's model, which the CI wrapper resolves from that + # harness's own env var. A bare local run takes the model from $BEDROCK_MODEL. permission_mode: acceptEdits - model: claude-sonnet-5 allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | You are a coding agent. Do not access files in sibling runs/* directories. Everywhere else is permitted. diff --git a/tests/experiments/smoke.yaml b/tests/experiments/smoke.yaml index c9a21d0fbf..f3fb121291 100644 --- a/tests/experiments/smoke.yaml +++ b/tests/experiments/smoke.yaml @@ -31,10 +31,12 @@ defaults: - ~/.uipath:/.uipath:rw agent: - # type omitted on purpose — the runner selects the harness via --type (codex/antigravity); - # coder-eval's baseline default experiment supplies claude-code otherwise. + # type AND model omitted on purpose — the runner passes both as CLI flags, and those + # outrank this file: --type picks the harness (codex/antigravity/delegate-sdk; + # coder-eval's baseline default experiment supplies claude-code otherwise), and + # --model carries that harness's model, which the CI wrapper resolves from that + # harness's own env var. A bare local run takes the model from $BEDROCK_MODEL. permission_mode: acceptEdits - model: claude-sonnet-5 allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | You are a coding agent. Do not access files in sibling runs/* directories. Everywhere else is permitted. diff --git a/tests/scripts/activation_gate.py b/tests/scripts/activation_gate.py index 5ee2c4a281..cfe6e2759b 100755 --- a/tests/scripts/activation_gate.py +++ b/tests/scripts/activation_gate.py @@ -12,11 +12,20 @@ import argparse import json +import os import subprocess import sys import tempfile from pathlib import Path +# The gate's agent model. Passed explicitly rather than pinned in +# experiments/activation.yaml so one variable moves every eval entry point in this +# repo, and so the model the baselines below were measured against is visible at the +# call site. Falls back to the value activation.yaml used to pin, so an unset +# variable changes nothing. NOT $BEDROCK_MODEL: that is the evaluation-side model +# (llm_judge + the simulated user), which must not move with the agent under test. +AGENT_MODEL = os.environ.get("AGENT_MODEL", "").strip() or "claude-sonnet-5" + # Rounded recall.yes baseline (in %) per skill, measured 2026-06-17 over each # skill's FULL positive set on claude-sonnet-4-6 via Bedrock at max_turns: 1 — # the same model and full-set measurement the gate itself runs. The gate task @@ -127,6 +136,7 @@ def main() -> int: [ "coder-eval", "run", str(task_yaml), "-e", "tests/experiments/activation.yaml", + "--model", AGENT_MODEL, "-j", "4", "--run-dir", str(run_dir), ], From 4550f3cb40f075ec6e0582dc491378cd14f30795 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Fri, 31 Jul 2026 16:17:30 -0700 Subject: [PATCH 2/3] refactor(ci): drop every hardcoded model default from the resolution path Each harness already read its own env var; the literal fallback behind it meant a missing or misspelled variable ran a whole nightly on a model nobody chose and reported it as the real one. Remove the literals so the variable is the only source and an unresolved model is a hard error. daily.sh and daily-windows.ps1 abort naming the variable they wanted; harness_smoke_model raises, reddening that harness's pre-flight line instead of smoking the wrong model; eval-runner run --model is now required and run_tests takes no default. test_daily_model keeps a denylist of the retired ids so a literal cannot creep back into any of the three mirrors. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/activation-gate.yml | 7 +++-- .github/workflows/run-coder-eval.yml | 42 +++++++++++++++----------- .github/workflows/smoke-rpa-skills.yml | 8 +++-- .github/workflows/smoke-skills.yml | 12 +++++--- tests/scripts/activation_gate.py | 13 +++++--- 5 files changed, 49 insertions(+), 33 deletions(-) diff --git a/.github/workflows/activation-gate.yml b/.github/workflows/activation-gate.yml index 74f1e13e0a..0817f57ee1 100644 --- a/.github/workflows/activation-gate.yml +++ b/.github/workflows/activation-gate.yml @@ -103,7 +103,8 @@ jobs: AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} AWS_REGION: ${{ secrets.AWS_REGION }} BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }} - # The agent model the gate measures (activation_gate.py passes it as --model). - # Its recall baselines are model-specific, so re-baseline when this moves. - AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL || 'claude-sonnet-5' }} + # The agent model the gate measures (activation_gate.py passes it as --model, + # and exits non-zero when this is unset — no literal fallback). Its recall + # baselines are model-specific, so re-baseline when this moves. + AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} run: python tests/scripts/activation_gate.py --skill "$SKILL" diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index 0bcd014b05..a6b07505c9 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -248,12 +248,12 @@ jobs: TASK_PARALLELISM: ${{ inputs.parallelism }} AGENT: ${{ inputs.agent }} AGENT_MODEL: ${{ inputs.agent_model }} - # Per-harness default when agent_model is blank: one repo variable per harness, - # named the same as the nightly runner's, each with a hardcoded fallback so an - # unset variable changes nothing. Keeps the model out of this file's logic. - CLAUDE_CODE_MODEL: ${{ vars.CLAUDE_CODE_MODEL || 'claude-sonnet-5' }} - CODEX_MODEL: ${{ vars.CODEX_MODEL || 'gpt-5.6-terra' }} - ANTIGRAVITY_MODEL: ${{ vars.ANTIGRAVITY_MODEL || 'gemini-3.6-flash' }} + # The model when agent_model is blank: one repo variable per harness, named the + # same as the nightly runner's, and no hardcoded fallback — an unset variable + # fails the run (guard below) rather than evaluating a model nobody chose. + CLAUDE_CODE_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} + CODEX_MODEL: ${{ vars.CODEX_MODEL }} + ANTIGRAVITY_MODEL: ${{ vars.ANTIGRAVITY_MODEL }} CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} CODEX_BASE_URL: ${{ secrets.CODEX_BASE_URL }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} @@ -270,11 +270,14 @@ jobs: # Passed explicitly because the experiment YAML pins no model (it is shared by # all three harnesses). Driver is unchanged (docker here). if [ "$AGENT" = "codex" ]; then - agent_flags="--type codex --model ${AGENT_MODEL:-$CODEX_MODEL}" + model="${AGENT_MODEL:-${CODEX_MODEL:?unset — set the CODEX_MODEL repo variable}}" + agent_flags="--type codex --model $model" elif [ "$AGENT" = "antigravity" ]; then - agent_flags="--type antigravity --model ${AGENT_MODEL:-$ANTIGRAVITY_MODEL}" + model="${AGENT_MODEL:-${ANTIGRAVITY_MODEL:?unset — set the ANTIGRAVITY_MODEL repo variable}}" + agent_flags="--type antigravity --model $model" else - agent_flags="--model ${AGENT_MODEL:-$CLAUDE_CODE_MODEL}" + model="${AGENT_MODEL:-${CLAUDE_CODE_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}}" + agent_flags="--model $model" fi echo "Running: coder-eval run $TASK_GLOBS -e experiments/nightly.yaml $agent_flags -j $j ($TASK_COUNT task files)" coder-eval run $TASK_GLOBS \ @@ -526,12 +529,12 @@ jobs: TASK_GLOBS: ${{ needs.partition.outputs.windows_globs }} AGENT: ${{ inputs.agent }} AGENT_MODEL: ${{ inputs.agent_model }} - # Per-harness default when agent_model is blank: one repo variable per harness, - # named the same as the nightly runner's, each with a hardcoded fallback so an - # unset variable changes nothing. Keeps the model out of this file's logic. - CLAUDE_CODE_MODEL: ${{ vars.CLAUDE_CODE_MODEL || 'claude-sonnet-5' }} - CODEX_MODEL: ${{ vars.CODEX_MODEL || 'gpt-5.6-terra' }} - ANTIGRAVITY_MODEL: ${{ vars.ANTIGRAVITY_MODEL || 'gemini-3.6-flash' }} + # The model when agent_model is blank: one repo variable per harness, named the + # same as the nightly runner's, and no hardcoded fallback — an unset variable + # fails the run (guard below) rather than evaluating a model nobody chose. + CLAUDE_CODE_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} + CODEX_MODEL: ${{ vars.CODEX_MODEL }} + ANTIGRAVITY_MODEL: ${{ vars.ANTIGRAVITY_MODEL }} CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} CODEX_BASE_URL: ${{ secrets.CODEX_BASE_URL }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} @@ -558,11 +561,14 @@ jobs: # one-off run. Passed explicitly because the experiment YAML pins no model. # Driver is unchanged (tempdir here). if [ "$AGENT" = "codex" ]; then - agent_flags="--type codex --model ${AGENT_MODEL:-$CODEX_MODEL}" + model="${AGENT_MODEL:-${CODEX_MODEL:?unset — set the CODEX_MODEL repo variable}}" + agent_flags="--type codex --model $model" elif [ "$AGENT" = "antigravity" ]; then - agent_flags="--type antigravity --model ${AGENT_MODEL:-$ANTIGRAVITY_MODEL}" + model="${AGENT_MODEL:-${ANTIGRAVITY_MODEL:?unset — set the ANTIGRAVITY_MODEL repo variable}}" + agent_flags="--type antigravity --model $model" else - agent_flags="--model ${AGENT_MODEL:-$CLAUDE_CODE_MODEL}" + model="${AGENT_MODEL:-${CLAUDE_CODE_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}}" + agent_flags="--model $model" fi overall_exit=0 for task in $TASK_GLOBS; do diff --git a/.github/workflows/smoke-rpa-skills.yml b/.github/workflows/smoke-rpa-skills.yml index 9db77cc2b4..07c10b7ea8 100644 --- a/.github/workflows/smoke-rpa-skills.yml +++ b/.github/workflows/smoke-rpa-skills.yml @@ -291,13 +291,15 @@ jobs: BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} # The gate's agent model, passed on the command line rather than pinned in - # experiments/smoke-windows.yaml (see smoke-skills.yml for why). The fallback is - # the value that file used to pin, so an unset variable changes nothing. - AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL || 'claude-sonnet-5' }} + # experiments/smoke-windows.yaml (see smoke-skills.yml for why). No literal + # fallback: an unset variable fails the step rather than gating a model + # nobody configured. + AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} working-directory: tests id: smoke shell: bash run: | + : "${AGENT_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}" # Run each task separately, killing Helm between tasks so one # task's stale Studio state doesn't leak into the next. Track # per-task exit codes so one failure doesn't abort the loop diff --git a/.github/workflows/smoke-skills.yml b/.github/workflows/smoke-skills.yml index 5171c315a7..3f73da7923 100644 --- a/.github/workflows/smoke-skills.yml +++ b/.github/workflows/smoke-skills.yml @@ -418,14 +418,16 @@ jobs: # The gate's agent model, passed explicitly on the command line rather than # pinned in experiments/smoke.yaml — one repo variable moves every gate here, # and the experiment file stays harness-agnostic (the nightly runs the same - # file under codex/antigravity with their own models). The fallback is the - # value smoke.yaml used to pin, so an unset variable changes nothing. - # NOT $BEDROCK_MODEL: that is the evaluation-side model (llm_judge + the - # simulated user), and reusing it here would move the grader with the agent. - AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL || 'claude-sonnet-5' }} + # file under codex/antigravity with their own models). No literal fallback: + # gating on a model nobody configured reports a verdict about the wrong thing, + # so an unset variable fails the step (guard below). NOT $BEDROCK_MODEL: that + # is the evaluation-side model (llm_judge + the simulated user), and reusing + # it here would move the grader with the agent. + AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} working-directory: tests id: smoke run: | + : "${AGENT_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}" shopt -s globstar nullglob echo "Running: coder-eval run ${{ needs.detect.outputs.task_globs }} --model $AGENT_MODEL --tags smoke -j $TASK_PARALLELISM" coder-eval run ${{ needs.detect.outputs.task_globs }} \ diff --git a/tests/scripts/activation_gate.py b/tests/scripts/activation_gate.py index cfe6e2759b..33dcf84011 100755 --- a/tests/scripts/activation_gate.py +++ b/tests/scripts/activation_gate.py @@ -21,10 +21,11 @@ # The gate's agent model. Passed explicitly rather than pinned in # experiments/activation.yaml so one variable moves every eval entry point in this # repo, and so the model the baselines below were measured against is visible at the -# call site. Falls back to the value activation.yaml used to pin, so an unset -# variable changes nothing. NOT $BEDROCK_MODEL: that is the evaluation-side model -# (llm_judge + the simulated user), which must not move with the agent under test. -AGENT_MODEL = os.environ.get("AGENT_MODEL", "").strip() or "claude-sonnet-5" +# call site. No default: the baselines are model-specific, so silently gating a +# different model than the one they were measured on would report a meaningless +# verdict. NOT $BEDROCK_MODEL: that is the evaluation-side model (llm_judge + the +# simulated user), which must not move with the agent under test. +AGENT_MODEL = os.environ.get("AGENT_MODEL", "").strip() # Rounded recall.yes baseline (in %) per skill, measured 2026-06-17 over each # skill's FULL positive set on claude-sonnet-4-6 via Bedrock at max_turns: 1 — @@ -112,6 +113,10 @@ def main() -> int: parser.add_argument("--skill", required=True) skill = parser.parse_args().skill + if not AGENT_MODEL: + print("ERROR: AGENT_MODEL is unset — set the CLAUDE_CODE_MODEL repo variable", file=sys.stderr) + return 2 + if skill not in BASELINES_PCT: print(f"SKIP: no baseline for {skill!r}", file=sys.stderr) return 0 From 79782fbbf9b39ac8d6a9b7da4999c3ee3202a495 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Fri, 31 Jul 2026 16:57:45 -0700 Subject: [PATCH 3/3] fix(ci): fail the smoke gate closed when no task runs The model guard added with the env-var switch sat as the first line of the smoke step, which carries continue-on-error, and both threshold steps read zero tasks as a pass. An unresolved CLAUDE_CODE_MODEL therefore ran nothing and reported green (run 30672478895 on this branch did exactly that). Hoist the guard into its own step in both smoke workflows so an unset variable fails before any task runs, and make the Linux pass-rate gate fail closed when the smoke step failed without producing a single task.json. A legitimate empty selection still passes: that path leaves the smoke step successful. The RPA score gate already failed closed on zero tasks, which is why it went red where the Linux gate went green. Also collapses the six identical five-line comment blocks in the experiment YAMLs to one line each. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/smoke-rpa-skills.yml | 15 ++++++++++--- .github/workflows/smoke-skills.yml | 21 +++++++++++++++++-- tests/experiments/activation.yaml | 6 +----- tests/experiments/default.yaml | 6 +----- tests/experiments/nightly.yaml | 6 +----- .../skill-comparison-template.yaml | 6 +----- tests/experiments/smoke-windows.yaml | 6 +----- tests/experiments/smoke.yaml | 6 +----- 8 files changed, 37 insertions(+), 35 deletions(-) diff --git a/.github/workflows/smoke-rpa-skills.yml b/.github/workflows/smoke-rpa-skills.yml index 07c10b7ea8..f2927bfd63 100644 --- a/.github/workflows/smoke-rpa-skills.yml +++ b/.github/workflows/smoke-rpa-skills.yml @@ -272,6 +272,16 @@ jobs: uip rpa list-instances --output json 2>&1 || true taskkill //F //IM UiPath.Studio.Helm.exe 2>/dev/null || true + # Model guard, deliberately its OWN step (mirror of smoke-skills.yml): the smoke + # step below carries continue-on-error, so a guard inside it is swallowed and the + # failure surfaces only as a downstream "no task.json" error. Fail here instead, + # named, before the run starts. + - name: Require an agent model + shell: bash + env: + AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} + run: ': "${AGENT_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}"' + - name: Run RPA smoke tests env: SKILLS_REPO_PATH: ${{ github.workspace }} @@ -292,14 +302,13 @@ jobs: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} # The gate's agent model, passed on the command line rather than pinned in # experiments/smoke-windows.yaml (see smoke-skills.yml for why). No literal - # fallback: an unset variable fails the step rather than gating a model - # nobody configured. + # fallback: an unset variable fails the guard step above rather than gating a + # model nobody configured. AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} working-directory: tests id: smoke shell: bash run: | - : "${AGENT_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}" # Run each task separately, killing Helm between tasks so one # task's stale Studio state doesn't leak into the next. Track # per-task exit codes so one failure doesn't abort the loop diff --git a/.github/workflows/smoke-skills.yml b/.github/workflows/smoke-skills.yml index 3f73da7923..536bd2b40e 100644 --- a/.github/workflows/smoke-skills.yml +++ b/.github/workflows/smoke-skills.yml @@ -396,6 +396,15 @@ jobs: uip agent guardrails list --output json ' + # Model guard, deliberately its OWN step: "Run smoke tests" below carries + # continue-on-error, and the threshold steps treat zero tasks as a pass, so a + # guard inside the smoke step is swallowed and the gate goes green having + # evaluated nothing. Fail here instead, before any task runs. + - name: Require an agent model + env: + AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} + run: ': "${AGENT_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}"' + - name: Run smoke tests env: SKILLS_REPO_PATH: ${{ github.workspace }} @@ -420,14 +429,13 @@ jobs: # and the experiment file stays harness-agnostic (the nightly runs the same # file under codex/antigravity with their own models). No literal fallback: # gating on a model nobody configured reports a verdict about the wrong thing, - # so an unset variable fails the step (guard below). NOT $BEDROCK_MODEL: that + # so an unset variable fails the guard step above. NOT $BEDROCK_MODEL: that # is the evaluation-side model (llm_judge + the simulated user), and reusing # it here would move the grader with the agent. AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} working-directory: tests id: smoke run: | - : "${AGENT_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}" shopt -s globstar nullglob echo "Running: coder-eval run ${{ needs.detect.outputs.task_globs }} --model $AGENT_MODEL --tags smoke -j $TASK_PARALLELISM" coder-eval run ${{ needs.detect.outputs.task_globs }} \ @@ -594,6 +602,11 @@ jobs: if: always() env: PASS_RATE_THRESHOLD: "0.95" + # The smoke step carries continue-on-error, so its failure only reaches the + # job through this gate. Zero tasks AFTER a failed smoke step means the run + # died before producing anything (bad install, unusable config) — fail closed + # rather than reading an empty set as a clean sweep. + SMOKE_OUTCOME: ${{ steps.smoke.outcome }} run: | if [ "${{ steps.smoke.outcome }}" = "cancelled" ]; then echo "::error::Smoke step cancelled (timeout) — failing without rate check" @@ -607,6 +620,10 @@ jobs: passed = sum(1 for d in results if d.get('final_status') == 'SUCCESS') n = len(results) if n == 0: + if os.environ.get('SMOKE_OUTCOME') == 'failure': + print('::error::Smoke step failed without producing a single task.json ' + '— failing closed instead of passing on an empty run') + sys.exit(1) print('No smoke-tagged tasks matched — skipping rate check') sys.exit(0) rate = passed / n diff --git a/tests/experiments/activation.yaml b/tests/experiments/activation.yaml index 449adee9fa..4a361f9fb3 100644 --- a/tests/experiments/activation.yaml +++ b/tests/experiments/activation.yaml @@ -38,11 +38,7 @@ defaults: turn_timeout: 120 agent: - # type AND model omitted on purpose — the runner passes both as CLI flags, and those - # outrank this file: --type picks the harness (codex/antigravity/delegate-sdk; - # coder-eval's baseline default experiment supplies claude-code otherwise), and - # --model carries that harness's model, which the CI wrapper resolves from that - # harness's own env var. A bare local run takes the model from $BEDROCK_MODEL. + # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file. permission_mode: acceptEdits # Skill must be allowed so the agent can invoke a skill when it decides # to. Read/Write/Bash included so the agent isn't starved of basic tools diff --git a/tests/experiments/default.yaml b/tests/experiments/default.yaml index cfe02ccb6e..dc2c4c9c11 100644 --- a/tests/experiments/default.yaml +++ b/tests/experiments/default.yaml @@ -13,11 +13,7 @@ defaults: driver: tempdir agent: - # type AND model omitted on purpose — the runner passes both as CLI flags, and those - # outrank this file: --type picks the harness (codex/antigravity/delegate-sdk; - # coder-eval's baseline default experiment supplies claude-code otherwise), and - # --model carries that harness's model, which the CI wrapper resolves from that - # harness's own env var. A bare local run takes the model from $BEDROCK_MODEL. + # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file. permission_mode: acceptEdits allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | diff --git a/tests/experiments/nightly.yaml b/tests/experiments/nightly.yaml index d839925dd9..755271fe7b 100644 --- a/tests/experiments/nightly.yaml +++ b/tests/experiments/nightly.yaml @@ -31,11 +31,7 @@ defaults: - ~/.uipath:/.uipath:rw agent: - # type AND model omitted on purpose — the runner passes both as CLI flags, and those - # outrank this file: --type picks the harness (codex/antigravity/delegate-sdk; - # coder-eval's baseline default experiment supplies claude-code otherwise), and - # --model carries that harness's model, which the CI wrapper resolves from that - # harness's own env var. A bare local run takes the model from $BEDROCK_MODEL. + # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file. permission_mode: acceptEdits allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | diff --git a/tests/experiments/skill-comparison-template.yaml b/tests/experiments/skill-comparison-template.yaml index dafb22408c..05895c028a 100644 --- a/tests/experiments/skill-comparison-template.yaml +++ b/tests/experiments/skill-comparison-template.yaml @@ -26,11 +26,7 @@ defaults: turn_timeout: 300 repeats: 1 agent: - # type AND model omitted on purpose — the runner passes both as CLI flags, and those - # outrank this file: --type picks the harness (codex/antigravity/delegate-sdk; - # coder-eval's baseline default experiment supplies claude-code otherwise), and - # --model carries that harness's model, which the CI wrapper resolves from that - # harness's own env var. A bare local run takes the model from $BEDROCK_MODEL. + # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file. permission_mode: bypassPermissions allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | diff --git a/tests/experiments/smoke-windows.yaml b/tests/experiments/smoke-windows.yaml index 056cf770a4..a67df4ec33 100644 --- a/tests/experiments/smoke-windows.yaml +++ b/tests/experiments/smoke-windows.yaml @@ -15,11 +15,7 @@ defaults: driver: tempdir agent: - # type AND model omitted on purpose — the runner passes both as CLI flags, and those - # outrank this file: --type picks the harness (codex/antigravity/delegate-sdk; - # coder-eval's baseline default experiment supplies claude-code otherwise), and - # --model carries that harness's model, which the CI wrapper resolves from that - # harness's own env var. A bare local run takes the model from $BEDROCK_MODEL. + # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file. permission_mode: acceptEdits allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | diff --git a/tests/experiments/smoke.yaml b/tests/experiments/smoke.yaml index f3fb121291..844d82bfa0 100644 --- a/tests/experiments/smoke.yaml +++ b/tests/experiments/smoke.yaml @@ -31,11 +31,7 @@ defaults: - ~/.uipath:/.uipath:rw agent: - # type AND model omitted on purpose — the runner passes both as CLI flags, and those - # outrank this file: --type picks the harness (codex/antigravity/delegate-sdk; - # coder-eval's baseline default experiment supplies claude-code otherwise), and - # --model carries that harness's model, which the CI wrapper resolves from that - # harness's own env var. A bare local run takes the model from $BEDROCK_MODEL. + # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file. permission_mode: acceptEdits allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: |