feat(tests): take the eval model from env vars, bump coder_eval to 0.9.2

.github/workflows/activation-gate.yml

@@ -103,4 +103,8 @@ jobs:
           AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
           AWS_REGION: ${{ secrets.AWS_REGION }}
           BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }}
+          # The agent model the gate measures (activation_gate.py passes it as --model,
+          # and exits non-zero when this is unset — no literal fallback). Its recall
+          # baselines are model-specific, so re-baseline when this moves.
+          AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }}
         run: python tests/scripts/activation_gate.py --skill "$SKILL"

.github/workflows/run-coder-eval.yml

@@ -55,9 +55,8 @@ on:
           - codex
           - antigravity
         default: claude
-      # Overrides the model for ANY agent. Blank keeps the per-agent default:
-      # claude → experiment YAML's sonnet (override e.g. opus / haiku),
-      # codex → gpt-5.6-terra, antigravity → gemini-3.5-flash.
+      # Overrides the model for ANY agent. Blank keeps that harness's own repo
+      # variable (CLAUDE_CODE_MODEL / CODEX_MODEL / ANTIGRAVITY_MODEL).
       agent_model:
         description: 'Model override (blank = default).'
         type: string
@@ -249,6 +248,12 @@ jobs:
           TASK_PARALLELISM:         ${{ inputs.parallelism }}
           AGENT:                    ${{ inputs.agent }}
           AGENT_MODEL:              ${{ inputs.agent_model }}
+          # The model when agent_model is blank: one repo variable per harness, named the
+          # same as the nightly runner's, and no hardcoded fallback — an unset variable
+          # fails the run (guard below) rather than evaluating a model nobody chose.
+          CLAUDE_CODE_MODEL:        ${{ vars.CLAUDE_CODE_MODEL }}
+          CODEX_MODEL:              ${{ vars.CODEX_MODEL }}
+          ANTIGRAVITY_MODEL:        ${{ vars.ANTIGRAVITY_MODEL }}
           CODEX_API_KEY:            ${{ secrets.CODEX_API_KEY }}
           CODEX_BASE_URL:           ${{ secrets.CODEX_BASE_URL }}
           GEMINI_API_KEY:           ${{ secrets.GEMINI_API_KEY }}
@@ -258,19 +263,21 @@ jobs:
           # Guard the empty case (e.g. a future non-dispatch trigger): the
           # `parallelism` input default (4) only applies on workflow_dispatch.
           j="${TASK_PARALLELISM:-4}"
-          # Agent selection. claude (default) takes the experiment YAML's
-          # claude-code/sonnet config; codex overrides to 5.6-terra (auth via
-          # CODEX_API_KEY/CODEX_BASE_URL); antigravity overrides to
-          # gemini-3.5-flash (auth via GEMINI_API_KEY; SDK baked into the
-          # agent image). AGENT_MODEL overrides the model for any agent (e.g.
-          # opus / haiku for claude). Driver is unchanged (docker here).
-          agent_flags=""
+          # Agent selection. claude is the default; codex authenticates via
+          # CODEX_API_KEY/CODEX_BASE_URL, antigravity via GEMINI_API_KEY (SDK baked into
+          # the agent image). The model always comes from the harness's own variable
+          # above, with the agent_model input overriding any of them for a one-off run.
+          # Passed explicitly because the experiment YAML pins no model (it is shared by
+          # all three harnesses). Driver is unchanged (docker here).
           if [ "$AGENT" = "codex" ]; then
-            agent_flags="--type codex --model ${AGENT_MODEL:-gpt-5.6-terra}"
+            model="${AGENT_MODEL:-${CODEX_MODEL:?unset — set the CODEX_MODEL repo variable}}"
+            agent_flags="--type codex --model $model"
           elif [ "$AGENT" = "antigravity" ]; then
-            agent_flags="--type antigravity --model ${AGENT_MODEL:-gemini-3.5-flash}"
-          elif [ -n "$AGENT_MODEL" ]; then
-            agent_flags="--model $AGENT_MODEL"
+            model="${AGENT_MODEL:-${ANTIGRAVITY_MODEL:?unset — set the ANTIGRAVITY_MODEL repo variable}}"
+            agent_flags="--type antigravity --model $model"
+          else
+            model="${AGENT_MODEL:-${CLAUDE_CODE_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}}"
+            agent_flags="--model $model"
           fi
           echo "Running: coder-eval run $TASK_GLOBS -e experiments/nightly.yaml $agent_flags -j $j ($TASK_COUNT task files)"
           coder-eval run $TASK_GLOBS \
@@ -522,6 +529,12 @@ jobs:
           TASK_GLOBS:               ${{ needs.partition.outputs.windows_globs }}
           AGENT:                    ${{ inputs.agent }}
           AGENT_MODEL:              ${{ inputs.agent_model }}
+          # The model when agent_model is blank: one repo variable per harness, named the
+          # same as the nightly runner's, and no hardcoded fallback — an unset variable
+          # fails the run (guard below) rather than evaluating a model nobody chose.
+          CLAUDE_CODE_MODEL:        ${{ vars.CLAUDE_CODE_MODEL }}
+          CODEX_MODEL:              ${{ vars.CODEX_MODEL }}
+          ANTIGRAVITY_MODEL:        ${{ vars.ANTIGRAVITY_MODEL }}
           CODEX_API_KEY:            ${{ secrets.CODEX_API_KEY }}
           CODEX_BASE_URL:           ${{ secrets.CODEX_BASE_URL }}
           GEMINI_API_KEY:           ${{ secrets.GEMINI_API_KEY }}
@@ -543,18 +556,19 @@ jobs:
           shopt -s globstar nullglob
           # Create the tilde-free temp root pinned via TMP/TEMP above.
           mkdir -p /c/cetmp
-          # Agent selection. claude (default) takes the experiment YAML config;
-          # codex overrides to gpt-5.6-terra (auth via CODEX_API_KEY/CODEX_BASE_URL);
-          # antigravity overrides to gemini-3.5-flash (auth via
-          # GEMINI_API_KEY). AGENT_MODEL overrides the model for any agent (e.g.
-          # opus / haiku for claude). Driver is unchanged (tempdir here).
-          agent_flags=""
+          # Agent selection (mirror of the Linux job): the model always comes from the
+          # harness's own variable above, with the agent_model input overriding it for a
+          # one-off run. Passed explicitly because the experiment YAML pins no model.
+          # Driver is unchanged (tempdir here).
           if [ "$AGENT" = "codex" ]; then
-            agent_flags="--type codex --model ${AGENT_MODEL:-gpt-5.6-terra}"
+            model="${AGENT_MODEL:-${CODEX_MODEL:?unset — set the CODEX_MODEL repo variable}}"
+            agent_flags="--type codex --model $model"
           elif [ "$AGENT" = "antigravity" ]; then
-            agent_flags="--type antigravity --model ${AGENT_MODEL:-gemini-3.5-flash}"
-          elif [ -n "$AGENT_MODEL" ]; then
-            agent_flags="--model $AGENT_MODEL"
+            model="${AGENT_MODEL:-${ANTIGRAVITY_MODEL:?unset — set the ANTIGRAVITY_MODEL repo variable}}"
+            agent_flags="--type antigravity --model $model"
+          else
+            model="${AGENT_MODEL:-${CLAUDE_CODE_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}}"
+            agent_flags="--model $model"
           fi
           overall_exit=0
           for task in $TASK_GLOBS; do

.github/workflows/smoke-rpa-skills.yml

@@ -272,6 +272,16 @@ jobs:
           uip rpa list-instances --output json 2>&1 || true
           taskkill //F //IM UiPath.Studio.Helm.exe 2>/dev/null || true
 
+      # Model guard, deliberately its OWN step (mirror of smoke-skills.yml): the smoke
+      # step below carries continue-on-error, so a guard inside it is swallowed and the
+      # failure surfaces only as a downstream "no task.json" error. Fail here instead,
+      # named, before the run starts.
+      - name: Require an agent model
+        shell: bash
+        env:
+          AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }}
+        run: ': "${AGENT_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}"'
+
       - name: Run RPA smoke tests
         env:
           SKILLS_REPO_PATH: ${{ github.workspace }}
@@ -290,6 +300,11 @@ jobs:
           AWS_REGION: ${{ secrets.AWS_REGION }}
           BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }}
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          # The gate's agent model, passed on the command line rather than pinned in
+          # experiments/smoke-windows.yaml (see smoke-skills.yml for why). No literal
+          # fallback: an unset variable fails the guard step above rather than gating a
+          # model nobody configured.
+          AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }}
         working-directory: tests
         id: smoke
         shell: bash
@@ -329,7 +344,7 @@ jobs:
             for attempt in 1 2 3; do
               echo "--- $task attempt $attempt ---"
               if coder-eval run "$task" \
-                  -e experiments/smoke-windows.yaml --tags smoke -j 1 -v; then
+                  -e experiments/smoke-windows.yaml --model "$AGENT_MODEL" --tags smoke -j 1 -v; then
                 break
               fi
               # Only retry on Bedrock content-filter ERRORs (not on real

.github/workflows/smoke-skills.yml

@@ -396,6 +396,15 @@ jobs:
               uip agent guardrails list --output json
             '
 
+      # Model guard, deliberately its OWN step: "Run smoke tests" below carries
+      # continue-on-error, and the threshold steps treat zero tasks as a pass, so a
+      # guard inside the smoke step is swallowed and the gate goes green having
+      # evaluated nothing. Fail here instead, before any task runs.
+      - name: Require an agent model
+        env:
+          AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }}
+        run: ': "${AGENT_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}"'
+
       - name: Run smoke tests
         env:
           SKILLS_REPO_PATH: ${{ github.workspace }}
@@ -415,13 +424,23 @@ jobs:
           BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }}
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           TASK_PARALLELISM: ${{ vars.TASK_PARALLELISM || '1' }}
+          # The gate's agent model, passed explicitly on the command line rather than
+          # pinned in experiments/smoke.yaml — one repo variable moves every gate here,
+          # and the experiment file stays harness-agnostic (the nightly runs the same
+          # file under codex/antigravity with their own models). No literal fallback:
+          # gating on a model nobody configured reports a verdict about the wrong thing,
+          # so an unset variable fails the guard step above. NOT $BEDROCK_MODEL: that
+          # is the evaluation-side model (llm_judge + the simulated user), and reusing
+          # it here would move the grader with the agent.
+          AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }}
         working-directory: tests
         id: smoke
         run: |
           shopt -s globstar nullglob
-          echo "Running: coder-eval run ${{ needs.detect.outputs.task_globs }} --tags smoke -j $TASK_PARALLELISM"
+          echo "Running: coder-eval run ${{ needs.detect.outputs.task_globs }} --model $AGENT_MODEL --tags smoke -j $TASK_PARALLELISM"
           coder-eval run ${{ needs.detect.outputs.task_globs }} \
             -e experiments/smoke.yaml \
+            --model "$AGENT_MODEL" \
             -j "$TASK_PARALLELISM" -v \
             --run-dir /tmp/runs \
             --tags smoke
@@ -583,6 +602,11 @@ jobs:
         if: always()
         env:
           PASS_RATE_THRESHOLD: "0.95"
+          # The smoke step carries continue-on-error, so its failure only reaches the
+          # job through this gate. Zero tasks AFTER a failed smoke step means the run
+          # died before producing anything (bad install, unusable config) — fail closed
+          # rather than reading an empty set as a clean sweep.
+          SMOKE_OUTCOME: ${{ steps.smoke.outcome }}
         run: |
           if [ "${{ steps.smoke.outcome }}" = "cancelled" ]; then
             echo "::error::Smoke step cancelled (timeout) — failing without rate check"
@@ -596,6 +620,10 @@ jobs:
           passed = sum(1 for d in results if d.get('final_status') == 'SUCCESS')
           n = len(results)
           if n == 0:
+              if os.environ.get('SMOKE_OUTCOME') == 'failure':
+                  print('::error::Smoke step failed without producing a single task.json '
+                        '— failing closed instead of passing on an empty run')
+                  sys.exit(1)
               print('No smoke-tagged tasks matched — skipping rate check')
               sys.exit(0)
           rate = passed / n

tests/.coder-eval-version

@@ -1 +1 @@
-0.9.1
+0.9.2

tests/experiments/activation.yaml

@@ -38,10 +38,8 @@ defaults:
     turn_timeout: 120
 
   agent:
-    # type omitted on purpose — the runner selects the harness via --type (codex/antigravity);
-    # coder-eval's baseline default experiment supplies claude-code otherwise.
+    # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file.
     permission_mode: acceptEdits
-    model: claude-sonnet-5
     # Skill must be allowed so the agent can invoke a skill when it decides
     # to. Read/Write/Bash included so the agent isn't starved of basic tools
     # when it decides NOT to use Skill — removing them would bias the

tests/experiments/default.yaml

@@ -13,10 +13,8 @@ defaults:
     driver: tempdir
 
   agent:
-    # type omitted on purpose — the runner selects the harness via --type (codex/antigravity);
-    # coder-eval's baseline default experiment supplies claude-code otherwise.
+    # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file.
     permission_mode: acceptEdits
-    model: claude-sonnet-5
     allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"]
     system_prompt: |
       You are a coding agent. Do not access files in sibling runs/* directories. Everywhere else is permitted.

tests/experiments/nightly.yaml

@@ -31,10 +31,8 @@ defaults:
         - ~/.uipath:/.uipath:rw
 
   agent:
-    # type omitted on purpose — the runner selects the harness via --type (codex/antigravity);
-    # coder-eval's baseline default experiment supplies claude-code otherwise.
+    # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file.
     permission_mode: acceptEdits
-    model: claude-sonnet-5
     allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"]
     system_prompt: |
       You are a coding agent. Do not access files in sibling runs/* directories. Everywhere else is permitted.

tests/experiments/skill-comparison-template.yaml

@@ -26,10 +26,8 @@ defaults:
     turn_timeout: 300
   repeats: 1
   agent:
-    # type omitted on purpose — the runner selects the harness via --type (codex/antigravity);
-    # coder-eval's baseline default experiment supplies claude-code otherwise.
+    # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file.
     permission_mode: bypassPermissions
-    model: claude-sonnet-5
     allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"]
     system_prompt: |
       You are a coding agent. Work only inside your current working directory.

tests/experiments/smoke-windows.yaml

@@ -15,10 +15,8 @@ defaults:
     driver: tempdir
 
   agent:
-    # type omitted on purpose — the runner selects the harness via --type (codex/antigravity);
-    # coder-eval's baseline default experiment supplies claude-code otherwise.
+    # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file.
     permission_mode: acceptEdits
-    model: claude-sonnet-5
     allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"]
     system_prompt: |
       You are a coding agent. Do not access files in sibling runs/* directories. Everywhere else is permitted.

tests/experiments/smoke.yaml

@@ -31,10 +31,8 @@ defaults:
         - ~/.uipath:/.uipath:rw
 
   agent:
-    # type omitted on purpose — the runner selects the harness via --type (codex/antigravity);
-    # coder-eval's baseline default experiment supplies claude-code otherwise.
+    # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file.
     permission_mode: acceptEdits
-    model: claude-sonnet-5
     allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"]
     system_prompt: |
       You are a coding agent. Do not access files in sibling runs/* directories. Everywhere else is permitted.

tests/scripts/activation_gate.py

@@ -12,11 +12,21 @@
 
 import argparse
 import json
+import os
 import subprocess
 import sys
 import tempfile
 from pathlib import Path
 
+# The gate's agent model. Passed explicitly rather than pinned in
+# experiments/activation.yaml so one variable moves every eval entry point in this
+# repo, and so the model the baselines below were measured against is visible at the
+# call site. No default: the baselines are model-specific, so silently gating a
+# different model than the one they were measured on would report a meaningless
+# verdict. NOT $BEDROCK_MODEL: that is the evaluation-side model (llm_judge + the
+# simulated user), which must not move with the agent under test.
+AGENT_MODEL = os.environ.get("AGENT_MODEL", "").strip()
+
 # Rounded recall.yes baseline (in %) per skill, measured 2026-06-17 over each
 # skill's FULL positive set on claude-sonnet-4-6 via Bedrock at max_turns: 1 —
 # the same model and full-set measurement the gate itself runs. The gate task
@@ -103,6 +113,10 @@ def main() -> int:
     parser.add_argument("--skill", required=True)
     skill = parser.parse_args().skill
 
+    if not AGENT_MODEL:
+        print("ERROR: AGENT_MODEL is unset — set the CLAUDE_CODE_MODEL repo variable", file=sys.stderr)
+        return 2
+
     if skill not in BASELINES_PCT:
         print(f"SKIP: no baseline for {skill!r}", file=sys.stderr)
         return 0
@@ -127,6 +141,7 @@ def main() -> int:
             [
                 "coder-eval", "run", str(task_yaml),
                 "-e", "tests/experiments/activation.yaml",
+                "--model", AGENT_MODEL,
                 "-j", "4",
                 "--run-dir", str(run_dir),
             ],