diff --git a/.github/workflows/activation-gate.yml b/.github/workflows/activation-gate.yml index 59f17978d9..0817f57ee1 100644 --- a/.github/workflows/activation-gate.yml +++ b/.github/workflows/activation-gate.yml @@ -103,4 +103,8 @@ jobs: AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} AWS_REGION: ${{ secrets.AWS_REGION }} BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }} + # The agent model the gate measures (activation_gate.py passes it as --model, + # and exits non-zero when this is unset — no literal fallback). Its recall + # baselines are model-specific, so re-baseline when this moves. + AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} run: python tests/scripts/activation_gate.py --skill "$SKILL" diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index 7e59d78019..a6b07505c9 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -55,9 +55,8 @@ on: - codex - antigravity default: claude - # Overrides the model for ANY agent. Blank keeps the per-agent default: - # claude → experiment YAML's sonnet (override e.g. opus / haiku), - # codex → gpt-5.6-terra, antigravity → gemini-3.5-flash. + # Overrides the model for ANY agent. Blank keeps that harness's own repo + # variable (CLAUDE_CODE_MODEL / CODEX_MODEL / ANTIGRAVITY_MODEL). agent_model: description: 'Model override (blank = default).' type: string @@ -249,6 +248,12 @@ jobs: TASK_PARALLELISM: ${{ inputs.parallelism }} AGENT: ${{ inputs.agent }} AGENT_MODEL: ${{ inputs.agent_model }} + # The model when agent_model is blank: one repo variable per harness, named the + # same as the nightly runner's, and no hardcoded fallback — an unset variable + # fails the run (guard below) rather than evaluating a model nobody chose. + CLAUDE_CODE_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} + CODEX_MODEL: ${{ vars.CODEX_MODEL }} + ANTIGRAVITY_MODEL: ${{ vars.ANTIGRAVITY_MODEL }} CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} CODEX_BASE_URL: ${{ secrets.CODEX_BASE_URL }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} @@ -258,19 +263,21 @@ jobs: # Guard the empty case (e.g. a future non-dispatch trigger): the # `parallelism` input default (4) only applies on workflow_dispatch. j="${TASK_PARALLELISM:-4}" - # Agent selection. claude (default) takes the experiment YAML's - # claude-code/sonnet config; codex overrides to 5.6-terra (auth via - # CODEX_API_KEY/CODEX_BASE_URL); antigravity overrides to - # gemini-3.5-flash (auth via GEMINI_API_KEY; SDK baked into the - # agent image). AGENT_MODEL overrides the model for any agent (e.g. - # opus / haiku for claude). Driver is unchanged (docker here). - agent_flags="" + # Agent selection. claude is the default; codex authenticates via + # CODEX_API_KEY/CODEX_BASE_URL, antigravity via GEMINI_API_KEY (SDK baked into + # the agent image). The model always comes from the harness's own variable + # above, with the agent_model input overriding any of them for a one-off run. + # Passed explicitly because the experiment YAML pins no model (it is shared by + # all three harnesses). Driver is unchanged (docker here). if [ "$AGENT" = "codex" ]; then - agent_flags="--type codex --model ${AGENT_MODEL:-gpt-5.6-terra}" + model="${AGENT_MODEL:-${CODEX_MODEL:?unset — set the CODEX_MODEL repo variable}}" + agent_flags="--type codex --model $model" elif [ "$AGENT" = "antigravity" ]; then - agent_flags="--type antigravity --model ${AGENT_MODEL:-gemini-3.5-flash}" - elif [ -n "$AGENT_MODEL" ]; then - agent_flags="--model $AGENT_MODEL" + model="${AGENT_MODEL:-${ANTIGRAVITY_MODEL:?unset — set the ANTIGRAVITY_MODEL repo variable}}" + agent_flags="--type antigravity --model $model" + else + model="${AGENT_MODEL:-${CLAUDE_CODE_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}}" + agent_flags="--model $model" fi echo "Running: coder-eval run $TASK_GLOBS -e experiments/nightly.yaml $agent_flags -j $j ($TASK_COUNT task files)" coder-eval run $TASK_GLOBS \ @@ -522,6 +529,12 @@ jobs: TASK_GLOBS: ${{ needs.partition.outputs.windows_globs }} AGENT: ${{ inputs.agent }} AGENT_MODEL: ${{ inputs.agent_model }} + # The model when agent_model is blank: one repo variable per harness, named the + # same as the nightly runner's, and no hardcoded fallback — an unset variable + # fails the run (guard below) rather than evaluating a model nobody chose. + CLAUDE_CODE_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} + CODEX_MODEL: ${{ vars.CODEX_MODEL }} + ANTIGRAVITY_MODEL: ${{ vars.ANTIGRAVITY_MODEL }} CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} CODEX_BASE_URL: ${{ secrets.CODEX_BASE_URL }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} @@ -543,18 +556,19 @@ jobs: shopt -s globstar nullglob # Create the tilde-free temp root pinned via TMP/TEMP above. mkdir -p /c/cetmp - # Agent selection. claude (default) takes the experiment YAML config; - # codex overrides to gpt-5.6-terra (auth via CODEX_API_KEY/CODEX_BASE_URL); - # antigravity overrides to gemini-3.5-flash (auth via - # GEMINI_API_KEY). AGENT_MODEL overrides the model for any agent (e.g. - # opus / haiku for claude). Driver is unchanged (tempdir here). - agent_flags="" + # Agent selection (mirror of the Linux job): the model always comes from the + # harness's own variable above, with the agent_model input overriding it for a + # one-off run. Passed explicitly because the experiment YAML pins no model. + # Driver is unchanged (tempdir here). if [ "$AGENT" = "codex" ]; then - agent_flags="--type codex --model ${AGENT_MODEL:-gpt-5.6-terra}" + model="${AGENT_MODEL:-${CODEX_MODEL:?unset — set the CODEX_MODEL repo variable}}" + agent_flags="--type codex --model $model" elif [ "$AGENT" = "antigravity" ]; then - agent_flags="--type antigravity --model ${AGENT_MODEL:-gemini-3.5-flash}" - elif [ -n "$AGENT_MODEL" ]; then - agent_flags="--model $AGENT_MODEL" + model="${AGENT_MODEL:-${ANTIGRAVITY_MODEL:?unset — set the ANTIGRAVITY_MODEL repo variable}}" + agent_flags="--type antigravity --model $model" + else + model="${AGENT_MODEL:-${CLAUDE_CODE_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}}" + agent_flags="--model $model" fi overall_exit=0 for task in $TASK_GLOBS; do diff --git a/.github/workflows/smoke-rpa-skills.yml b/.github/workflows/smoke-rpa-skills.yml index 2fb564a451..f2927bfd63 100644 --- a/.github/workflows/smoke-rpa-skills.yml +++ b/.github/workflows/smoke-rpa-skills.yml @@ -272,6 +272,16 @@ jobs: uip rpa list-instances --output json 2>&1 || true taskkill //F //IM UiPath.Studio.Helm.exe 2>/dev/null || true + # Model guard, deliberately its OWN step (mirror of smoke-skills.yml): the smoke + # step below carries continue-on-error, so a guard inside it is swallowed and the + # failure surfaces only as a downstream "no task.json" error. Fail here instead, + # named, before the run starts. + - name: Require an agent model + shell: bash + env: + AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} + run: ': "${AGENT_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}"' + - name: Run RPA smoke tests env: SKILLS_REPO_PATH: ${{ github.workspace }} @@ -290,6 +300,11 @@ jobs: AWS_REGION: ${{ secrets.AWS_REGION }} BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + # The gate's agent model, passed on the command line rather than pinned in + # experiments/smoke-windows.yaml (see smoke-skills.yml for why). No literal + # fallback: an unset variable fails the guard step above rather than gating a + # model nobody configured. + AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} working-directory: tests id: smoke shell: bash @@ -329,7 +344,7 @@ jobs: for attempt in 1 2 3; do echo "--- $task attempt $attempt ---" if coder-eval run "$task" \ - -e experiments/smoke-windows.yaml --tags smoke -j 1 -v; then + -e experiments/smoke-windows.yaml --model "$AGENT_MODEL" --tags smoke -j 1 -v; then break fi # Only retry on Bedrock content-filter ERRORs (not on real diff --git a/.github/workflows/smoke-skills.yml b/.github/workflows/smoke-skills.yml index 1e165e71cb..536bd2b40e 100644 --- a/.github/workflows/smoke-skills.yml +++ b/.github/workflows/smoke-skills.yml @@ -396,6 +396,15 @@ jobs: uip agent guardrails list --output json ' + # Model guard, deliberately its OWN step: "Run smoke tests" below carries + # continue-on-error, and the threshold steps treat zero tasks as a pass, so a + # guard inside the smoke step is swallowed and the gate goes green having + # evaluated nothing. Fail here instead, before any task runs. + - name: Require an agent model + env: + AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} + run: ': "${AGENT_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}"' + - name: Run smoke tests env: SKILLS_REPO_PATH: ${{ github.workspace }} @@ -415,13 +424,23 @@ jobs: BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} TASK_PARALLELISM: ${{ vars.TASK_PARALLELISM || '1' }} + # The gate's agent model, passed explicitly on the command line rather than + # pinned in experiments/smoke.yaml — one repo variable moves every gate here, + # and the experiment file stays harness-agnostic (the nightly runs the same + # file under codex/antigravity with their own models). No literal fallback: + # gating on a model nobody configured reports a verdict about the wrong thing, + # so an unset variable fails the guard step above. NOT $BEDROCK_MODEL: that + # is the evaluation-side model (llm_judge + the simulated user), and reusing + # it here would move the grader with the agent. + AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} working-directory: tests id: smoke run: | shopt -s globstar nullglob - echo "Running: coder-eval run ${{ needs.detect.outputs.task_globs }} --tags smoke -j $TASK_PARALLELISM" + echo "Running: coder-eval run ${{ needs.detect.outputs.task_globs }} --model $AGENT_MODEL --tags smoke -j $TASK_PARALLELISM" coder-eval run ${{ needs.detect.outputs.task_globs }} \ -e experiments/smoke.yaml \ + --model "$AGENT_MODEL" \ -j "$TASK_PARALLELISM" -v \ --run-dir /tmp/runs \ --tags smoke @@ -583,6 +602,11 @@ jobs: if: always() env: PASS_RATE_THRESHOLD: "0.95" + # The smoke step carries continue-on-error, so its failure only reaches the + # job through this gate. Zero tasks AFTER a failed smoke step means the run + # died before producing anything (bad install, unusable config) — fail closed + # rather than reading an empty set as a clean sweep. + SMOKE_OUTCOME: ${{ steps.smoke.outcome }} run: | if [ "${{ steps.smoke.outcome }}" = "cancelled" ]; then echo "::error::Smoke step cancelled (timeout) — failing without rate check" @@ -596,6 +620,10 @@ jobs: passed = sum(1 for d in results if d.get('final_status') == 'SUCCESS') n = len(results) if n == 0: + if os.environ.get('SMOKE_OUTCOME') == 'failure': + print('::error::Smoke step failed without producing a single task.json ' + '— failing closed instead of passing on an empty run') + sys.exit(1) print('No smoke-tagged tasks matched — skipping rate check') sys.exit(0) rate = passed / n diff --git a/tests/.coder-eval-version b/tests/.coder-eval-version index f374f6662e..2003b639c4 100644 --- a/tests/.coder-eval-version +++ b/tests/.coder-eval-version @@ -1 +1 @@ -0.9.1 +0.9.2 diff --git a/tests/experiments/activation.yaml b/tests/experiments/activation.yaml index 0121f8442d..4a361f9fb3 100644 --- a/tests/experiments/activation.yaml +++ b/tests/experiments/activation.yaml @@ -38,10 +38,8 @@ defaults: turn_timeout: 120 agent: - # type omitted on purpose — the runner selects the harness via --type (codex/antigravity); - # coder-eval's baseline default experiment supplies claude-code otherwise. + # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file. permission_mode: acceptEdits - model: claude-sonnet-5 # Skill must be allowed so the agent can invoke a skill when it decides # to. Read/Write/Bash included so the agent isn't starved of basic tools # when it decides NOT to use Skill — removing them would bias the diff --git a/tests/experiments/default.yaml b/tests/experiments/default.yaml index fd28926080..dc2c4c9c11 100644 --- a/tests/experiments/default.yaml +++ b/tests/experiments/default.yaml @@ -13,10 +13,8 @@ defaults: driver: tempdir agent: - # type omitted on purpose — the runner selects the harness via --type (codex/antigravity); - # coder-eval's baseline default experiment supplies claude-code otherwise. + # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file. permission_mode: acceptEdits - model: claude-sonnet-5 allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | You are a coding agent. Do not access files in sibling runs/* directories. Everywhere else is permitted. diff --git a/tests/experiments/nightly.yaml b/tests/experiments/nightly.yaml index f55397c79a..755271fe7b 100644 --- a/tests/experiments/nightly.yaml +++ b/tests/experiments/nightly.yaml @@ -31,10 +31,8 @@ defaults: - ~/.uipath:/.uipath:rw agent: - # type omitted on purpose — the runner selects the harness via --type (codex/antigravity); - # coder-eval's baseline default experiment supplies claude-code otherwise. + # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file. permission_mode: acceptEdits - model: claude-sonnet-5 allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | You are a coding agent. Do not access files in sibling runs/* directories. Everywhere else is permitted. diff --git a/tests/experiments/skill-comparison-template.yaml b/tests/experiments/skill-comparison-template.yaml index 0646ab0466..05895c028a 100644 --- a/tests/experiments/skill-comparison-template.yaml +++ b/tests/experiments/skill-comparison-template.yaml @@ -26,10 +26,8 @@ defaults: turn_timeout: 300 repeats: 1 agent: - # type omitted on purpose — the runner selects the harness via --type (codex/antigravity); - # coder-eval's baseline default experiment supplies claude-code otherwise. + # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file. permission_mode: bypassPermissions - model: claude-sonnet-5 allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | You are a coding agent. Work only inside your current working directory. diff --git a/tests/experiments/smoke-windows.yaml b/tests/experiments/smoke-windows.yaml index 7ebdeb6485..a67df4ec33 100644 --- a/tests/experiments/smoke-windows.yaml +++ b/tests/experiments/smoke-windows.yaml @@ -15,10 +15,8 @@ defaults: driver: tempdir agent: - # type omitted on purpose — the runner selects the harness via --type (codex/antigravity); - # coder-eval's baseline default experiment supplies claude-code otherwise. + # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file. permission_mode: acceptEdits - model: claude-sonnet-5 allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | You are a coding agent. Do not access files in sibling runs/* directories. Everywhere else is permitted. diff --git a/tests/experiments/smoke.yaml b/tests/experiments/smoke.yaml index c9a21d0fbf..844d82bfa0 100644 --- a/tests/experiments/smoke.yaml +++ b/tests/experiments/smoke.yaml @@ -31,10 +31,8 @@ defaults: - ~/.uipath:/.uipath:rw agent: - # type omitted on purpose — the runner selects the harness via --type (codex/antigravity); - # coder-eval's baseline default experiment supplies claude-code otherwise. + # No agent.type / agent.model here: the runner passes both as CLI flags, which outrank this file. permission_mode: acceptEdits - model: claude-sonnet-5 allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] system_prompt: | You are a coding agent. Do not access files in sibling runs/* directories. Everywhere else is permitted. diff --git a/tests/scripts/activation_gate.py b/tests/scripts/activation_gate.py index 5ee2c4a281..33dcf84011 100755 --- a/tests/scripts/activation_gate.py +++ b/tests/scripts/activation_gate.py @@ -12,11 +12,21 @@ import argparse import json +import os import subprocess import sys import tempfile from pathlib import Path +# The gate's agent model. Passed explicitly rather than pinned in +# experiments/activation.yaml so one variable moves every eval entry point in this +# repo, and so the model the baselines below were measured against is visible at the +# call site. No default: the baselines are model-specific, so silently gating a +# different model than the one they were measured on would report a meaningless +# verdict. NOT $BEDROCK_MODEL: that is the evaluation-side model (llm_judge + the +# simulated user), which must not move with the agent under test. +AGENT_MODEL = os.environ.get("AGENT_MODEL", "").strip() + # Rounded recall.yes baseline (in %) per skill, measured 2026-06-17 over each # skill's FULL positive set on claude-sonnet-4-6 via Bedrock at max_turns: 1 — # the same model and full-set measurement the gate itself runs. The gate task @@ -103,6 +113,10 @@ def main() -> int: parser.add_argument("--skill", required=True) skill = parser.parse_args().skill + if not AGENT_MODEL: + print("ERROR: AGENT_MODEL is unset — set the CLAUDE_CODE_MODEL repo variable", file=sys.stderr) + return 2 + if skill not in BASELINES_PCT: print(f"SKIP: no baseline for {skill!r}", file=sys.stderr) return 0 @@ -127,6 +141,7 @@ def main() -> int: [ "coder-eval", "run", str(task_yaml), "-e", "tests/experiments/activation.yaml", + "--model", AGENT_MODEL, "-j", "4", "--run-dir", str(run_dir), ],