From 7d7918ed441c5aca92773dac56fdaab112b32fd3 Mon Sep 17 00:00:00 2001 From: samiamorwas Date: Thu, 20 Aug 2026 18:40:53 -0400 Subject: [PATCH] feat(run): drive every dispatch from the runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only tasks declaring scripted `turns` were runner-driven; every one-shot task and every judge was dispatched by a human or an agent pasting a generated `jq`/`xargs` pipeline out of RUNBOOK.md. #244 needs real tasks with a dynamic number of turns, repeated enough times to be statistically meaningful, which is not drivable by hand — and nothing bounded a dispatch, so one hung task hung the campaign. `eval-magic dispatch` now runs the whole plan: eval-magic dispatch [--jobs 4] [--timeout 1800] [--task-index N]… [--overwrite] [--judges] It owns what `xargs` was doing badly. `--jobs` is a bounded thread pool over the plan's private per-task environments. `--timeout` gives each task a deadline and records an overrun as a `timed_out` conversation rather than letting it stall the batch. A failed task is recorded and named while the rest continues, and because a failure writes no `conversation.json`, rerunning the same command retries exactly the failures and skips what finished. `dispatch-task` is removed, folded into `--task-index`. Judges dispatch the same way. A judge is a one-shot task whose prompt happens to be a rubric, so it reuses the harness's own `exec_template` with its placeholders bound differently — the iteration directory, the judge prompt, and a capture directory derived from the response path so several assertions in one condition cannot overwrite each other's transcript. That removes the whole recipe surface: `render_parallel_dispatch_recipe`, `render_judge_dispatch_recipe`, the `parallel_command_template` and `judge_command_template` descriptor fields, their validation, the probe's render-only checks, `POSIX_RECIPE_TOOLS`, and `require_posix_toolchain`. `jq` stops being a requirement anywhere, so `POSIX_TOOLING_REQUIREMENT` and AFTER_HELP now ask only for a POSIX shell. Notable decisions: - Dispatched children get null stdout/stderr rather than inheriting them. Killing the shell at a deadline leaves the harness grandchild holding an inherited pipe, which kept the caller blocked ~5s past a 1s timeout. Every shipped exec_template already redirects both into the outputs directory. - Every task now carries `conversation_path`, so `record_runs` keys its "incomplete conversation" skip on `turns` instead. The flat one-shot transcript path stays as a fallback; slowdini/eval-magic#266 records what removing it involves. - One-shot transcripts move to `outputs/turn-1/`, the layout ingest already read for scripted rounds. Schema: `conversation.schema.json` gains `timed_out` and `timed_out_in_round`, and relaxes `events.minItems` to 1 for a task that timed out before its first answer. `harness-descriptor.schema.json` drops the two removed template fields. Verified with cargo fmt --check, cargo build, cargo clippy --all-targets --all-features -D warnings, and cargo test --all-targets (1202 passed) run under EVAL_MAGIC_REQUIRE_POSIX_TOOLS=1 so no test skips, plus a manual run → dispatch → ingest → dispatch --judges → finalize → teardown against a stub harness. Closes #256 Co-Authored-By: Claude Opus 5 --- AGENTS.md | 29 +- README.md | 29 +- docs/claude-notes.md | 8 +- docs/cline-notes.md | 3 +- docs/developer_overview.md | 41 +- docs/guides/byoh.md | 6 +- docs/guides/isolation.md | 4 +- docs/opencode-notes.md | 2 +- docs/progressive-enhancements.md | 61 ++- harnesses/claude-code.toml | 24 +- harnesses/cline.toml | 26 +- harnesses/codex.toml | 25 +- harnesses/opencode.toml | 24 +- harnesses/template.toml | 19 +- profiles/shared/runbook.md | 24 +- schema/conversation.schema.json | 39 +- schema/harness-descriptor.schema.json | 10 +- src/adapters/cli_command.rs | 405 +----------------- src/adapters/descriptor.rs | 6 - src/adapters/descriptor/validation.rs | 66 +-- .../descriptor/validation/tests/dispatch.rs | 34 -- src/adapters/descriptor_adapter.rs | 198 ++------- src/adapters/harness.rs | 47 +- src/adapters/mod.rs | 2 +- src/cli/args.rs | 109 +++-- src/cli/commands/fixture.rs | 25 ++ src/cli/commands/harness.rs | 2 +- src/cli/commands/harness/probe.rs | 245 +---------- src/cli/commands/mod.rs | 2 +- src/cli/commands/pipeline.rs | 25 +- src/cli/commands/run.rs | 87 +++- src/cli/help.rs | 18 +- src/cli/mod.rs | 4 +- src/cli/run/conversation.rs | 307 ++++++++----- src/cli/run/dispatch.rs | 113 +++-- src/cli/run/dispatch/tests/conversation.rs | 96 ++--- src/cli/run/drive.rs | 280 ++++++++++++ src/cli/run/drive/judges.rs | 199 +++++++++ src/cli/run/golden_tests.rs | 122 ++---- src/cli/run/mod.rs | 3 +- src/cli/run/orchestrate/build.rs | 9 +- src/cli/run/orchestrate/mod.rs | 19 +- src/cli/run/orchestrate/shell.rs | 94 ++-- src/cli/run/runbook.rs | 122 ++---- src/cli/run/util.rs | 6 +- src/core/mod.rs | 4 +- src/core/runtime.rs | 251 ++++++++--- src/core/types.rs | 9 +- src/pipeline/grade/transcript_check.rs | 1 + src/pipeline/record_runs.rs | 14 +- tests/cli/basics.rs | 14 +- tests/cli/docs.rs | 21 +- tests/cli/grade_models.rs | 6 +- tests/cli/harness.rs | 30 -- .../golden/claude-code/judge-recipe.golden.md | 37 -- .../claude-code/manifest-nomodel.golden.md | 127 ------ tests/golden/claude-code/manifest.golden.md | 43 +- tests/golden/claude-code/runbook.golden.md | 66 +-- tests/golden/cline/judge-recipe.golden.md | 37 -- tests/golden/cline/manifest.golden.md | 45 +- .../golden/cline/next-steps-model.golden.txt | 11 - .../cline/next-steps-nomodel.golden.txt | 11 - tests/golden/cline/next-steps.golden.txt | 3 + tests/golden/cline/runbook.golden.md | 68 +-- .../codex/judge-recipe-noguard.golden.md | 37 -- tests/golden/codex/judge-recipe.golden.md | 37 -- tests/golden/codex/manifest-noguard.golden.md | 129 ------ tests/golden/codex/manifest.golden.md | 44 +- tests/golden/codex/runbook.golden.md | 67 +-- tests/golden/opencode/judge-recipe.golden.md | 37 -- tests/golden/opencode/manifest.golden.md | 43 +- .../opencode/next-steps-model.golden.txt | 9 - .../opencode/next-steps-nomodel.golden.txt | 9 - tests/golden/opencode/next-steps.golden.txt | 3 + tests/golden/opencode/runbook.golden.md | 66 +-- tests/run/agent_env.rs | 35 +- tests/run/byoh.rs | 24 +- tests/run/claude_cli.rs | 28 +- tests/run/codex.rs | 41 +- tests/run/codex_guard.rs | 61 +-- tests/run/conversation.rs | 226 +++++++++- tests/run/conversation/dispatch.rs | 401 +++++++++++++++++ tests/run/judges.rs | 313 ++++++++++++++ tests/run/main.rs | 1 + tests/run/runbook.rs | 74 +++- 85 files changed, 2762 insertions(+), 2740 deletions(-) create mode 100644 src/cli/run/drive.rs create mode 100644 src/cli/run/drive/judges.rs delete mode 100644 tests/golden/claude-code/judge-recipe.golden.md delete mode 100644 tests/golden/claude-code/manifest-nomodel.golden.md delete mode 100644 tests/golden/cline/judge-recipe.golden.md delete mode 100644 tests/golden/cline/next-steps-model.golden.txt delete mode 100644 tests/golden/cline/next-steps-nomodel.golden.txt create mode 100644 tests/golden/cline/next-steps.golden.txt delete mode 100644 tests/golden/codex/judge-recipe-noguard.golden.md delete mode 100644 tests/golden/codex/judge-recipe.golden.md delete mode 100644 tests/golden/codex/manifest-noguard.golden.md delete mode 100644 tests/golden/opencode/judge-recipe.golden.md delete mode 100644 tests/golden/opencode/next-steps-model.golden.txt delete mode 100644 tests/golden/opencode/next-steps-nomodel.golden.txt create mode 100644 tests/golden/opencode/next-steps.golden.txt create mode 100644 tests/run/conversation/dispatch.rs create mode 100644 tests/run/judges.rs diff --git a/AGENTS.md b/AGENTS.md index b2d4b94..c92f4d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,24 +63,23 @@ binary, `cargo test --lib` alone does not build it — run `cargo test`, or `car compilation and clippy on the other host and hides the coverage gap. Instead, probe for what the test actually needs and call `report_skip` (`src/core/runtime.rs`), which prints the reason and returns `true`. Setting `EVAL_MAGIC_REQUIRE_POSIX_TOOLS=1` turns every skip into a failure; CI sets -it on both runners, so neither can quietly stop covering something. Three capabilities are gated -today: the recipe tools beyond the shell itself (`require_posix_toolchain` — in practice `jq`), -symlink creation, which Windows allows only under Developer Mode, and creating a path past +it on both runners, so neither can quietly stop covering something. Two capabilities are gated +today: symlink creation, which Windows allows only under Developer Mode, and creating a path past Windows' 259-character limit (`deep_task_root`, `src/cli/run/orchestrate/git.rs`). The Windows -runner is provisioned for those rather than exempted from them, so a skip there is a red build. The -shell is not one of them; it is a hard requirement, per the section below. -`require_posix_toolchain` is not test-only either — the `run` preflight uses it to warn about the -same gap. Where a genuine per-OS difference is the behavior under test — signals, path separators — -branch on `cfg!(windows)` at runtime so both arms still compile everywhere. +runner is provisioned for both rather than exempted from them, so a skip there is a red build. The +shell is not one of them; it is a hard requirement, per the section below. Where a genuine per-OS +difference is the behavior under test — signals, path separators — branch on `cfg!(windows)` at +runtime so both arms still compile everywhere. **A POSIX shell is required, for use and for development.** Harness `exec_template`s are POSIX -command lines, so the dispatch and probe paths spawn `sh` via `posix_shell()` -(`src/core/runtime.rs`) rather than a hardcoded `/bin/sh`: it searches `PATH`, then a Git for -Windows install. Set `EVAL_MAGIC_SH` to override it. `cargo test` inherits the requirement — the -scripted-turn tests spawn a `#!/bin/sh` harness stub through the resolved shell and do not skip — -so a host without `sh` fails the suite instead of quietly covering less. `jq` is required alongside -it for the parallel-dispatch and judge recipes; Git for Windows supplies the shell, `xargs`, `tr`, -and `wc`, but not `jq`. `POSIX_TOOLING_REQUIREMENT` (`src/core/runtime.rs`) is the one wording the +command lines, so the dispatch and probe paths spawn `sh` through `run_in_posix_shell` / +`posix_shell()` (`src/core/runtime.rs`) rather than a hardcoded `/bin/sh`: it searches `PATH`, then +a Git for Windows install. Set `EVAL_MAGIC_SH` to override it. `cargo test` inherits the +requirement — the dispatch tests spawn a `#!/bin/sh` harness stub through the resolved shell and do +not skip — so a host without `sh` fails the suite instead of quietly covering less. The shell is +the whole requirement: `jq` was needed only while operators pasted the generated dispatch and judge +recipes, and `eval-magic dispatch` drives both itself. +`POSIX_TOOLING_REQUIREMENT` (`src/core/runtime.rs`) is the one wording the Markdown-carrying surfaces reuse: the shell-discovery errors, the `run` preflight warnings, `RUNBOOK.md`, and `dispatch-manifest.md`. State the requirement from there rather than rephrasing it. `--help` is the one deliberate restatement (`AFTER_HELP` in `src/cli/help.rs`), hard-wrapped and diff --git a/README.md b/README.md index 65b8759..eef601b 100644 --- a/README.md +++ b/README.md @@ -28,20 +28,19 @@ eval-magic runs the same task in two controlled conditions—such as a new skill versus no skill, or an edited skill versus its previous version—and grades both results against shared assertions. It -builds isolated task workspaces, stages skills, generates harness-specific dispatch instructions, -ingests transcripts and final state, and produces comparison artifacts. You dispatch the agent -sessions with Claude Code, Cline, Codex, OpenCode, or a descriptor-backed harness of your own. +builds isolated task workspaces, stages skills, dispatches the agent sessions itself, ingests +transcripts and final state, and produces comparison artifacts. It drives Claude Code, Cline, +Codex, OpenCode, or a descriptor-backed harness of your own. The installed CLI is the primary manual. Start with `eval-magic --help`, and use `eval-magic --help` whenever you reach a new phase. ## Install -Git is required at runtime, plus a POSIX shell with `jq`: the dispatch and judge recipes eval-magic -generates are POSIX command lines built on `jq`, `xargs`, `tr`, and `wc`. The shell that runs them -has to resolve the same paths the workspace was prepared with. On Windows that is Git Bash (Git for -Windows), with `jq` installed separately — Git for Windows does not bundle it. WSL resolves a -different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. +Git is required at runtime, plus a POSIX shell: harness dispatch commands are POSIX command lines, +and `eval-magic dispatch` runs them itself, so the host it runs on needs a shell that resolves the +workspace's own paths. On Windows that is Git Bash (Git for Windows). WSL resolves a different +filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set `EVAL_MAGIC_SH` to select a specific `sh`. Windows support runs through Git Bash and is deprecated: a future release will require WSL. @@ -101,10 +100,10 @@ eval-magic run --harness codex eval-magic run --harness opencode ``` -`run` prepares the campaign; it does not dispatch agents. Review the printed task and model-usage -summary before continuing. Then read the generated `RUNBOOK.md` from beginning to end. It contains -the exact dispatch, ingest, judge, finalize, and `eval-magic teardown` commands for that campaign -and harness. +`run` prepares the campaign; `eval-magic dispatch` runs it. Review the printed task and model-usage +summary before continuing — dispatch is where model usage is spent. Then read the generated +`RUNBOOK.md` from beginning to end. It contains the exact dispatch, ingest, judge, finalize, and +`eval-magic teardown` commands for that campaign and harness. After finalization, open the generated `benchmark.json` to compare pass rates, token and duration measurements, and validity warnings. Use `eval-magic aggregate --help` when you need to combine @@ -152,9 +151,9 @@ Issues and planned work are tracked in the ## Development -Development carries the same host requirement as use: a POSIX shell with `jq`. The scripted-turn -tests spawn `#!/bin/sh` harness stubs through the resolved shell and do not skip, so the suite -cannot pass without one. Tests that need `jq` or symlink creation report a skip instead. +Development carries the same host requirement as use: a POSIX shell. The dispatch tests spawn +`#!/bin/sh` harness stubs through the resolved shell and do not skip, so the suite cannot pass +without one. Tests that need symlink creation report a skip instead. ```bash cargo fmt --check diff --git a/docs/claude-notes.md b/docs/claude-notes.md index e606af9..3bb0728 100644 --- a/docs/claude-notes.md +++ b/docs/claude-notes.md @@ -37,9 +37,9 @@ hook-entry and `hookSpecificOutput` verdict templates) is rendered by the generi ## Permission mode -Every dispatch and judge recipe carries `--permission-mode bypassPermissions`. The obvious -alternative, `acceptEdits`, is wrong here: it auto-approves *file edits* but **not Bash**, and -because the recipe detaches stdin (`/skills` are unloaded too. Verified 2026-08-06 by A/B within one campaign — -the judge recipe carries no `--setting-sources` and its capture lists both `~/.claude/skills` +the judge dispatch carries no `--setting-sources` and its capture lists both `~/.claude/skills` entries and every `:` id, while all 48 isolated eval dispatches list neither. Project-local staged skills are independent of installed plugins, so they still load and the diff --git a/docs/cline-notes.md b/docs/cline-notes.md index 56e0405..f42224c 100644 --- a/docs/cline-notes.md +++ b/docs/cline-notes.md @@ -68,8 +68,7 @@ the descriptor references. "Probe capture" refers to the observed dispatches des | Plugin hook contract | `beforeTool({snapshot, tool, toolCall, input})`; block with `{skip: true, reason}`; 3000ms default hook budget (plugin spawns with a 2s timeout so a hung arbiter fails open); `spawnSync` works from the plugin sandbox | 3.0.53 spike capture + the binary's runtime hook loop; the docs' `tool_call_before`/`fail_closed` vocabulary lags the binary | | `shadow.preflight` | `cline-skills` | 3.0.53 root probe (one uniquely-named skill per candidate root): dispatch cwd's `.cline/skills` read, ancestor's NOT (no project walk), `~/.agents/skills` IS read (and receives `cline skill install` global installs); `$CLINE_DIR` overrides the `~/.cline` default (3.0.53 binary) | | `dispatch.capture_prefix` | `cline` | chosen name (judge capture files `$response_base.cline-events.jsonl`) | -| `dispatch.exec_template` / `parallel_command_template` | see descriptor | flags from `cline --help` (`--act` from the 3.0.52 binary’s hidden option registration + behavioral write test); `--json` NDJSON stdout and `` report the built-in and only the capabilities its descriptor actually declares. -5. Wire enhancements in leverage order — dispatch recipes and transcript ingest first (they carry +5. Wire enhancements in leverage order — the dispatch command and transcript ingest first (they carry the most fidelity and are prerequisites for conversation resume), then conversation resume, staging, model flag, guard (guard requires built-in status — user descriptors may not declare one). Most enhancements are diff --git a/harnesses/claude-code.toml b/harnesses/claude-code.toml index 00770ec..15562cf 100644 --- a/harnesses/claude-code.toml +++ b/harnesses/claude-code.toml @@ -75,9 +75,8 @@ preflight = "claude-plugins" capture_prefix = "claude" next_steps_template = ''' -Next: iterate the tasks[] array in dispatch.json and dispatch each task (from the env dir — `claude` has no --cd flag) with: -{exec_command} -Then run `ingest{target_args} --iteration {iteration} --harness claude-code`.''' +Next: eval-magic dispatch{target_args} --iteration {iteration} --harness claude-code +Then run `eval-magic ingest{target_args} --iteration {iteration} --harness claude-code`.''' # `--output-format stream-json` requires `--verbose` in -p mode; there is no # --cd flag, so the dispatch runs from the env dir; and there is no # --output-last-message, so the final message is recovered from the stream-json @@ -97,29 +96,16 @@ cd && claude -p --output-format stream-json --verbose --permission-m /claude-events.jsonl \ 2> /claude-stderr.log''' -parallel_command_template = ''' - cd "$eval_root" && claude -p --output-format stream-json --verbose --permission-mode bypassPermissions{model_arg} \ - "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response your closing summary." \ - "$outputs_dir/claude-events.jsonl" \ - 2> "$outputs_dir/claude-stderr.log"''' -judge_command_template = ' cd "{cwd}" && claude -p --output-format stream-json --verbose --permission-mode bypassPermissions $model_arg \' manifest_template = ''' -After all dispatches (Claude Code): +Harness dispatch (Claude Code): -Run one fresh `claude -p` per task from the env dir (`cd ` — `claude` has no --cd flag). `--output-format stream-json` requires `--verbose`; detach stdin with `` — `claude` has no --cd flag). `--output-format stream-json` requires `--verbose`; detach stdin with `/claude-events.jsonl` and stderr as `outputs/turn-/claude-stderr.log`. ```bash {exec_command} ``` -Parallel dispatch from this iteration directory: - -```bash -{parallel_recipe} -``` - -Then run `eval-magic ingest --harness claude-code`; ingest reads each task's `outputs/claude-events.jsonl`. +Then run `eval-magic ingest --harness claude-code`; ingest reads each task's `outputs/turn-/claude-events.jsonl`. ''' [conversation] diff --git a/harnesses/cline.toml b/harnesses/cline.toml index c8043cd..d170810 100644 --- a/harnesses/cline.toml +++ b/harnesses/cline.toml @@ -126,9 +126,8 @@ preflight = "cline-skills" capture_prefix = "cline" next_steps_template = ''' -Next: iterate the tasks[] array in dispatch.json and dispatch each task with: -{exec_command} -Then run `ingest{target_args} --iteration {iteration} --harness cline`.''' +Next: eval-magic dispatch{target_args} --iteration {iteration} --harness cline +Then run `eval-magic ingest{target_args} --iteration {iteration} --harness cline`.''' exec_template = ''' cline --cwd --act --json --auto-approve true{model_arg} \ "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ @@ -137,29 +136,14 @@ cline --cwd --act --json --auto-approve true{model_arg} \ 2> /cline-stderr.log; \ jq -rj 'select(.type == "run_result") | .text' /cline-events.jsonl \ > /final-message.md''' -parallel_command_template = ''' - cline --cwd "$eval_root" --act --json --auto-approve true{model_arg} \ - "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response your closing summary." \ - "$outputs_dir/cline-events.jsonl" \ - 2> "$outputs_dir/cline-stderr.log"; \ - jq -rj "select(.type == \"run_result\") | .text" "$outputs_dir/cline-events.jsonl" \ - > "$outputs_dir/final-message.md"''' -judge_command_template = ' cline --cwd "{cwd}" --act --json --auto-approve true $model_arg \' manifest_template = ''' -After all dispatches (Cline): +Harness dispatch (Cline): -Run one fresh `cline --cwd --act --json --auto-approve true` per task. Detach stdin with `` so piped task data cannot become extra prompt context; capture stdout as `outputs/cline-events.jsonl` and stderr as `outputs/cline-stderr.log`. The trailing jq step recovers `outputs/final-message.md` from the terminal `run_result` event. +`eval-magic dispatch` runs one fresh `cline --cwd --act --json --auto-approve true` per task. Detach stdin with `` so piped task data cannot become extra prompt context; capture stdout as `outputs/turn-/cline-events.jsonl` and stderr as `outputs/turn-/cline-stderr.log`. `eval-magic dispatch` writes `outputs/final-message.md` itself from the parsed transcript; the template's trailing jq step is a belt-and-braces copy of the terminal `run_result` event. ```bash {exec_command} ``` -Parallel dispatch from this iteration directory: - -```bash -{parallel_recipe} -``` - -Then run `eval-magic ingest --harness cline`; ingest reads each task's `outputs/cline-events.jsonl`. +Then run `eval-magic ingest --harness cline`; ingest reads each task's `outputs/turn-/cline-events.jsonl`. ''' diff --git a/harnesses/codex.toml b/harnesses/codex.toml index d9978b0..adf8630 100644 --- a/harnesses/codex.toml +++ b/harnesses/codex.toml @@ -103,9 +103,8 @@ capture_prefix = "codex" guard_args = " --dangerously-bypass-hook-trust" next_steps_template = ''' -Next: iterate the tasks[] array in dispatch.json and dispatch each task with: -{exec_command} -Then run `ingest{target_args} --iteration {iteration} --harness codex`.''' +Next: eval-magic dispatch{target_args} --iteration {iteration} --harness codex +Then run `eval-magic ingest{target_args} --iteration {iteration} --harness codex`.''' # Stdin is detached so a surrounding `xargs`/pipe cannot be treated as extra # prompt context. exec_template = ''' @@ -115,30 +114,16 @@ codex --ask-for-approval never exec --cd --sandbox workspace-write{g /codex-events.jsonl \ 2> /codex-stderr.log''' -parallel_command_template = ''' - codex --ask-for-approval never exec --cd "$eval_root" --sandbox workspace-write{guard_args}{model_arg} --json \ - --output-last-message "$outputs_dir/final-message.md" \ - "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to $outputs_dir/final-message.md." \ - "$outputs_dir/codex-events.jsonl" \ - 2> "$outputs_dir/codex-stderr.log"''' -judge_command_template = ' codex --ask-for-approval never exec --cd "{cwd}" --sandbox workspace-write{guard_args} $model_arg --json \' manifest_template = ''' -After all dispatches (Codex): +Harness dispatch (Codex): -Run one fresh `codex --ask-for-approval never exec --json` per task. Detach stdin with `/codex-events.jsonl` and stderr as `outputs/turn-/codex-stderr.log`. ```bash {exec_command} ``` -Parallel dispatch from this iteration directory: - -```bash -{parallel_recipe} -``` - -Then run `eval-magic ingest --harness codex`; Codex transcript ingest reads each task's `outputs/codex-events.jsonl`. +Then run `eval-magic ingest --harness codex`; Codex transcript ingest reads each task's `outputs/turn-/codex-events.jsonl`. ''' [conversation] diff --git a/harnesses/opencode.toml b/harnesses/opencode.toml index d28e4fb..611d480 100644 --- a/harnesses/opencode.toml +++ b/harnesses/opencode.toml @@ -92,38 +92,24 @@ preflight = "opencode-skills" capture_prefix = "opencode" next_steps_template = ''' -Next: iterate the tasks[] array in dispatch.json and dispatch each task with: -{exec_command} -Then run `ingest{target_args} --iteration {iteration} --harness opencode`.''' +Next: eval-magic dispatch{target_args} --iteration {iteration} --harness opencode +Then run `eval-magic ingest{target_args} --iteration {iteration} --harness opencode`.''' exec_template = ''' opencode run --dir --format json --auto{model_arg} \ "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ /opencode-events.jsonl \ 2> /opencode-stderr.log''' -parallel_command_template = ''' - opencode run --dir "$eval_root" --format json --auto{model_arg} \ - "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response your closing summary." \ - "$outputs_dir/opencode-events.jsonl" \ - 2> "$outputs_dir/opencode-stderr.log"''' -judge_command_template = ' opencode run --dir "{cwd}" --format json --auto $model_arg \' manifest_template = ''' -After all dispatches (OpenCode): +Harness dispatch (OpenCode): -Run one fresh `opencode run --format json --auto` per task. Detach stdin with `/opencode-events.jsonl` and stderr as `outputs/turn-/opencode-stderr.log`. ```bash {exec_command} ``` -Parallel dispatch from this iteration directory: - -```bash -{parallel_recipe} -``` - -Then run `eval-magic ingest --harness opencode`; OpenCode transcript ingest reads each task's `outputs/opencode-events.jsonl`. +Then run `eval-magic ingest --harness opencode`; OpenCode transcript ingest reads each task's `outputs/turn-/opencode-events.jsonl`. ''' [conversation] diff --git a/harnesses/template.toml b/harnesses/template.toml index d4a96f9..221f0e8 100644 --- a/harnesses/template.toml +++ b/harnesses/template.toml @@ -49,26 +49,17 @@ label = "{label}" # [dispatch] # exec_template = '{label} run --cd {model_arg} "Read the file at and follow its instructions exactly." > /final-message.md' -## capture_prefix names the judge recipe's per-task capture files -## ($response_base.-events.jsonl); required by judge_command_template. +## capture_prefix names this harness's transcript file (-events.jsonl), which +## `eval-magic dispatch` reads back after each round. # capture_prefix = "{label}" -## parallel_command_template is the per-task command block spliced into the shared parallel -## dispatch scaffold. -# parallel_command_template = 'cd "$eval_root" && {label} run{model_arg} "Read the file at $prompt_path and follow its instructions exactly." > "$outputs_dir/final-message.md"' - -## judge_command_template splices into the shared judge recipe. Contract: requires [model].flag -## and capture_prefix, must reference $model_arg and {cwd}, and must end with a shell line -## continuation (" \") so the recipe's prompt line follows it. -# judge_command_template = 'cd "{cwd}" && {label} run $model_arg \' - ## next_steps_template is the post-run handoff text ({exec_command}, {target_args}, {iteration}, ## {model_note} placeholders; referencing {model_note} requires the model_note field). # next_steps_template = "\nNext: dispatch each task in dispatch.json with:\n{exec_command}\nThen run `ingest --harness {label}`." -## manifest_template is this harness's dispatch-manifest section ({exec_command} and -## {parallel_recipe} placeholders); it must end with exactly one trailing newline. -# manifest_template = "After all dispatches ({label}):\n\n{exec_command}\n\nParallel dispatch from the iteration directory:\n\n{parallel_recipe}\n" +## manifest_template is this harness's dispatch-manifest section ({exec_command} +## placeholder); it must end with exactly one trailing newline. +# manifest_template = "Harness dispatch ({label}):\n\n{exec_command}\n" ## guard_args (guard-only args spliced at {guard_args}) and model_note (sentence spliced at ## {model_note}) are niche — see the schema; guard_args only matters to guarded built-ins. diff --git a/profiles/shared/runbook.md b/profiles/shared/runbook.md index 8a48504..450ca56 100644 --- a/profiles/shared/runbook.md +++ b/profiles/shared/runbook.md @@ -11,7 +11,21 @@ repo. - **Dispatches:** {{NUM_TASKS}} (the `tasks[]` array in `{{DISPATCH_JSON}}`) ## 1. Dispatch the eval agents, then ingest -{{DISPATCH_RECIPE}} + +``` +{{DISPATCH_CMD}} +``` + +`dispatch` runs every task in its own private environment, `--jobs` of them at a time, and writes +each task's `conversation.json`. A task that already has one is skipped, so rerunning the same +command retries only what did not finish. A task that exceeds `--timeout` is recorded as timed out +rather than left to stall the campaign, and a task that fails is recorded and named while the rest +of the batch continues. A conversation that stops at a scripted gate is valid eval data, not a +failure. + +``` +{{INGEST_CMD}} +``` `ingest` records each run, backfills transcripts, scans for stray writes, collects guarded-task blocks into `guard-denials.json`, and grades every mechanical assertion. Inspect any denial @@ -19,7 +33,13 @@ warning before trusting the affected task. It then prints any `llm_judge` tasks grade itself. ## 2. Dispatch the judge agents, then finalize -{{JUDGE_RECIPE}} + +``` +{{JUDGE_CMD}} +``` + +Verdicts that are already present are skipped; the summary prints `N/M verdicts present` and exits +nonzero until every task has one, so rerun the same command to fill the gaps. Then merge the verdicts and aggregate: diff --git a/schema/conversation.schema.json b/schema/conversation.schema.json index 9efc2e6..ef68fde 100644 --- a/schema/conversation.schema.json +++ b/schema/conversation.schema.json @@ -1,15 +1,15 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://slow-powers.dev/schemas/conversation.schema.json", - "title": "Scripted Conversation Completion", - "description": "Runner-owned completion artifact for one scripted multi-turn eval task.", + "title": "Task Conversation Completion", + "description": "Runner-owned completion artifact for one dispatched eval task.", "type": "object", "required": ["status", "delivered_followups", "events"], "additionalProperties": false, "properties": { "status": { "type": "string", - "enum": ["completed", "stopped"] + "enum": ["completed", "stopped", "timed_out"] }, "delivered_followups": { "type": "integer", @@ -23,9 +23,14 @@ "type": "integer", "minimum": 1 }, + "timed_out_in_round": { + "type": "integer", + "minimum": 1, + "description": "The round the dispatch was killed in, when it outran its per-task deadline." + }, "events": { "type": "array", - "minItems": 2, + "minItems": 1, "items": { "oneOf": [ { "$ref": "#/definitions/userMessage" }, @@ -42,7 +47,8 @@ "required": ["status"] }, "then": { - "required": ["stop_reason", "stopped_before_followup"] + "required": ["stop_reason", "stopped_before_followup"], + "not": { "required": ["timed_out_in_round"] } } }, { @@ -51,6 +57,22 @@ "required": ["status"] }, "then": { + "not": { + "anyOf": [ + { "required": ["stop_reason"] }, + { "required": ["stopped_before_followup"] }, + { "required": ["timed_out_in_round"] } + ] + } + } + }, + { + "if": { + "properties": { "status": { "const": "timed_out" } }, + "required": ["status"] + }, + "then": { + "required": ["timed_out_in_round"], "not": { "anyOf": [ { "required": ["stop_reason"] }, @@ -58,6 +80,13 @@ ] } } + }, + { + "if": { + "properties": { "status": { "enum": ["completed", "stopped"] } }, + "required": ["status"] + }, + "then": { "properties": { "events": { "minItems": 2 } } } } ], "definitions": { diff --git a/schema/harness-descriptor.schema.json b/schema/harness-descriptor.schema.json index c2ec632..6de0a99 100644 --- a/schema/harness-descriptor.schema.json +++ b/schema/harness-descriptor.schema.json @@ -370,17 +370,9 @@ "type": "string", "description": "Copy/pasteable single-dispatch command; -style angle placeholders are prose for the human." }, - "parallel_command_template": { - "type": "string", - "description": "Per-task command block spliced into the shared jq/xargs parallel scaffold." - }, - "judge_command_template": { - "type": "string", - "description": "Judge command line spliced into the shared judge recipe; must reference $model_arg and {cwd} and end with a shell line continuation." - }, "manifest_template": { "type": "string", - "description": "The dispatch-manifest harness section; {exec_command}/{parallel_recipe} placeholders; ends with exactly one newline." + "description": "The dispatch-manifest harness section; {exec_command} placeholder; ends with exactly one newline." } } }, diff --git a/src/adapters/cli_command.rs b/src/adapters/cli_command.rs index 41e99d6..7fcfac4 100644 --- a/src/adapters/cli_command.rs +++ b/src/adapters/cli_command.rs @@ -56,205 +56,11 @@ pub(crate) fn render_cli_model_arg(flag: Option<&str>, model: Option<&str>) -> S format!(" {flag} {}", shell_quote_arg(model)) } -/// Render the shared parallel-dispatch recipe: the jq/xargs scaffold over -/// `dispatch.json` tasks with the harness's per-task command block spliced in. -/// -/// `command_block` is the (possibly multi-line) command run per task inside -/// the `sh -c` body; it references `$eval_root` / `$prompt_path` / -/// `$outputs_dir`. The three values travel as separate NUL-delimited arguments -/// so BSD xargs does not apply its 255-byte `-I` replacement limit. -/// -/// The separators come from `tr`, not from a `\u0000` escape inside the jq -/// program: an escape only works if it reaches jq unresolved, and a tool that -/// materialises this recipe writes real NUL bytes into the program text -/// instead, where they do not survive argument passing — jq then emits no -/// separators and `xargs -0` collapses every field into one bogus dispatch -/// that exits 0. Paths containing a newline remain unsupported, as before. -/// -/// `tr -d '\r'` runs before that: jq's native Windows build writes CRLF, and -/// `tr '\n' '\0'` converts only the newline, so without it every path reaches -/// the dispatch with a carriage return on the end. -pub(crate) fn render_parallel_dispatch_recipe( - command_block: &str, - one_shot_only: bool, - environment: &BTreeMap, -) -> String { - let tasks = if one_shot_only { - ".tasks[] | select(.turns == null)" - } else { - ".tasks[]" - }; - let mut lines = vec![ - "JOBS=${JOBS:-4}".to_string(), - format!( - "jq -r '{tasks} | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \\" - ), - " | tr -d '\\r' \\".to_string(), - " | tr '\\n' '\\0' \\".to_string(), - " | xargs -0 -P \"$JOBS\" -n 3 sh -c '".to_string(), - " eval_root=\"$1\"".to_string(), - " prompt_path=\"$2\"".to_string(), - " outputs_dir=\"$3\"".to_string(), - " mkdir -p \"$outputs_dir\"".to_string(), - format!(" {}", git_environment_prelude()), - ]; - lines.extend( - environment - .iter() - .map(|(name, value)| format!(" export {name}={}", shell_quote_arg(value))), - ); - lines.extend([command_block.to_string(), " ' sh".to_string()]); - lines.join("\n") -} - -/// Render the shared judge-dispatch recipe: the jq/xargs scaffold over -/// `judge-tasks.json` with the harness command line spliced in. -/// -/// `command_line` must reference `$model_arg` (empty when the task declares no -/// model, ` ` otherwise) and end with ` \`; `model_flag` fills the -/// `model_arg` assignment; `capture_prefix` names the per-task -/// `$response_base.-events.jsonl` / `.-stderr.log` captures. -/// -/// Every jq call is piped through `tr -d '\r'`: jq's native Windows build -/// writes CRLF, and none of the three readers here drop it — `read -r` keeps a -/// carriage return by definition, `tr '\n' '\0'` converts only the newline, and -/// `[ "$judge_present" -eq "$judge_total" ]` needs a bare integer. -pub(crate) fn render_judge_dispatch_recipe( - command_line: &str, - model_flag: &str, - capture_prefix: &str, -) -> String { - [ - "Dispatch each judge task from judge-tasks.json with:".to_string(), - "Existing nonempty response files are skipped; delete one to dispatch that judge again." - .to_string(), - "The final `N/M verdicts present` summary exits nonzero until every task has one." - .to_string(), - String::new(), - "```bash".to_string(), - "JOBS=${JOBS:-4}".to_string(), - "jq -r '.tasks[] | .dispatch_prompt_path, .response_path, (\"model=\" + (.model // \"\"))' judge-tasks.json \\".to_string(), - " | tr -d '\\r' \\".to_string(), - " | tr '\\n' '\\0' \\".to_string(), - " | xargs -0 -P \"$JOBS\" -n 3 sh -c '".to_string(), - " prompt_path=\"$1\"".to_string(), - " response_path=\"$2\"".to_string(), - " model=\"${3#model=}\"".to_string(), - " if [ -s \"$response_path\" ]; then exit 0; fi".to_string(), - " response_base=\"${response_path%.json}\"".to_string(), - " mkdir -p \"$(dirname \"$response_path\")\"".to_string(), - format!(" model_arg=\"\"; [ -n \"$model\" ] && model_arg=\"{model_flag} $model\""), - command_line.to_string(), - " \"Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers.\" \\".to_string(), - " \"$response_base.{capture_prefix}-events.jsonl\" \\"), - format!(" 2> \"$response_base.{capture_prefix}-stderr.log\""), - " ' sh".to_string(), - "judge_dispatch_status=$?".to_string(), - "judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\\r')".to_string(), - "judge_present=$(".to_string(), - " jq -r '.tasks[].response_path' judge-tasks.json \\".to_string(), - " | tr -d '\\r' \\".to_string(), - " | while IFS= read -r response_path; do".to_string(), - " if [ -s \"$response_path\" ]; then printf '%s\\n' \"$response_path\"; fi" - .to_string(), - " done \\".to_string(), - " | wc -l \\".to_string(), - " | tr -d '[:space:]'".to_string(), - ")".to_string(), - "printf '%s/%s verdicts present\\n' \"$judge_present\" \"$judge_total\"".to_string(), - "[ \"$judge_dispatch_status\" -eq 0 ] && [ \"$judge_present\" -eq \"$judge_total\" ]" - .to_string(), - "```".to_string(), - ] - .join("\n") -} - #[cfg(test)] mod tests { use std::collections::BTreeMap; - use std::fs; - use std::path::Path; - use std::process::{Command, Output}; - - use serde_json::json; - - use super::{ - render_agent_dispatch_command, render_cli_model_arg, render_judge_dispatch_recipe, - render_parallel_dispatch_recipe, shell_quote_arg, - }; - - fn write_judge_tasks(cwd: &Path, response_paths: &[&Path]) { - let tasks = response_paths - .iter() - .enumerate() - .map(|(index, response_path)| { - json!({ - "dispatch_prompt_path": cwd.join(format!("prompt-{index}.txt")), - "response_path": response_path, - "model": null, - }) - }) - .collect::>(); - fs::write( - cwd.join("judge-tasks.json"), - serde_json::to_vec(&json!({ "tasks": tasks })).unwrap(), - ) - .unwrap(); - } - - /// The shell to run a rendered recipe in, or `None` after reporting a skip. - /// - /// These three tests execute shipped POSIX pipeline text, so no portable - /// fixture can stand in for the toolchain — the pipeline *is* the subject. - /// Gating on the capability rather than the OS lets them run on any host - /// that has the tools (including Windows with `jq` installed) and stops them - /// failing inscrutably on a Linux box that happens to lack `jq`. - fn recipe_shell(test: &str) -> Option<&'static Path> { - match crate::core::runtime::require_posix_toolchain(crate::core::POSIX_RECIPE_TOOLS) { - Ok(shell) => Some(shell), - Err(missing) => { - crate::core::runtime::report_skip(test, &missing); - None - } - } - } - - /// A `jq` that ends every line with CRLF, the way jq's native Windows build - /// does with stdout in text mode. A shell function rather than a `PATH` - /// shim: nothing has to be marked executable, so it reads and behaves the - /// same on every host, and only the outer pipeline calls jq — the `xargs` - /// child never does, so it does not need the definition. - const CRLF_JQ: &str = "jq() { command jq \"$@\" | tr -d '\\r' \ - | while IFS= read -r line; do printf '%s\\r\\n' \"$line\"; done; }\n"; - fn run_judge_recipe(shell: &Path, cwd: &Path, command_line: &str) -> Output { - run_judge_recipe_prefixed(shell, cwd, command_line, "") - } - - /// Run the rendered recipe with `preamble` in front of it, so a test can - /// replace a tool the recipe shells out to. - fn run_judge_recipe_prefixed( - shell: &Path, - cwd: &Path, - command_line: &str, - preamble: &str, - ) -> Output { - let recipe = render_judge_dispatch_recipe(command_line, "--model", "judge"); - let program = recipe - .split_once("```bash\n") - .unwrap() - .1 - .strip_suffix("\n```") - .unwrap(); - Command::new(shell) - .arg("-c") - .arg(format!("{preamble}{program}")) - .current_dir(cwd) - .env("JOBS", "1") - .output() - .unwrap() - } + use super::{render_agent_dispatch_command, render_cli_model_arg, shell_quote_arg}; #[test] fn agent_dispatch_environment_is_sorted_and_shell_quoted() { @@ -307,213 +113,4 @@ mod tests { " -m 'gpt 5'" ); } - - #[test] - fn parallel_recipe_batches_three_nul_delimited_arguments_without_replacement() { - let recipe = - render_parallel_dispatch_recipe(" run \"$eval_root\"", false, &BTreeMap::new()); - - assert!(recipe.contains( - "jq -r '.tasks[] | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json" - )); - assert!(recipe.contains("| tr '\\n' '\\0' \\"), "{recipe}"); - assert!( - recipe.contains("| xargs -0 -P \"$JOBS\" -n 3 sh -c '"), - "{recipe}" - ); - assert!(recipe.contains("eval_root=\"$1\""), "{recipe}"); - assert!(recipe.contains("prompt_path=\"$2\""), "{recipe}"); - assert!(recipe.contains("outputs_dir=\"$3\""), "{recipe}"); - assert!(!recipe.contains("-I{}"), "{recipe}"); - assert!(!recipe.contains("cut -f"), "{recipe}"); - } - - /// The separators must be produced by `tr`, never spelled as an escape - /// inside the jq program: a tool that materialises the recipe and resolves - /// `\u0000` writes real NUL bytes into the program text, where they do not - /// survive argument passing. jq then emits no separators, `xargs -0` - /// collapses every field into one argument, and the result is a single - /// bogus dispatch that exits 0 without dispatching anything. - #[test] - fn recipes_carry_no_nul_escape_for_a_materialiser_to_resolve() { - for recipe in [ - render_parallel_dispatch_recipe(" run \"$eval_root\"", false, &BTreeMap::new()), - render_parallel_dispatch_recipe(" run \"$eval_root\"", true, &BTreeMap::new()), - render_judge_dispatch_recipe(" judge $model_arg \\", "--model", "judge"), - ] { - assert!(!recipe.contains("\\u0000"), "{recipe}"); - assert!(!recipe.contains('\0'), "{recipe}"); - } - } - - #[test] - fn judge_recipe_preserves_an_empty_model_and_skips_existing_responses() { - let recipe = render_judge_dispatch_recipe(" judge $model_arg \\", "--model", "judge"); - - assert!(recipe.contains( - "jq -r '.tasks[] | .dispatch_prompt_path, .response_path, (\"model=\" + (.model // \"\"))' judge-tasks.json" - )); - assert!(recipe.contains("| tr '\\n' '\\0' \\"), "{recipe}"); - assert!( - recipe.contains("| xargs -0 -P \"$JOBS\" -n 3 sh -c '"), - "{recipe}" - ); - assert!(recipe.contains("prompt_path=\"$1\""), "{recipe}"); - assert!(recipe.contains("response_path=\"$2\""), "{recipe}"); - assert!(recipe.contains("model=\"${3#model=}\""), "{recipe}"); - assert!( - recipe.contains("if [ -s \"$response_path\" ]; then exit 0; fi"), - "{recipe}" - ); - assert!( - recipe.contains( - "Existing nonempty response files are skipped; delete one to dispatch that judge again." - ), - "{recipe}" - ); - assert!( - recipe.contains( - "The final `N/M verdicts present` summary exits nonzero until every task has one." - ), - "{recipe}" - ); - assert!(!recipe.contains("-I{}"), "{recipe}"); - assert!(!recipe.contains("cut -f"), "{recipe}"); - } - - #[test] - fn judge_recipe_reports_partial_completion_and_exits_nonzero() { - let Some(shell) = recipe_shell("judge_recipe_reports_partial_completion_and_exits_nonzero") - else { - return; - }; - let tmp = tempfile::TempDir::new().unwrap(); - let responses_dir = tmp.path().join("judge responses"); - fs::create_dir_all(&responses_dir).unwrap(); - let existing_response = responses_dir.join("existing.json"); - let missing_response = responses_dir.join("missing.json"); - fs::write(&existing_response, "{}\n").unwrap(); - write_judge_tasks(tmp.path(), &[&existing_response, &missing_response]); - - let output = run_judge_recipe(shell, tmp.path(), " true $model_arg \\"); - - assert!(!output.status.success(), "{output:?}"); - assert_eq!( - String::from_utf8(output.stdout).unwrap(), - "1/2 verdicts present\n" - ); - } - - /// jq's native Windows build opens stdout in text mode, so every `\n` it - /// writes arrives as `\r\n`. The recipes read that output as paths, and - /// neither reader drops the CR: `read -r` keeps it by definition, and - /// `tr '\n' '\0'` converts only the newline. The carriage return then rides - /// on the end of every path — `[ -s "$response_path" ]` matches nothing, the - /// summary reports zero verdicts present, and each dispatched task gets a - /// corrupted `$eval_root`. Git Bash with `jq` installed is the documented - /// Windows setup, so jq's output has to be normalised before anything reads - /// it. Same expectations as the plain-jq partial-completion test above; only - /// jq's line endings differ. - #[test] - fn judge_recipe_counts_verdicts_when_jq_emits_crlf() { - let Some(shell) = recipe_shell("judge_recipe_counts_verdicts_when_jq_emits_crlf") else { - return; - }; - let tmp = tempfile::TempDir::new().unwrap(); - let responses_dir = tmp.path().join("judge responses"); - fs::create_dir_all(&responses_dir).unwrap(); - let existing_response = responses_dir.join("existing.json"); - let missing_response = responses_dir.join("missing.json"); - fs::write(&existing_response, "{}\n").unwrap(); - write_judge_tasks(tmp.path(), &[&existing_response, &missing_response]); - - let output = - run_judge_recipe_prefixed(shell, tmp.path(), " true $model_arg \\", CRLF_JQ); - - assert!(!output.status.success(), "{output:?}"); - assert_eq!( - String::from_utf8(output.stdout).unwrap(), - "1/2 verdicts present\n", - "stderr: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - /// Every recipe that reads jq's output has to strip the carriage return - /// jq's Windows build adds, and it has to do so once per call: the judge - /// recipe pipes jq into `xargs`, into `read`, and into an arithmetic - /// comparison, and a CR left on any one of them breaks that stage alone. - #[test] - fn recipes_strip_the_carriage_return_a_windows_jq_emits() { - for recipe in [ - render_parallel_dispatch_recipe(" run \"$eval_root\"", false, &BTreeMap::new()), - render_parallel_dispatch_recipe(" run \"$eval_root\"", true, &BTreeMap::new()), - render_judge_dispatch_recipe(" judge $model_arg \\", "--model", "judge"), - ] { - let calls = recipe.lines().filter(|line| line.contains("jq ")).count(); - assert!(calls > 0, "{recipe}"); - assert_eq!( - calls, - recipe.matches("tr -d '\\r'").count(), - "every jq call needs its own CR strip\n{recipe}" - ); - } - } - - #[test] - fn judge_recipe_reports_complete_resumed_batch_and_exits_zero() { - let Some(shell) = - recipe_shell("judge_recipe_reports_complete_resumed_batch_and_exits_zero") - else { - return; - }; - let tmp = tempfile::TempDir::new().unwrap(); - let first_response = tmp.path().join("first.json"); - let second_response = tmp.path().join("second.json"); - fs::write(&first_response, "first\n").unwrap(); - fs::write(&second_response, "second\n").unwrap(); - write_judge_tasks(tmp.path(), &[&first_response, &second_response]); - - let output = run_judge_recipe(shell, tmp.path(), " false $model_arg \\"); - - assert!(output.status.success(), "{output:?}"); - assert_eq!( - String::from_utf8(output.stdout).unwrap(), - "2/2 verdicts present\n" - ); - assert_eq!(fs::read_to_string(first_response).unwrap(), "first\n"); - assert_eq!(fs::read_to_string(second_response).unwrap(), "second\n"); - } - - #[test] - fn judge_recipe_preserves_dispatch_failure_after_response_is_written() { - let Some(shell) = - recipe_shell("judge_recipe_preserves_dispatch_failure_after_response_is_written") - else { - return; - }; - let tmp = tempfile::TempDir::new().unwrap(); - let response = tmp.path().join("response.json"); - let failing_judge = tmp.path().join("failing-judge"); - fs::write( - &failing_judge, - "#!/bin/sh\nprintf '{}\\n' > \"$1\"\nexit 7\n", - ) - .unwrap(); - write_judge_tasks(tmp.path(), &[&response]); - - let output = run_judge_recipe( - shell, - tmp.path(), - " sh ./failing-judge \"$response_path\" $model_arg \\", - ); - - assert!(!output.status.success(), "{output:?}"); - assert_eq!( - String::from_utf8_lossy(&output.stdout), - "1/1 verdicts present\n", - "{output:?}" - ); - assert!(response.exists()); - } } diff --git a/src/adapters/descriptor.rs b/src/adapters/descriptor.rs index ef1a893..fd025e1 100644 --- a/src/adapters/descriptor.rs +++ b/src/adapters/descriptor.rs @@ -278,10 +278,6 @@ pub struct DispatchSection { #[serde(skip_serializing_if = "Option::is_none")] pub exec_template: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub parallel_command_template: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub judge_command_template: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub manifest_template: Option, } @@ -311,8 +307,6 @@ impl DispatchSection { && self.model_note.is_none() && self.next_steps_template.is_none() && self.exec_template.is_none() - && self.parallel_command_template.is_none() - && self.judge_command_template.is_none() && self.manifest_template.is_none() } } diff --git a/src/adapters/descriptor/validation.rs b/src/adapters/descriptor/validation.rs index 8d20a28..148e961 100644 --- a/src/adapters/descriptor/validation.rs +++ b/src/adapters/descriptor/validation.rs @@ -39,7 +39,6 @@ const CHECKS: &[Check] = &[ transcript::check_tiers, conversation::validate, check_tool_roles_disjoint, - check_judge_command_template, check_template_placeholder_backing, check_manifest_template_newline, check_skills_block_item, @@ -364,56 +363,11 @@ fn check_tool_roles_disjoint(d: &HarnessDescriptor) -> Result<(), String> { Ok(()) } -/// The judge command line splices into the shared judge recipe; its contract -/// (see cli_command::render_judge_dispatch_recipe) is checkable here rather -/// than at render time. -fn check_judge_command_template(d: &HarnessDescriptor) -> Result<(), String> { - let Some(judge) = &d.dispatch.judge_command_template else { - return Ok(()); - }; - if d.model.is_none() { - return Err( - "dispatch.judge_command_template requires model.flag — the judge recipe \ - splices \"$model_arg\" from each task's model via the model flag" - .into(), - ); - } - if d.dispatch.capture_prefix.is_none() { - return Err( - "dispatch.judge_command_template requires dispatch.capture_prefix — it names \ - the per-task $response_base capture files" - .into(), - ); - } - if !judge.contains("$model_arg") { - return Err( - "dispatch.judge_command_template must reference $model_arg (empty when a task \ - declares no model)" - .into(), - ); - } - if !judge.contains("{cwd}") { - return Err( - "dispatch.judge_command_template must contain {cwd} — judges run from the \ - iteration dir" - .into(), - ); - } - if !judge.ends_with(" \\") { - return Err( - "dispatch.judge_command_template must end with a shell line continuation \ - (\" \\\") so the recipe's prompt line follows it" - .into(), - ); - } - Ok(()) -} - /// Placeholders must have a backing field, or the template renders with the /// token left in (the artifact tests' `!contains("{{")` rule, at load time). fn check_template_placeholder_backing(d: &HarnessDescriptor) -> Result<(), String> { let dispatch = &d.dispatch; - let pairings: [(&Option, &str, &str, bool); 7] = [ + let pairings: [(&Option, &str, &str, bool); 4] = [ ( &dispatch.next_steps_template, "next_steps_template", @@ -432,30 +386,12 @@ fn check_template_placeholder_backing(d: &HarnessDescriptor) -> Result<(), Strin "{exec_command}", dispatch.exec_template.is_some(), ), - ( - &dispatch.manifest_template, - "manifest_template", - "{parallel_recipe}", - dispatch.parallel_command_template.is_some(), - ), ( &dispatch.exec_template, "exec_template", "{guard_args}", dispatch.guard_args.is_some(), ), - ( - &dispatch.parallel_command_template, - "parallel_command_template", - "{guard_args}", - dispatch.guard_args.is_some(), - ), - ( - &dispatch.judge_command_template, - "judge_command_template", - "{guard_args}", - dispatch.guard_args.is_some(), - ), ]; for (template, template_name, placeholder, backed) in pairings { if template.as_deref().is_some_and(|t| t.contains(placeholder)) && !backed { diff --git a/src/adapters/descriptor/validation/tests/dispatch.rs b/src/adapters/descriptor/validation/tests/dispatch.rs index d8ff763..aa3de89 100644 --- a/src/adapters/descriptor/validation/tests/dispatch.rs +++ b/src/adapters/descriptor/validation/tests/dispatch.rs @@ -1,35 +1,5 @@ use super::{MINIMAL, err_of}; -#[test] -fn rejects_judge_template_without_model_flag() { - let err = err_of(&format!( - "{MINIMAL}\n[dispatch]\ncapture_prefix = \"demo\"\njudge_command_template = ' demo --cd \"{{cwd}}\" $model_arg \\'\n" - )); - assert!(err.contains("model.flag"), "{err}"); -} - -#[test] -fn rejects_judge_template_violating_the_recipe_contract() { - for (template, needle) in [ - ("' demo --cd \"{cwd}\" \\'", "$model_arg"), - ("' demo $model_arg \\'", "{cwd}"), - ("' demo --cd \"{cwd}\" $model_arg'", "line continuation"), - ] { - let err = err_of(&format!( - "{MINIMAL}\n[model]\nflag = \"-m\"\n\n[dispatch]\ncapture_prefix = \"demo\"\njudge_command_template = {template}\n" - )); - assert!(err.contains(needle), "expected {needle} in: {err}"); - } -} - -#[test] -fn rejects_judge_template_without_capture_prefix() { - let err = err_of(&format!( - "{MINIMAL}\n[model]\nflag = \"-m\"\n\n[dispatch]\njudge_command_template = ' demo --cd \"{{cwd}}\" $model_arg \\'\n" - )); - assert!(err.contains("capture_prefix"), "{err}"); -} - #[test] fn rejects_template_placeholders_without_backing_fields() { for (dispatch_body, needle) in [ @@ -42,10 +12,6 @@ fn rejects_template_placeholders_without_backing_fields() { "{model_note}", ), ("exec_template = \"demo{guard_args} run\"", "{guard_args}"), - ( - "exec_template = \"demo run\"\nmanifest_template = \"use:\\n{exec_command}\\n{parallel_recipe}\\n\"", - "{parallel_recipe}", - ), ] { let err = err_of(&format!("{MINIMAL}\n[dispatch]\n{dispatch_body}\n")); assert!(err.contains(needle), "expected {needle} in: {err}"); diff --git a/src/adapters/descriptor_adapter.rs b/src/adapters/descriptor_adapter.rs index 4399499..858099b 100644 --- a/src/adapters/descriptor_adapter.rs +++ b/src/adapters/descriptor_adapter.rs @@ -10,20 +10,14 @@ use std::time::Duration; use regex::Regex; -use crate::core::fs::artifact_path; use crate::core::{AvailableSkill, HarnessRunCapabilities, ToolInvocation}; use crate::sandbox::GuardMarker; -use super::cli_command::{ - render_agent_dispatch_command, render_cli_model_arg, render_judge_dispatch_recipe, - render_parallel_dispatch_recipe, -}; +use super::cli_command::{render_agent_dispatch_command, render_cli_model_arg}; use super::descriptor::{ HarnessDescriptor, TranscriptSection, render_staged_slug, stage_name_error, subst, }; -use super::harness::{ - CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter, ToolVocabulary, -}; +use super::harness::{CliDispatchContext, CliManifestContext, HarnessAdapter, ToolVocabulary}; use super::skill_shadow::{PluginShadowReport, ShadowSource}; use super::skills_block::{DEFAULT_HEADER, DEFAULT_ITEM, render_skills_block}; use super::{PermissionDenial, SessionSurface, TranscriptSummary}; @@ -460,71 +454,21 @@ impl HarnessAdapter for DescriptorAdapter { ); }; let exec_command = self.render_exec_command(ctx.guard, ctx.agent_model, ctx.agent_env); - let parallel_recipe = match &self.descriptor.dispatch.parallel_command_template { - Some(block_template) => { - let model_arg = render_cli_model_arg(self.model_flag(), ctx.agent_model); - render_parallel_dispatch_recipe( - &subst( - block_template, - &[ - ("model_arg", &model_arg), - ("guard_args", self.guard_args(ctx.guard)), - ], - ), - ctx.one_shot_only, - ctx.agent_env, - ) - } - None => String::new(), - }; Some( - subst( - template, - &[ - ("exec_command", &exec_command), - ("parallel_recipe", ¶llel_recipe), - ], - ) - .split('\n') - .map(String::from) - .collect(), + subst(template, &[("exec_command", &exec_command)]) + .split('\n') + .map(String::from) + .collect(), ) } - - fn cli_judge_next_steps(&self, ctx: CliJudgeContext<'_>) -> Option { - let template = self.descriptor.dispatch.judge_command_template.as_ref()?; - // Embedded in a shell command line, so it carries the wire-format - // spelling every other generated path uses. - let cwd = artifact_path(ctx.iteration_dir); - let command_line = subst( - template, - // Judges run from the iteration metadata directory, outside every - // guarded task env. Hook-trust bypass is only for eval-agent - // dispatches whose cwd actually contains the vetted guard hook. - &[("cwd", &cwd), ("guard_args", self.guard_args(false))], - ); - Some(render_judge_dispatch_recipe( - &command_line, - // Both guaranteed by descriptor validation when the template is set. - self.model_flag().unwrap_or_default(), - self.descriptor - .dispatch - .capture_prefix - .as_deref() - .unwrap_or_default(), - )) - } } #[cfg(test)] mod tests { use std::collections::BTreeMap; - use std::path::Path; use std::sync::LazyLock; - use crate::adapters::harness::{ - CliDispatchContext, CliJudgeContext, CliManifestContext, TokenUsageAggregation, - }; + use crate::adapters::harness::{CliDispatchContext, CliManifestContext, TokenUsageAggregation}; use crate::adapters::registry::adapter_for; use crate::core::{AvailableSkill, Harness}; @@ -541,16 +485,6 @@ mod tests { } } - fn next_steps(harness: Harness, agent_model: Option<&str>) -> String { - adapter_for(harness).cli_next_steps(CliDispatchContext { - guard: harness == Harness::resolve("codex").unwrap(), - target_args: " --skill-dir /tmp/skills --skill widget-skill", - iteration: 2, - agent_model, - agent_env: empty_env(), - }) - } - fn adapter_from(toml_src: &str) -> super::DescriptorAdapter { super::DescriptorAdapter::from_descriptor( crate::adapters::descriptor::load_descriptor(toml_src, "test.toml").unwrap(), @@ -582,7 +516,6 @@ mod tests { guard: false, agent_model: None, agent_env: empty_env(), - one_shot_only: false, }) .expect("an exec template earns a generic manifest recipe") .join("\n"); @@ -661,16 +594,17 @@ mod tests { .cli_resume_command(false, None, empty_env()) .unwrap(); assert!(resume.starts_with(prelude), "{resume}"); + // The manifest quotes the same exec command, so it inherits the + // prelude rather than carrying an indented copy of its own. let manifest = adapter .cli_manifest_section(CliManifestContext { guard: false, agent_model: None, agent_env: empty_env(), - one_shot_only: false, }) .unwrap() .join("\n"); - assert!(manifest.contains(&format!(" {prelude}\n")), "{manifest}"); + assert!(manifest.contains(prelude), "{manifest}"); } } @@ -697,38 +631,41 @@ mod tests { guard: false, agent_model: None, agent_env: empty_env(), - one_shot_only: false, }) .is_none(), "the manifest's generic header already covers the no-recipe baseline" ); } + /// The command the runner spawns, not the hand-off text: `cli_next_steps` + /// names `eval-magic dispatch` now, so model and guard rendering is only + /// observable on the exec command itself. + fn exec_command(harness: Harness, guard: bool, agent_model: Option<&str>) -> String { + adapter_for(harness) + .cli_exec_command(guard, agent_model, empty_env()) + .expect("a built-in harness declares an exec template") + } + #[test] fn exec_recipe_includes_model_only_when_declared() { - let with = next_steps(Harness::resolve("claude-code").unwrap(), Some("opus")); + let harness = Harness::resolve("claude-code").unwrap(); + let with = exec_command(harness, false, Some("opus")); assert!(with.contains("--model opus"), "{with}"); - let without = next_steps(Harness::resolve("claude-code").unwrap(), None); + let without = exec_command(harness, false, None); assert!(!without.contains("--model "), "{without}"); } #[test] fn codex_recipes_gate_hook_trust_on_guard() { - let guarded = next_steps(Harness::resolve("codex").unwrap(), Some("gpt-5-mini")); + let harness = Harness::resolve("codex").unwrap(); + let guarded = exec_command(harness, true, Some("gpt-5-mini")); assert!( guarded.contains( "codex --ask-for-approval never exec --cd --sandbox workspace-write --dangerously-bypass-hook-trust -m gpt-5-mini --json \\" ), "{guarded}" ); - let unguarded = - adapter_for(Harness::resolve("codex").unwrap()).cli_next_steps(CliDispatchContext { - guard: false, - target_args: "", - iteration: 2, - agent_model: None, - agent_env: empty_env(), - }); + let unguarded = exec_command(harness, false, None); assert!( !unguarded.contains("--dangerously-bypass-hook-trust"), "{unguarded}" @@ -737,17 +674,15 @@ mod tests { #[test] fn opencode_exec_recipe_carries_dir_auto_and_the_model_flag() { - let with = next_steps( - Harness::resolve("opencode").unwrap(), - Some("opencode/gpt-5-nano"), - ); + let harness = Harness::resolve("opencode").unwrap(); + let with = exec_command(harness, false, Some("opencode/gpt-5-nano")); assert!( with.contains( "opencode run --dir --format json --auto -m opencode/gpt-5-nano \\" ), "{with}" ); - let without = next_steps(Harness::resolve("opencode").unwrap(), None); + let without = exec_command(harness, false, None); assert!( without.contains("opencode run --dir --format json --auto \\"), "{without}" @@ -755,83 +690,6 @@ mod tests { assert!(!without.contains(" -m "), "{without}"); } - #[test] - fn codex_judge_recipe_splices_model_arg_in_one_command_shape() { - let recipe = adapter_for(Harness::resolve("codex").unwrap()) - .cli_judge_next_steps(CliJudgeContext { - guard: true, - iteration_dir: Path::new("/work/iter-1"), - }) - .expect("codex judge recipe is wired"); - // One command shape: the optional model flag is spliced via $model_arg - // (same structure as the Claude judge recipe), not an if/else pair. - assert!( - recipe.contains( - " codex --ask-for-approval never exec --cd \"/work/iter-1\" --sandbox workspace-write $model_arg --json \\" - ), - "{recipe}" - ); - assert!( - !recipe.contains("--dangerously-bypass-hook-trust"), - "judges run outside guarded task envs: {recipe}" - ); - assert!( - recipe.contains(" model_arg=\"\"; [ -n \"$model\" ] && model_arg=\"-m $model\""), - "{recipe}" - ); - assert!(!recipe.contains("if [ -n"), "{recipe}"); - } - - #[test] - fn claude_judge_recipe_snapshot_is_stable() { - // Full-string pin carried over from the pre-descriptor adapter: locks - // the Claude judge recipe byte-for-byte through the descriptor path. - let recipe = adapter_for(Harness::resolve("claude-code").unwrap()) - .cli_judge_next_steps(CliJudgeContext { - guard: false, - iteration_dir: Path::new("/work/iter-1"), - }) - .expect("claude judge recipe is wired"); - let expected = r#"Dispatch each judge task from judge-tasks.json with: -Existing nonempty response files are skipped; delete one to dispatch that judge again. -The final `N/M verdicts present` summary exits nonzero until every task has one. - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - prompt_path="$1" - response_path="$2" - model="${3#model=}" - if [ -s "$response_path" ]; then exit 0; fi - response_base="${response_path%.json}" - mkdir -p "$(dirname "$response_path")" - model_arg=""; [ -n "$model" ] && model_arg="--model $model" - cd "/work/iter-1" && claude -p --output-format stream-json --verbose --permission-mode bypassPermissions $model_arg \ - "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ - "$response_base.claude-events.jsonl" \ - 2> "$response_base.claude-stderr.log" - ' sh -judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') -judge_present=$( - jq -r '.tasks[].response_path' judge-tasks.json \ - | tr -d '\r' \ - | while IFS= read -r response_path; do - if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi - done \ - | wc -l \ - | tr -d '[:space:]' -) -printf '%s/%s verdicts present\n' "$judge_present" "$judge_total" -[ "$judge_dispatch_status" -eq 0 ] && [ "$judge_present" -eq "$judge_total" ] -```"#; - assert_eq!(recipe, expected); - } - #[test] fn skills_blocks_render_each_harness_native_shape() { let skills = vec![skill("zebra", "z skill"), skill("alpha", "a skill")]; diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index f261336..dbe1dd0 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -399,15 +399,14 @@ pub trait HarnessAdapter { format!("\n{trimmed}\n") } - // ── Enhancement: dispatch recipes (defaulted) ──────────────────────────── - // Fallback without them: `run` prints the generic handoff and the runbook - // carries no copy-pasteable per-task command. - - /// **Enhancement: dispatch recipes.** Whether a copy-pasteable per-task - /// exec command is wired (the descriptor's `[dispatch] exec_template`). - /// `false` means `RUNBOOK.md` / `dispatch-manifest.md` carry handoff - /// guidance without a per-task command recipe, and the `run` preflight - /// warns naming that limitation. + // ── Enhancement: dispatch commands (defaulted) ─────────────────────────── + // There is no fallback: without an exec template the runner has nothing to + // spawn, so `dispatch` fails for that harness and `run` warns at prep time. + + /// **Enhancement: dispatch commands.** Whether a per-task exec command is + /// wired (the descriptor's `[dispatch] exec_template`). `false` means + /// `eval-magic dispatch` has nothing to run for this harness, and the `run` + /// preflight warns naming that. fn has_dispatch_recipes(&self) -> bool { false } @@ -446,27 +445,20 @@ pub trait HarnessAdapter { None } - /// **Enhancement: dispatch recipes.** The `Next:` guidance printed after - /// `run`: how to dispatch each task through this harness's one-shot CLI - /// and then ingest. Empty when no dispatch recipe is wired. + /// **Enhancement: dispatch commands.** The `Next:` guidance printed after + /// `run`: the dispatch and ingest commands for this harness. Empty when the + /// descriptor wires no `next_steps_template`. fn cli_next_steps(&self, _ctx: CliDispatchContext<'_>) -> String { String::new() } - /// **Enhancement: dispatch recipes.** Extra `dispatch-manifest.md` lines - /// describing this harness's dispatch recipe (command template, parallel - /// recipe, ingest note). `None` when the harness contributes no manifest - /// section. + /// **Enhancement: dispatch commands.** Extra `dispatch-manifest.md` lines + /// describing what the runner will spawn for this harness (the command + /// template and any ingest note). `None` when the harness contributes no + /// manifest section. fn cli_manifest_section(&self, _ctx: CliManifestContext<'_>) -> Option> { None } - - /// **Enhancement: dispatch recipes.** The post-`grade` / post-`ingest` - /// judge dispatch guidance for this harness. `None` leaves the generic - /// judge handoff in place. - fn cli_judge_next_steps(&self, _ctx: CliJudgeContext<'_>) -> Option { - None - } } /// The shared (human-followed) `RUNBOOK.md` template used by every run, @@ -489,15 +481,6 @@ pub struct CliManifestContext<'a> { pub guard: bool, pub agent_model: Option<&'a str>, pub agent_env: &'a BTreeMap, - /// Exclude scripted tasks from a mixed suite's one-shot recipe. - pub one_shot_only: bool, -} - -/// Context for rendering a harness's one-shot CLI judge-dispatch guidance. -#[derive(Debug, Clone, Copy)] -pub struct CliJudgeContext<'a> { - pub guard: bool, - pub iteration_dir: &'a Path, } #[cfg(test)] diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index bf75860..ff28e43 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -39,7 +39,7 @@ mod skills_block; pub mod transcript; pub use harness::{ - CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter, RUNBOOK_TEMPLATE, + CliDispatchContext, CliManifestContext, HarnessAdapter, RUNBOOK_TEMPLATE, TokenUsageAggregation, ToolVocabulary, }; pub use registry::{ diff --git a/src/cli/args.rs b/src/cli/args.rs index 69d0aff..4683dfd 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -13,10 +13,9 @@ use clap::{Args, Parser, Subcommand}; /// An eval dispatches a fresh subagent twice per test case — once with the skill /// loaded, once without (or old version vs. new) — and grades both outputs against /// assertions. The pass-rate delta tells you whether the skill is worth shipping. -/// This CLI builds the workspace, stages skills for discovery, generates dispatch -/// prompts, assembles run records from transcripts, grades, and aggregates; your -/// agent harness supplies the one thing it never does itself: dispatching the -/// subagents. +/// This CLI builds the workspace, stages skills for discovery, dispatches every +/// subagent through your chosen harness CLI, assembles run records from the +/// transcripts, grades, and aggregates. /// /// The run loop is one canonical workflow in both modes: /// @@ -86,8 +85,8 @@ pub struct CommonArgs { pub mode: Option, /// Target harness: `claude-code` (default), `cline`, `codex`, or `opencode`. /// - /// All four built-ins support staged skills, transcript ingest, and dispatch - /// recipes; `claude-code`, `codex`, and `opencode` additionally support + /// All four built-ins support staged skills, transcript ingest, and runner + /// dispatch; `claude-code`, `codex`, and `opencode` additionally support /// scripted same-session follow-ups and the automatically armed write guard. /// Each reads its own per-task events file; Claude Code stages skills under /// `.claude/skills`, Cline under `.cline/skills`, Codex under @@ -215,10 +214,7 @@ pub(crate) enum HarnessCommands { /// the descriptor end-to-end: renders `dispatch.exec_template` with a /// trivial prompt in a throwaway temp dir, runs it via `sh -c` from /// the temp `eval_root`, and verifies `outputs/final-message.md` is - /// recovered (non-empty). It additionally render-only-validates - /// `parallel_command_template` and `judge_command_template` for - /// placeholder-shape errors — rendering each with stand-in values and - /// reporting any unresolved `{token}` the run would later surface. + /// recovered (non-empty). /// /// `--probe` invokes the real harness CLI (network, tokens, usage /// limits), so it is opt-in and never runs as part of standard CI checks @@ -449,7 +445,7 @@ pub struct RunArgs { /// staging — with `--no-stage` the guard stays off and the run is unguarded. /// Codex eval-agent dispatches must include /// `--dangerously-bypass-hook-trust` so the vetted project-local eval hook - /// runs; judge recipes omit it because judges run outside guarded task envs. + /// runs; judge dispatches omit it because judges run outside guarded task envs. /// Unguarded, stray writes are only *detected* after the fact by /// `detect-stray-writes`, never blocked. /// Under Claude Code the `PreToolUse` hook is staged in each env's @@ -512,9 +508,10 @@ pub struct RunArgs { /// Agent-under-test model for CLI dispatches; otherwise recorded as /// provenance. /// - /// The run's dispatch recipes include the harness-native model flag when the - /// adapter supports one (e.g. Codex's `-m`, Claude Code's `--model`); otherwise - /// the value is persisted to `conditions.json` for `promote-baseline`. + /// The commands `dispatch` spawns include the harness-native model flag when + /// the adapter supports one (e.g. Codex's `-m`, Claude Code's `--model`); + /// otherwise the value is persisted to `conditions.json` for + /// `promote-baseline`. #[arg(long)] pub agent_model: Option, /// Environment override for eval-agent dispatches (`KEY=VALUE`, repeatable). @@ -531,8 +528,8 @@ pub struct RunArgs { /// Default judge model for emitted judge tasks. /// /// `grade` writes this into `judge-tasks.json` for judge tasks that do not - /// have an assertion-level `model` override, and Cli harness judge recipes - /// pass it through using the harness-native model flag. Also persists to + /// have an assertion-level `model` override, and `dispatch --judges` passes + /// it through using the harness-native model flag. Also persists to /// `conditions.json` for `promote-baseline`. #[arg(long)] pub judge_model: Option, @@ -544,18 +541,28 @@ pub struct RunArgs { pub label: Option, } -/// Execute one runner-owned multi-turn task from a generated dispatch plan. #[derive(Debug, Args)] -pub struct DispatchTaskArgs { - /// Path to the runner-generated dispatch.json. - #[arg(long, value_name = "PATH")] - pub dispatch: String, - /// Zero-based index into dispatch.json's tasks array. - #[arg(long)] - pub task_index: usize, - /// Replace an existing conversation.json and rerun the task. - #[arg(long)] - pub overwrite: bool, +pub struct DispatchArgs { + #[command(flatten)] + pub common: CommonArgs, + /// Dispatch only these zero-based `tasks[]` indices; repeatable. Every task + /// in the plan by default. + #[arg(long = "task-index")] + pub task_index: Vec, + /// Kill a task that runs longer than this many seconds and record it as + /// timed out, so one hung dispatch cannot stall the campaign. `0` disables + /// the deadline entirely. + #[arg(long, default_value_t = 1800, value_name = "SECONDS")] + pub timeout: u64, + /// How many tasks to dispatch at once. Each task owns a private environment, + /// so they are independent. + #[arg(long, default_value_t = 4, value_parser = clap::value_parser!(u32).range(1..))] + pub jobs: u32, + /// Dispatch the judge tasks `ingest` emitted instead of the eval tasks. + /// Skips existing nonempty responses, prints `N/M verdicts present`, and + /// exits nonzero while any are missing; rerun to fill the gaps. + #[arg(long)] + pub judges: bool, } /// Every subcommand on the CLI. @@ -565,10 +572,10 @@ pub(crate) enum Commands { /// /// Builds the iteration workspace, snapshots the `SKILL.md`, stages skills, and /// emits `dispatch.json` (machine-readable) alongside `dispatch-manifest.md` - /// (human-readable). It prepares the run but does not dispatch agents. After - /// setup, read `RUNBOOK.md` end to end; that generated file is the authority - /// for dispatch, ingest, judge, finalize, and teardown commands for the selected - /// harness. + /// (human-readable). It prepares the run but does not dispatch agents — + /// `eval-magic dispatch` does. After setup, read `RUNBOOK.md` end to end; that + /// generated file is the authority for the dispatch, ingest, judge, finalize, + /// and teardown commands for the selected harness. /// /// A case with effective run count `R` creates `2R` native agent sessions: one /// per condition and repetition. Scripted follow-ups add up to `2R × F` model @@ -590,17 +597,25 @@ pub(crate) enum Commands { /// Isolating each dispatch from those sources, and confirming it worked, is /// `eval-magic docs isolation`. Run(RunArgs), - /// Execute one scripted multi-turn task through its harness CLI. - /// - /// Starts the task, resumes the same native session for every delivered - /// follow-up, and writes the task's `conversation.json` completion artifact. - /// A completed or normally stopped conversation records - /// `delivered_followups`; an interrupted task commits no artifact. Each round - /// must report the same native session ID or the command fails. Inspect the - /// per-round assistant messages and delivered count to verify the script ran as - /// intended. One-shot tasks continue to use the commands in - /// `dispatch-manifest.md`. - DispatchTask(DispatchTaskArgs), + /// Run every task in a prepared iteration through its harness CLI. + /// + /// Reads `dispatch.json`, executes each task in its own private environment, + /// and writes the task's `conversation.json` completion artifact. A task that + /// already has one is skipped, so rerunning after a failure retries only what + /// did not finish; `--overwrite` redispatches regardless. + /// + /// A task that fails is recorded and the batch continues — one bad dispatch + /// does not abandon the campaign. The command exits nonzero if any task + /// failed. A conversation that stops at a scripted gate is valid eval data, + /// not a failure. + /// + /// A task declaring scripted follow-up turns resumes the same native session + /// for every turn it delivers, and each round must report the same native + /// session ID or that task fails. A completed or normally stopped + /// conversation records `delivered_followups`; an interrupted task commits no + /// artifact, so a rerun picks it up. Inspect the per-round assistant messages + /// and the delivered count to verify a script ran as intended. + Dispatch(DispatchArgs), /// Snapshot a workspace baseline. /// /// Snapshots the skill as a Mode B baseline under @@ -636,9 +651,8 @@ pub(crate) enum Commands { /// scope is captured before held-out files are injected. Then stops at the /// judge hand-off, listing a judge task per `llm_judge` assertion. Requires /// `--iteration`; reads each task's `outputs/-events.jsonl` when the - /// harness exposes transcripts. When the harness provides a judge recipe, it - /// skips existing nonempty responses, prints `N/M verdicts present`, and exits - /// nonzero while any are missing; rerun the same recipe to fill the gaps. + /// harness exposes transcripts, under `outputs/turn-/`. Dispatch the judge + /// tasks it lists with `eval-magic dispatch --judges`. /// Re-running after a fix is safe — every sub-step skips work already done. Ingest(CommonArgs), /// Finalize grading after judge responses are in. @@ -867,6 +881,11 @@ pub struct FixtureArgs { /// output larger than the diagnostic truncation limit. #[arg(long)] pub pad: Option, + /// Sleep this many milliseconds before doing anything else, so a caller can + /// overrun a deadline. The delay lives here rather than in a `sleep` call + /// because Windows has no such binary. + #[arg(long = "sleep-ms")] + pub sleep_ms: Option, /// Joins the fragments. Empty by default. #[arg(long, default_value = "")] pub separator: String, diff --git a/src/cli/commands/fixture.rs b/src/cli/commands/fixture.rs index 377fb3d..e600090 100644 --- a/src/cli/commands/fixture.rs +++ b/src/cli/commands/fixture.rs @@ -36,6 +36,11 @@ fn execute_fixture( out: &mut impl Write, err: &mut impl Write, ) -> anyhow::Result { + // Ahead of every effect, so a caller waiting on this process overruns its + // deadline before any output or file write suggests progress. + if let Some(millis) = args.sleep_ms { + std::thread::sleep(std::time::Duration::from_millis(millis)); + } let satisfied = requirements_met(args)?; let emitted = emitted_output(args); @@ -154,6 +159,26 @@ mod tests { ) } + /// `--sleep-ms` delays the fixture before it does anything else, which is + /// what lets a dispatch-timeout test overrun a deadline on any host. `sleep` + /// is a POSIX binary Windows lacks, so the delay has to live in the fixture + /// itself. + #[test] + fn sleep_ms_delays_the_fixture_before_it_emits() { + let started = std::time::Instant::now(); + let (code, out, _) = run(&FixtureArgs { + sleep_ms: Some(120), + text: vec!["done".into()], + ..args() + }); + assert_eq!((code, out.as_str()), (0, "done")); + assert!( + started.elapsed() >= std::time::Duration::from_millis(120), + "expected the fixture to sleep at least 120ms, took {:?}", + started.elapsed() + ); + } + #[test] fn emits_nothing_and_exits_zero_by_default() { assert_eq!(run(&args()), (0, String::new(), String::new())); diff --git a/src/cli/commands/harness.rs b/src/cli/commands/harness.rs index f0e28f0..f611e5e 100644 --- a/src/cli/commands/harness.rs +++ b/src/cli/commands/harness.rs @@ -495,7 +495,7 @@ mod tests { let rendered = subst(INIT_TEMPLATE, &[("label", "demo")]); assert!(!rendered.contains("{label}")); // Placeholders that belong to the examples survive substitution. - for survivor in ["{prefix}", "{model_arg}", "{name}", "{cwd}"] { + for survivor in ["{prefix}", "{model_arg}", "{name}"] { assert!( rendered.contains(survivor), "{survivor} should pass through subst" diff --git a/src/cli/commands/harness/probe.rs b/src/cli/commands/harness/probe.rs index 7bd7661..9ea1079 100644 --- a/src/cli/commands/harness/probe.rs +++ b/src/cli/commands/harness/probe.rs @@ -1,24 +1,21 @@ //! The `harness lint --probe` live dispatch check: render //! `dispatch.exec_template` with a trivial prompt in a throwaway temp dir, -//! execute it, and verify `outputs/final-message.md` is recovered; also -//! render-only-validate `parallel_command_template` / -//! `judge_command_template` for placeholder-shape errors. Invokes the real -//! harness CLI, so it is opt-in and never part of standard CI checks. +//! execute it, and verify `outputs/final-message.md` is recovered. Invokes the +//! real harness CLI, so it is opt-in and never part of standard CI checks. +use std::collections::BTreeMap; use std::io::{self, BufRead, Write}; use std::path::Path; -use std::process::{Command, ExitStatus, Stdio}; -use std::thread::sleep; -use std::time::{Duration, Instant}; +use std::process::ExitStatus; +use std::time::Duration; use anyhow::{Context, bail}; -use regex::Regex; use crate::adapters::cli_command::{ render_agent_dispatch_command, render_cli_model_arg, shell_quote_arg, }; use crate::adapters::descriptor::{HarnessDescriptor, subst}; -use crate::core::posix_shell; +use crate::core::{ShellOutcome, run_in_posix_shell}; /// Options carried from the parsed `--probe` flags into [`run_probe`]. #[derive(Debug, Clone, Copy)] @@ -55,8 +52,6 @@ pub(crate) enum ProbeError { FinalMessageMissing, #[error("outputs/final-message.md is empty")] FinalMessageEmpty, - #[error("unresolved placeholder {0:?} remains after substitution")] - UnresolvedBrace(String), } /// Render the exec template with the angle placeholders (``, @@ -87,43 +82,6 @@ fn render_probe_exec( ) } -/// Execute `command` via the resolved POSIX shell with `cwd` as the subprocess -/// working directory, killing the child if it exceeds `timeout`. The child's -/// stdin is `null`: the parent reads the `y/N` confirm on its own stdin and -/// never wants the dispatched agent CLI to consume it. -/// -/// Only the direct child is killed on timeout, not its process group — a shell -/// that has already forked leaves the grandchild running until it exits. -fn execute_with_timeout( - command: &str, - cwd: &Path, - timeout: Duration, -) -> Result { - let shell = posix_shell().map_err(|message| ProbeError::SpawnFailed(message.to_string()))?; - let mut child = Command::new(shell) - .arg("-c") - .arg(command) - .current_dir(cwd) - .stdin(Stdio::null()) - .spawn() - .map_err(|e| ProbeError::SpawnFailed(e.to_string()))?; - let deadline = Instant::now() + timeout; - loop { - match child.try_wait() { - Ok(Some(status)) => return Ok(status), - Ok(None) => { - if Instant::now() >= deadline { - let _ = child.kill(); - let _ = child.wait(); - return Err(ProbeError::Timeout(timeout)); - } - sleep(Duration::from_millis(10)); - } - Err(e) => return Err(ProbeError::SpawnFailed(e.to_string())), - } - } -} - /// Verify the final-message recovery contract: `outputs_dir/final-message.md` /// exists and is non-empty after trimming. fn verify_final_message(outputs_dir: &Path) -> Result<(), ProbeError> { @@ -138,52 +96,9 @@ fn verify_final_message(outputs_dir: &Path) -> Result<(), ProbeError> { Ok(()) } -/// Render-only validate `template`: substitute the supplied stand-in `{vars}`, -/// then fail if any `{alpha_token}` placeholder remains unresolved — anywhere -/// in the rendered text, including embedded mid-token. Catches typos like -/// `{cwdd}` before a real run exercises the template. Shell idioms that are -/// not placeholder tokens (`${JOBS:-4}`, `-I{}`) do not match the pattern and -/// pass through cleanly. -fn render_only_check(template: &str, vars: &[(&str, &str)]) -> Result<(), ProbeError> { - let rendered = subst(template, vars); - static PLACEHOLDER: std::sync::OnceLock = std::sync::OnceLock::new(); - let re = PLACEHOLDER - .get_or_init(|| Regex::new(r"\{[a-zA-Z_][a-zA-Z0-9_]*\}").expect("placeholder regex")); - if let Some(m) = re.find(&rendered) { - return Err(ProbeError::UnresolvedBrace(m.as_str().to_string())); - } - Ok(()) -} - -/// Trivial prompt body written to `/probe-prompt.md`. The probe does -/// not assert on reply content — only that `outputs/final-message.md` ends up -/// non-empty — so the prompt is intentionally minimal and generic across every -/// harness. +/// The trivial prompt the probe dispatches: short, deterministic, and cheap. const PROBE_PROMPT: &str = "Reply with the single word: ok\n"; -/// Stand-in `{var}` values used by the render-only checks so a missing backing -/// field never masquerades as a clean render. Every placeholder the dispatch -/// path fills needs an entry here, or the check reports a token the real run -/// resolves. The values are visible markers rather than faithful fragments — -/// the rendered text is only scanned for leftover braces, never executed — so -/// `{guard_args}` carries one too instead of the empty fragment the probe hands -/// the exec template. -const RENDER_STAND_INS: [(&str, &str); 3] = [ - ("cwd", "/probe/stand-in/cwd"), - ("model_arg", "stand-in-model"), - ("guard_args", "--stand-in-guard"), -]; - -/// The live dispatch probe. Renders `dispatch.exec_template` with a trivial -/// prompt in a throwaway temp dir, asks for confirmation, runs it under a -/// timeout through the resolved POSIX shell from that dir, then verifies the final-message -/// recovery contract. Also render-only-validates `parallel_command_template` -/// and `judge_command_template` for placeholder-shape errors. Invokes the real -/// harness CLI and is opt-in; never part of standard CI checks. -/// -/// `target_display` is the provenance string `lint` already printed under -/// `"Linted …"` (a file path or the joined layer chain); it appears only in the -/// confirm banner so the operator can see which descriptor is about to run. pub(crate) fn run_probe( descriptor: HarnessDescriptor, target_display: &str, @@ -200,8 +115,6 @@ pub(crate) fn run_probe( // The probe never arms the guard, so {guard_args} resolves to the empty // fragment. let model_flag = descriptor.model.as_ref().map(|m| m.flag.as_str()); - let parallel_template = descriptor.dispatch.parallel_command_template.clone(); - let judge_template = descriptor.dispatch.judge_command_template.clone(); let agent_env = descriptor.dispatch.env.clone(); let model_arg = render_cli_model_arg(model_flag, None); let guard_args = ""; @@ -253,50 +166,32 @@ pub(crate) fn run_probe( } let mut failed = 0u32; - match execute_with_timeout(&command, eval_root, opts.timeout) { - Ok(status) if status.success() => match verify_final_message(&outputs_dir) { - Ok(()) => println!("✓ live exec template: final-message recovered"), - Err(e) => { - eprintln!("✗ {e}"); - failed += 1; + let probed = run_in_posix_shell(&command, eval_root, &BTreeMap::new(), Some(opts.timeout)) + .map_err(ProbeError::SpawnFailed); + match probed { + Ok(ShellOutcome::Exited(status)) if status.success() => { + match verify_final_message(&outputs_dir) { + Ok(()) => println!("✓ live exec template: final-message recovered"), + Err(e) => { + eprintln!("✗ {e}"); + failed += 1; + } } - }, - Ok(status) => { + } + Ok(ShellOutcome::Exited(status)) => { eprintln!("✗ {}", ProbeError::ExecFailed(status)); failed += 1; } + Ok(ShellOutcome::TimedOut) => { + eprintln!("✗ {}", ProbeError::Timeout(opts.timeout)); + failed += 1; + } Err(e) => { eprintln!("✗ {e}"); failed += 1; } } - // Render-only static checks: render each recipe with stand-in vars and fail - // on any `{token}` the run would later surface. Catches typos like - // `{cwdd}` before a real dispatch spends usage. - if let Some(template) = parallel_template.as_deref() { - match render_only_check(template, &RENDER_STAND_INS) { - Ok(()) => println!("✓ render: parallel_command_template"), - Err(e) => { - eprintln!("✗ render: parallel_command_template: {e}"); - failed += 1; - } - } - } else { - println!("· parallel_command_template not declared — skipped"); - } - if let Some(template) = judge_template.as_deref() { - match render_only_check(template, &RENDER_STAND_INS) { - Ok(()) => println!("✓ render: judge_command_template"), - Err(e) => { - eprintln!("✗ render: judge_command_template: {e}"); - failed += 1; - } - } - } else { - println!("· judge_command_template not declared — skipped"); - } - if failed > 0 { bail!("probe failed for {label}: {failed} check(s) failed"); } @@ -306,7 +201,6 @@ pub(crate) fn run_probe( #[cfg(test)] mod tests { use super::*; - use crate::adapters::descriptor::{EMBEDDED_DESCRIPTORS, load_descriptor}; use std::fs; use std::path::PathBuf; @@ -352,23 +246,6 @@ mod tests { assert!(!rendered.contains("{model_arg}")); } - #[test] - fn execute_with_timeout_returns_status_on_success() { - let status = execute_with_timeout("true", Path::new("."), Duration::from_secs(5)) - .expect("true should succeed"); - assert!(status.success()); - } - - #[test] - fn execute_with_timeout_kills_on_overrun() { - let err = execute_with_timeout("sleep 5", Path::new("."), Duration::from_millis(100)) - .expect_err("sleep should time out"); - assert!( - matches!(err, ProbeError::Timeout(d) if d == Duration::from_millis(100)), - "got {err:?}" - ); - } - #[test] fn verify_final_message_accepts_a_non_empty_file() { let tmp = tempfile::TempDir::new().unwrap(); @@ -393,80 +270,4 @@ mod tests { let err = verify_final_message(tmp.path()).expect_err("blank should fail"); assert!(matches!(err, ProbeError::FinalMessageEmpty), "got {err:?}"); } - - #[test] - fn render_only_check_passes_a_resolved_template() { - let template = "judge --cd {cwd} $model_arg"; - let vars = [("cwd", "/work"), ("model_arg", "gpt-x")]; - render_only_check(template, &vars).expect("fully resolved template should pass"); - } - - #[test] - fn render_only_check_fails_on_an_unresolved_brace() { - let template = "judge --cd {cwd} --model {cwdd}"; - let vars = [("cwd", "/work"), ("model_arg", "gpt-x")]; - let err = render_only_check(template, &vars).expect_err("typo should fail"); - assert!( - matches!(err, ProbeError::UnresolvedBrace(ref t) if t == "{cwdd}"), - "got {err:?}" - ); - } - - #[test] - fn render_only_check_fails_on_a_brace_embedded_in_a_token() { - // `--out {cwd}/final` substitutes {cwd} cleanly, but `--out {cwdd}/final` - // (typo) leaves {cwdd} embedded mid-token — the check must still catch - // it, not only standalone {cwdd}. - let template = "agent --out {cwdd}/final"; - let vars = [("cwd", "/work"), ("model_arg", "gpt-x")]; - let err = render_only_check(template, &vars).expect_err("embedded typo should fail"); - assert!( - matches!(err, ProbeError::UnresolvedBrace(ref t) if t == "{cwdd}"), - "got {err:?}" - ); - } - - #[test] - fn render_only_check_passes_shell_brace_tokens_through() { - // `${JOBS:-4}` and `-I{}` are real shell idioms the runbook uses; they - // must not be mistaken for unresolved placeholders. - let template = "xargs -I{} sh -c 'echo ${JOBS:-4}' {cwd}"; - let vars = [("cwd", "/work"), ("model_arg", "gpt-x")]; - render_only_check(template, &vars).expect("shell braces plus a resolved {cwd} should pass"); - } - - #[test] - fn render_stand_ins_cover_guard_args() { - // Guarded harnesses splice {guard_args} onto a preceding flag value. - // A stand-in must back it or the probe reports a placeholder the real - // dispatch path resolves. - let template = "agent exec --sandbox workspace-write{guard_args} --cd {cwd}"; - render_only_check(template, &RENDER_STAND_INS).expect("{guard_args} must have a stand-in"); - } - - #[test] - fn render_stand_ins_cover_every_shipped_dispatch_template() { - // The render-only checks run against the shipped descriptors, so the - // stand-ins have to cover every placeholder those descriptors use. - // Anything missing surfaces as a false `✗ render:` failure. - for (source, toml_src) in EMBEDDED_DESCRIPTORS { - let descriptor = load_descriptor(toml_src, source) - .unwrap_or_else(|e| panic!("embedded descriptor {source} is invalid: {e}")); - let dispatch = &descriptor.dispatch; - for (field, template) in [ - ( - "parallel_command_template", - dispatch.parallel_command_template.as_deref(), - ), - ( - "judge_command_template", - dispatch.judge_command_template.as_deref(), - ), - ] { - let Some(template) = template else { continue }; - render_only_check(template, &RENDER_STAND_INS) - .unwrap_or_else(|e| panic!("{source} {field}: {e}")); - } - } - } } diff --git a/src/cli/commands/mod.rs b/src/cli/commands/mod.rs index 46c0514..9760c4c 100644 --- a/src/cli/commands/mod.rs +++ b/src/cli/commands/mod.rs @@ -23,6 +23,6 @@ pub(crate) use pipeline::{ run_aggregate, run_detect_stray_writes, run_fill_transcripts, run_finalize, run_grade, run_ingest, run_record_runs, }; -pub(crate) use run::{run_dispatch_task, run_run}; +pub(crate) use run::{run_dispatch, run_run}; pub(crate) use validate::run_validate; pub(crate) use workspace::{run_promote_baseline, run_snapshot, run_teardown}; diff --git a/src/cli/commands/pipeline.rs b/src/cli/commands/pipeline.rs index d4f8a91..50d4eca 100644 --- a/src/cli/commands/pipeline.rs +++ b/src/cli/commands/pipeline.rs @@ -4,7 +4,6 @@ use anyhow::bail; -use crate::adapters::{CliJudgeContext, adapter_for}; use crate::cli::args::{CommonArgs, GradeArgs}; use crate::cli::command_target_args; use crate::cli::run; @@ -15,23 +14,15 @@ use crate::sandbox; use crate::validation; use std::path::{Path, PathBuf}; -const JUDGE_WORKER_PROMPT: &str = "Read the file at and follow it exactly. You are a judge worker only: write the JSON verdict to , then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers."; - +/// The command that dispatches the judge tasks `ingest` emitted. Harness- +/// independent: the runner drives judges the same way it drives eval tasks, so +/// the only thing that varies is the `--harness` selector. fn judge_dispatch_guidance(ctx: &RunContext, iteration: u32) -> String { - let iteration_dir = ctx - .workspace_root - .join(&ctx.skill_name) - .join(format!("iteration-{iteration}")); - adapter_for(ctx.harness) - .cli_judge_next_steps(CliJudgeContext { - guard: sandbox::guard_is_armed(&ctx.stage_root), - iteration_dir: &iteration_dir, - }) - .unwrap_or_else(|| { - format!( - "Dispatch each task from judge-tasks.json with:\n {JUDGE_WORKER_PROMPT}\nModel selection is recorded in judge-tasks.json, but this harness adapter has no judge CLI recipe wired yet." - ) - }) + format!( + "eval-magic dispatch --judges{} --iteration {iteration} --harness {}", + command_target_args(ctx), + ctx.harness.name() + ) } /// Execute one chain step by mapping its [`run::steps::StepKind`] to the stage diff --git a/src/cli/commands/run.rs b/src/cli/commands/run.rs index 5619258..36ee434 100644 --- a/src/cli/commands/run.rs +++ b/src/cli/commands/run.rs @@ -1,14 +1,13 @@ -//! `run` and `dispatch-task` handlers. +//! `run` and `dispatch` handlers. use std::collections::BTreeMap; -use std::path::Path; -use anyhow::anyhow; +use anyhow::{anyhow, bail}; use crate::adapters::adapter_for; -use crate::cli::args::{DispatchTaskArgs, RunArgs}; +use crate::cli::args::{DispatchArgs, RunArgs}; use crate::cli::run; -use crate::cli::{parse_id_list, run_context_with_bootstrap}; +use crate::cli::{iteration_dir, parse_id_list, run_context_from, run_context_with_bootstrap}; use crate::core::validate_agent_environment_entry; fn parse_agent_environment(values: &[String]) -> anyhow::Result> { @@ -59,12 +58,78 @@ pub(crate) fn run_run(args: RunArgs) -> anyhow::Result<()> { Ok(()) } -pub(crate) fn run_dispatch_task(args: DispatchTaskArgs) -> anyhow::Result<()> { - run::conversation::command_dispatch_task( - Path::new(&args.dispatch), - args.task_index, - args.overwrite, - ) +/// Execute a prepared iteration's tasks through the harness. +pub(crate) fn run_dispatch(args: DispatchArgs) -> anyhow::Result<()> { + let ctx = run_context_from(&args.common)?; + let iteration_dir = iteration_dir(&ctx, args.common.iteration)?; + // `--timeout 0` means "no deadline", which is the only way to say it with a + // plain seconds flag. + let timeout = (args.timeout > 0).then(|| std::time::Duration::from_secs(args.timeout)); + if args.judges { + return dispatch_judges(&iteration_dir, &args, timeout); + } + let dispatch_path = iteration_dir.join("dispatch.json"); + if !dispatch_path.is_file() { + bail!( + "{} not found — run `eval-magic run` to prepare the iteration first", + dispatch_path.display() + ); + } + + let summary = run::drive::command_dispatch( + &dispatch_path, + &args.task_index, + args.common.overwrite, + timeout, + args.jobs as usize, + )?; + + println!( + "\nDispatched {} task(s): {}", + summary.reports.len(), + summary.tally() + ); + for warning in summary.warnings() { + eprintln!("⚠ {warning}"); + } + if summary.unusable() > 0 { + bail!( + "{} task(s) produced no usable result; rerun `eval-magic dispatch` to retry the \ + failures (a timed-out task keeps its record — pass --overwrite to redo it)", + summary.unusable() + ); + } + Ok(()) +} + +/// Dispatch the judge tasks `ingest` emitted, reporting verdict completeness. +fn dispatch_judges( + iteration_dir: &std::path::Path, + args: &DispatchArgs, + timeout: Option, +) -> anyhow::Result<()> { + let summary = run::drive::judges::command_dispatch_judges( + iteration_dir, + args.common.overwrite, + timeout, + args.jobs as usize, + )?; + println!( + "\nDispatched {} judge task(s), skipped {}: {}", + summary.dispatched, + summary.skipped, + summary.verdict_line() + ); + for failure in &summary.failures { + eprintln!("⚠ {failure}"); + } + if !summary.complete() { + bail!( + "{} — rerun `eval-magic dispatch --judges` to fill the gaps", + summary.verdict_line() + ); + } + Ok(()) } #[cfg(test)] diff --git a/src/cli/help.rs b/src/cli/help.rs index 8806e71..7c14544 100644 --- a/src/cli/help.rs +++ b/src/cli/help.rs @@ -8,20 +8,20 @@ /// Worked examples shown at the end of `eval-magic --help`. pub(super) const AFTER_HELP: &str = "\ REQUIREMENTS: - Git, plus a POSIX shell with jq, xargs, tr, and wc. The dispatch and judge - recipes in the generated RUNBOOK.md are POSIX command lines, and the shell - that runs them has to resolve the same paths the workspace was prepared - with. On Windows that is Git Bash (Git for Windows), with jq installed - separately. WSL resolves a different filesystem namespace, so run - eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH - to select a specific sh. + Git, plus a POSIX shell. Harness dispatch commands are POSIX command + lines, and eval-magic dispatch runs them itself, so the host it runs on + needs a shell that resolves the workspace's own paths. On Windows that is + Git Bash (Git for Windows). WSL resolves a different filesystem namespace, + so run eval-magic inside WSL rather than dispatching into it. Set + EVAL_MAGIC_SH to select a specific sh. EXAMPLES: # Scaffold a first eval and prepare its isolated comparison environments eval-magic init eval-magic run - # run prepares the workspace but does not dispatch. Read the generated - # RUNBOOK.md end to end and follow it through ingest, judges, finalize, and teardown. + # run prepares the workspace but does not dispatch; eval-magic dispatch does. + # Read the generated RUNBOOK.md end to end and follow it through dispatch, + # ingest, judges, finalize, and teardown. # Artifacts land outside the skill's own repository; run prints the path, and # every command it suggests carries --workspace-dir. Set EVAL_MAGIC_WORKSPACE_DIR # to move the default. See: eval-magic docs isolation diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 6a73d2a..1735a51 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -7,7 +7,7 @@ //! - [`commands`] — one thin handler per subcommand, grouped by concern. Each //! maps parsed args onto a library module and renders the result. //! - [`run`] — the `run` orchestrator. This is the bulk of the module: staging, -//! dispatch-task assembly, and the `ingest`/`finalize` chains. It lives here +//! dispatch plan assembly, and the `ingest`/`finalize` chains. It lives here //! rather than in a library module because it is a CLI-shaped workflow — //! it drives the operator hand-off, not just data transformation — and it //! carries its own unit tests (`run/staging/tests/`, `run/dispatch/tests/`, @@ -103,7 +103,7 @@ fn dispatch(command: Option, harness_file: Option<&str>) -> anyhow::Re match command { Commands::Run(args) => run_run(args), - Commands::DispatchTask(args) => run_dispatch_task(args), + Commands::Dispatch(args) => run_dispatch(args), Commands::Ingest(args) => run_ingest(args), Commands::Finalize(args) => run_finalize(args), Commands::Init(args) => run_init(args), diff --git a/src/cli/run/conversation.rs b/src/cli/run/conversation.rs index 3d6472b..f9d052e 100644 --- a/src/cli/run/conversation.rs +++ b/src/cli/run/conversation.rs @@ -1,106 +1,89 @@ -//! Runner-owned execution of one scripted multi-turn dispatch task. +//! Runner-owned execution of one dispatched task. //! -//! The run workspace persists the resolved harness descriptor alongside each -//! task. This driver uses that frozen descriptor to start one native session, -//! gate and deliver each canned user follow-up, and write an ordered -//! `conversation.json` completion artifact for ingest. +//! Given a frozen harness descriptor and one task, this starts a native session, +//! gates and delivers each canned user follow-up a scripted task declares, and +//! writes the ordered `conversation.json` completion artifact ingest reads. A +//! one-shot task takes the same path with no follow-ups to deliver. +//! +//! Loading the plan these tasks come from belongs to [`super::drive`]. use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::time::{Duration, Instant}; use anyhow::{Context, anyhow, bail}; use regex::Regex; -use serde::Deserialize; use crate::adapters::cli_command::shell_quote_arg; -use crate::adapters::descriptor::{finalize_descriptor, subst}; +use crate::adapters::descriptor::subst; use crate::adapters::descriptor_adapter::DescriptorAdapter; use crate::adapters::harness::HarnessAdapter; use crate::adapters::transcript::{TranscriptEvent, TranscriptSummary}; use crate::core::{ ConversationEvent, ConversationRecord, ConversationStatus, ConversationStopReason, DeliverWhen, - ScriptedTurn, posix_shell, validate_agent_environment_entry, + ScriptedTurn, ShellOutcome, run_in_posix_shell, }; use crate::validation::{SchemaName, validate_against_schema}; use super::dispatch::DispatchTask; -#[derive(Debug, Deserialize)] -struct DispatchEnvelope { - #[serde(default)] - guard: bool, - #[serde(default)] - agent_model: Option, - #[serde(default)] - agent_env: BTreeMap, - harness_descriptor: serde_json::Value, - tasks: Vec, +/// How one dispatched task ended. A failure is not represented here — it stays +/// an `Err`, which the batch driver records per task rather than propagating. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TaskOutcome { + Completed { delivered_followups: u32 }, + Stopped { before_followup: u32 }, + TimedOut { round: u32 }, + SkippedExisting, } -/// Execute task `task_index` from a runner-generated dispatch plan. -pub fn command_dispatch_task( - dispatch_path: &Path, - task_index: usize, - overwrite: bool, -) -> anyhow::Result<()> { - let raw = fs::read_to_string(dispatch_path) - .with_context(|| format!("failed to read {}", dispatch_path.display()))?; - let envelope: DispatchEnvelope = serde_json::from_str(&raw) - .with_context(|| format!("failed to parse {}", dispatch_path.display()))?; - for (name, value) in &envelope.agent_env { - validate_agent_environment_entry(name, value).map_err(|message| { - anyhow!( - "invalid agent_env in {}: {message}", - dispatch_path.display() - ) - })?; +impl TaskOutcome { + /// The one-line human summary of this outcome. + pub fn summary(&self) -> String { + match self { + Self::Completed { + delivered_followups: 0, + } => "completed".to_string(), + Self::Completed { + delivered_followups, + } => format!("completed with {delivered_followups} scripted follow-up turn(s)"), + Self::Stopped { before_followup } => { + format!("stopped before scripted follow-up {before_followup}") + } + Self::TimedOut { round } => format!("timed out in round {round}"), + Self::SkippedExisting => "skipped (already complete)".to_string(), + } } - let descriptor = finalize_descriptor( - &envelope.harness_descriptor, - &format!("{}#harness_descriptor", dispatch_path.display()), - )?; - let adapter = DescriptorAdapter::from_descriptor(descriptor); - let task = envelope.tasks.get(task_index).ok_or_else(|| { - anyhow!( - "--task-index {task_index} is out of range for {} task(s)", - envelope.tasks.len() - ) - })?; - run_task( - &adapter, - task, - envelope.guard, - envelope.agent_model.as_deref(), - &envelope.agent_env, - overwrite, - ) } -fn run_task( +/// Execute one task: start a native session, deliver every scripted follow-up +/// whose gate is met, and write the `conversation.json` completion artifact. +pub fn run_task( adapter: &DescriptorAdapter, task: &DispatchTask, guard: bool, agent_model: Option<&str>, agent_env: &BTreeMap, overwrite: bool, -) -> anyhow::Result<()> { - let turns = task - .turns - .as_deref() - .filter(|turns| !turns.is_empty()) - .ok_or_else(|| anyhow!("selected task does not declare scripted follow-up turns"))?; + timeout: Option, +) -> anyhow::Result { + // One budget for the whole task, not per round: a scripted conversation is + // a single dispatch from the operator's point of view. + let deadline = timeout.map(|timeout| Instant::now() + timeout); + // Empty for a one-shot task: the runner drives every dispatch, so the + // follow-up loop below simply has nothing to deliver. + let turns = task.turns.as_deref().unwrap_or_default(); let conversation_path = task .conversation_path .as_deref() .map(PathBuf::from) .ok_or_else(|| anyhow!("multi-turn task is missing conversation_path"))?; + // A finished task is skipped rather than refused: a rerun of `dispatch` + // is how an operator retries the failures in a batch, and the tasks that + // already completed must not be redone on the way. if conversation_path.exists() && !overwrite { - bail!( - "{} already exists; pass --overwrite to rerun this conversation", - conversation_path.display() - ); + return Ok(TaskOutcome::SkippedExisting); } let eval_root = task .eval_root @@ -112,9 +95,18 @@ fn run_task( let initial_template = adapter .cli_exec_command(guard, agent_model, agent_env) .ok_or_else(|| anyhow!("harness declares no initial dispatch command"))?; - let resume_template = adapter - .cli_resume_command(guard, agent_model, agent_env) - .ok_or_else(|| anyhow!("harness declares no native conversation resume command"))?; + // Only a scripted task resumes a session, and a harness may support one-shot + // dispatch without declaring `[conversation]` at all (cline does). Requiring + // the template up front would make those harnesses undispatchable. + let resume_template = if turns.is_empty() { + None + } else { + Some( + adapter + .cli_resume_command(guard, agent_model, agent_env) + .ok_or_else(|| anyhow!("harness declares no native conversation resume command"))?, + ) + }; if overwrite && conversation_path.exists() { fs::remove_file(&conversation_path).with_context(|| { format!( @@ -142,13 +134,31 @@ fn run_task( None, 1, ); - execute_round( + if execute_round( &initial_command, Path::new(eval_root), &first_outputs, agent_env, 1, - )?; + deadline, + )? == RoundOutcome::TimedOut + { + // Turn 1 never answered, so there is no transcript to parse and no + // session to resume. The seeded user message is the whole record. + return write_conversation( + &conversation_path, + base_outputs, + ConversationRecord { + status: ConversationStatus::TimedOut, + delivered_followups: 0, + stop_reason: None, + stopped_before_followup: None, + timed_out_in_round: Some(1), + events, + }, + None, + ); + } let first_summary = parse_round(adapter, &first_outputs, &events_filename, 1)?; let session_id = first_summary .session_id @@ -165,6 +175,7 @@ fn run_task( let mut delivered_followups = 0_u32; let mut stop_reason = None; let mut stopped_before_followup = None; + let mut timed_out_in_round = None; for (index, turn) in turns.iter().enumerate() { let followup = u32::try_from(index + 1).unwrap_or(u32::MAX); @@ -184,8 +195,11 @@ fn run_task( delivered_followups = delivered_followups.saturating_add(1); let round_outputs = base_outputs.join(format!("turn-{round}")); + let resume_template = resume_template + .as_deref() + .expect("a task with turns resolved a resume template above"); let command = render_command( - &resume_template, + resume_template, eval_root, &task.dispatch_prompt_path, &round_outputs, @@ -193,13 +207,18 @@ fn run_task( Some(&turn.prompt), round, ); - execute_round( + if execute_round( &command, Path::new(eval_root), &round_outputs, agent_env, round, - )?; + deadline, + )? == RoundOutcome::TimedOut + { + timed_out_in_round = Some(round); + break; + } let summary = parse_round(adapter, &round_outputs, &events_filename, round)?; if let Some(observed) = summary.session_id.as_deref() && observed != session_id @@ -214,45 +233,62 @@ fn run_task( .expect("append_summary_events requires final_text"); } - let conversation = ConversationRecord { - status: if stop_reason.is_some() { - ConversationStatus::Stopped - } else { - ConversationStatus::Completed - }, - delivered_followups, - stop_reason, - stopped_before_followup, - events, + // A timeout outranks a gate stop: the conversation was cut short, so what + // the last round would have gated on was never observed. + let status = match (timed_out_in_round, stop_reason) { + (Some(_), _) => ConversationStatus::TimedOut, + (None, Some(_)) => ConversationStatus::Stopped, + (None, None) => ConversationStatus::Completed, }; + write_conversation( + &conversation_path, + base_outputs, + ConversationRecord { + status, + delivered_followups, + stop_reason: timed_out_in_round.map_or(stop_reason, |_| None), + stopped_before_followup: timed_out_in_round.map_or(stopped_before_followup, |_| None), + timed_out_in_round, + events, + }, + Some(final_message), + ) +} + +/// Validate, commit, and report one task's completion artifact. `final_message` +/// is absent when no round produced one, which is only possible for a task that +/// timed out before its first answer. +fn write_conversation( + conversation_path: &Path, + base_outputs: &Path, + conversation: ConversationRecord, + final_message: Option, +) -> anyhow::Result { let _: ConversationRecord = validate_against_schema( SchemaName::Conversation, &serde_json::to_value(&conversation)?, &conversation_path.to_string_lossy(), )?; - write_json_atomic(&conversation_path, &conversation)?; + write_json_atomic(conversation_path, &conversation)?; fs::create_dir_all(base_outputs)?; - fs::write( - base_outputs.join("final-message.md"), - format!("{}\n", final_message.trim_end()), - )?; - - match conversation.status { - ConversationStatus::Completed => println!( - "Completed {} scripted follow-up turn(s): {}", - delivered_followups, - conversation_path.display() - ), - ConversationStatus::Stopped => println!( - "Stopped before scripted follow-up {} ({:?}): {}", - conversation.stopped_before_followup.unwrap_or_default(), - conversation - .stop_reason - .expect("stopped conversations have a reason"), - conversation_path.display() - ), + if let Some(final_message) = final_message { + fs::write( + base_outputs.join("final-message.md"), + format!("{}\n", final_message.trim_end()), + )?; } - Ok(()) + + Ok(match conversation.status { + ConversationStatus::Completed => TaskOutcome::Completed { + delivered_followups: conversation.delivered_followups, + }, + ConversationStatus::Stopped => TaskOutcome::Stopped { + before_followup: conversation.stopped_before_followup.unwrap_or_default(), + }, + ConversationStatus::TimedOut => TaskOutcome::TimedOut { + round: conversation.timed_out_in_round.unwrap_or(1), + }, + }) } fn parse_round( @@ -339,6 +375,26 @@ fn unmet_gate( Ok(None) } +/// Render a one-shot dispatch command: the exec template with its task +/// placeholders bound and no session to resume. Judge dispatch uses this too, +/// binding the iteration directory and the judge prompt. +pub fn render_dispatch_command( + template: &str, + eval_root: &str, + dispatch_prompt_path: &str, + outputs_dir: &Path, +) -> String { + render_command( + template, + eval_root, + dispatch_prompt_path, + outputs_dir, + None, + None, + 1, + ) +} + fn render_command( template: &str, eval_root: &str, @@ -364,27 +420,42 @@ fn render_command( ) } +/// Run one round's harness command. `deadline` is the whole task's, not this +/// round's: a scripted conversation is one dispatch from the operator's point +/// of view, so its budget spans every turn it delivers. +/// +/// A round that outruns the deadline returns `Ok(RoundOutcome::TimedOut)`. That +/// is a recorded result, unlike a nonzero exit, which is a failure. fn execute_round( command: &str, eval_root: &Path, outputs_dir: &Path, agent_env: &BTreeMap, round: u32, -) -> anyhow::Result<()> { + deadline: Option, +) -> anyhow::Result { fs::create_dir_all(outputs_dir) .with_context(|| format!("failed to create turn {round} outputs"))?; - let shell = posix_shell().map_err(|message| anyhow!("{message}"))?; - let status = Command::new(shell) - .arg("-c") - .arg(command) - .current_dir(eval_root) - .envs(agent_env) - .status() - .with_context(|| format!("failed to start harness command for turn {round}"))?; - if !status.success() { - bail!("harness command for turn {round} exited with {status}"); + // Saturating: a deadline already passed leaves zero budget, so a turn that + // cannot finish is not begun. + let remaining = deadline.map(|deadline| deadline.saturating_duration_since(Instant::now())); + let outcome = run_in_posix_shell(command, eval_root, agent_env, remaining) + .map_err(|message| anyhow!("turn {round}: {message}"))?; + match outcome { + ShellOutcome::Exited(status) if status.success() => Ok(RoundOutcome::Completed), + ShellOutcome::Exited(status) => { + bail!("harness command for turn {round} exited with {status}") + } + ShellOutcome::TimedOut => Ok(RoundOutcome::TimedOut), } - Ok(()) +} + +/// How one round's harness command ended, once a nonzero exit has been ruled +/// out by [`execute_round`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RoundOutcome { + Completed, + TimedOut, } fn write_json_atomic(path: &Path, value: &impl serde::Serialize) -> anyhow::Result<()> { @@ -461,7 +532,7 @@ mod tests { shell_quote_arg(&events.to_string_lossy()) ); - execute_round(&command, tmp.path(), &outputs, &BTreeMap::new(), 1).unwrap(); + execute_round(&command, tmp.path(), &outputs, &BTreeMap::new(), 1, None).unwrap(); assert_eq!( std::fs::read_to_string(events).unwrap(), diff --git a/src/cli/run/dispatch.rs b/src/cli/run/dispatch.rs index 291ea2e..5661745 100644 --- a/src/cli/run/dispatch.rs +++ b/src/cli/run/dispatch.rs @@ -280,9 +280,10 @@ pub fn build_dispatch_task(opts: &DispatchTaskOpts) -> Result::to_vec), - conversation_path: opts - .turns - .map(|_| artifact_path(&cond_dir.join("conversation.json"))), + // Unconditional: the runner drives every task, so every task ends with + // this completion artifact. Its presence is also what lets a rerun skip + // finished work, which a one-shot task needs as much as a scripted one. + conversation_path: Some(artifact_path(&cond_dir.join("conversation.json"))), agent_description, dispatch_prompt_path: artifact_path(&Path::new(&outputs_dir).join("dispatch-prompt.txt")), outputs_dir, @@ -391,7 +392,8 @@ pub fn get_skill_description(skill_path: &Path) -> String { pub use crate::core::Mode; -/// Harness-specific knobs for the human dispatch manifest. +/// Harness-specific knobs for the human dispatch manifest: what the runner will +/// spawn per task, and under what conditions. #[derive(Debug, Clone, Copy)] pub struct ManifestContext<'a> { pub harness: Harness, @@ -410,6 +412,12 @@ pub fn build_manifest( tasks: &[DispatchTask], context: ManifestContext<'_>, ) -> String { + let ManifestContext { + harness, + guard, + agent_model, + agent_env, + } = context; let mode_str = match mode { Mode::NewSkill => "new-skill", Mode::Revision => "revision", @@ -429,57 +437,43 @@ pub fn build_manifest( String::new(), "In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the task with a short \"read this file and follow it\" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`.".to_string(), String::new(), - // The recipes below are POSIX command lines, so the manifest states the + // Dispatch shells out to POSIX command lines, so the manifest states the // requirement the same way RUNBOOK.md does (issue #248). format!("**Requires:** {POSIX_TOOLING_REQUIREMENT}"), String::new(), ]; - let scripted: Vec = tasks - .iter() - .enumerate() - .filter_map(|(index, task)| task.turns.as_ref().map(|_| index)) - .collect(); - if !scripted.is_empty() { - header.extend([ - "## Scripted multi-turn dispatch".to_string(), - String::new(), - "Run these tasks through eval-magic's conversation driver. It resumes one native \ - session, enforces each delivery gate, and writes the task's conversation.json. A \ - gate stop is valid eval data; a task interrupted before conversation.json is \ - incomplete and ingest skips it." - .to_string(), - String::new(), - ]); - for index in &scripted { - header.push(format!( - "eval-magic dispatch-task --dispatch dispatch.json --task-index {index}" - )); - } - header.push(String::new()); - } - if scripted.len() < tasks.len() - && let Some(lines) = adapter_for(context.harness).cli_manifest_section(CliManifestContext { - guard: context.guard, - agent_model: context.agent_model, - agent_env: context.agent_env, - one_shot_only: !scripted.is_empty(), - }) - { - if !scripted.is_empty() { - header.extend([ - "The harness recipe below applies only to task entries whose `turns` field is \ - absent." - .to_string(), - String::new(), - ]); - } + header.extend([ + "## Dispatch".to_string(), + String::new(), + "Every task is runner-driven — one-shot and scripted alike — so one command runs the \ + whole plan from this iteration directory:" + .to_string(), + String::new(), + "eval-magic dispatch --iteration --harness ".to_string(), + String::new(), + "It runs `--jobs` tasks at a time, each in its own private environment, and writes each \ + task's conversation.json. A task that already has one is skipped, so rerunning retries \ + only what did not finish. A task exceeding `--timeout` is recorded as timed out, and a \ + failing task is recorded while the rest of the batch continues. A conversation that \ + stops at a scripted gate is valid eval data; a task with no conversation.json is \ + incomplete and ingest skips it." + .to_string(), + String::new(), + ]); + // The harness section is what the descriptor still contributes: the command + // the runner will spawn, and whatever is peculiar about reading it back. + if let Some(lines) = adapter_for(harness).cli_manifest_section(CliManifestContext { + guard, + agent_model, + agent_env, + }) { header.extend(lines); } header.extend([ "After all dispatches:".to_string(), String::new(), - "1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ \"total_tokens\": , \"duration_ms\": , \"source\": \"completion-event\" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill.".to_string(), - "2. Dispatch the judge tasks ingest lists, then run `eval-magic finalize` for the benchmark.".to_string(), + "1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/turn-/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ \"total_tokens\": , \"duration_ms\": , \"source\": \"completion-event\" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill.".to_string(), + "2. Run `eval-magic dispatch --judges --harness ` to grade the judge tasks ingest listed, then `eval-magic finalize` for the benchmark.".to_string(), String::new(), "On a harness without persisted transcripts, instead write each task's `run.json` (matching `skills/evaluating-skills/schema/run-record.schema.json`, enforced at runtime by grade/fill-transcripts/detect-stray-writes) and `timing.json` by hand when its subagent returns: carry over `eval_id`, `condition`, `skill_path` (`null` on the without_skill arm), `prompt`, and `files` from the task; populate `final_message` from the subagent's reply; leave `tool_invocations` as `[]`; capture `total_tokens`/`duration_ms` from the task completion event immediately — they may not be persisted anywhere else.".to_string(), String::new(), @@ -754,6 +748,33 @@ mod tests { assert!(out.get("eval_root").is_none()); } + /// Every task is runner-driven, so every task has the completion artifact + /// the driver writes and `dispatch` reads to decide what a rerun may skip. + /// Gating this on `turns` would leave one-shot tasks with no resume marker. + #[test] + fn every_task_carries_a_conversation_path_whether_or_not_it_is_scripted() { + let turns = vec![ScriptedTurn { + prompt: "Use US timezones.".into(), + deliver_when: crate::core::DeliverWhen::AgentAsks, + agent_response_matches: None, + }]; + let scripted = build_dispatch_task(&DispatchTaskOpts { + turns: Some(&turns), + ..base_opts() + }) + .unwrap(); + let one_shot = build_dispatch_task(&base_opts()).unwrap(); + assert_eq!( + scripted.conversation_path.as_deref(), + Some("/tmp/cond/conversation.json") + ); + assert_eq!( + one_shot.conversation_path.as_deref(), + Some("/tmp/cond/conversation.json"), + "a one-shot task needs the same completion artifact" + ); + } + #[test] fn dispatch_prompt_path_under_outputs_dir() { let task = build_dispatch_task(&base_opts()).unwrap(); diff --git a/src/cli/run/dispatch/tests/conversation.rs b/src/cli/run/dispatch/tests/conversation.rs index 58b0e74..5e6c27d 100644 --- a/src/cli/run/dispatch/tests/conversation.rs +++ b/src/cli/run/dispatch/tests/conversation.rs @@ -1,47 +1,14 @@ use super::*; +/// The manifest names one dispatch command whatever the plan holds: scripted +/// and one-shot tasks are both runner-driven, so nothing branches on the mix. #[test] -fn manifest_routes_scripted_tasks_through_the_conversation_driver() { +fn manifest_names_one_dispatch_command_for_scripted_and_one_shot_alike() { let turns = vec![ScriptedTurn { prompt: "Use US timezones.".into(), deliver_when: crate::core::DeliverWhen::AgentAsks, agent_response_matches: None, }]; - let task = build_dispatch_task(&DispatchTaskOpts { - turns: Some(&turns), - ..base_opts() - }) - .unwrap(); - let manifest = build_manifest( - "foo", - Mode::NewSkill, - None, - 1, - "2026-01-01T00:00:00Z", - &[task], - ManifestContext { - harness: Harness::resolve("codex").unwrap(), - guard: false, - agent_model: None, - agent_env: &Default::default(), - }, - ); - assert!(manifest.contains("eval-magic dispatch-task")); - assert!(manifest.contains("--task-index 0")); - assert!(manifest.contains("conversation.json")); - assert!( - !manifest.contains("codex --ask-for-approval never exec --cd"), - "all-scripted manifests must not advertise the one-shot command" - ); -} - -#[test] -fn mixed_manifest_filters_scripted_tasks_out_of_the_one_shot_recipe() { - let turns = vec![ScriptedTurn { - prompt: "Use US timezones.".into(), - deliver_when: crate::core::DeliverWhen::Always, - agent_response_matches: None, - }]; let scripted = build_dispatch_task(&DispatchTaskOpts { turns: Some(&turns), ..base_opts() @@ -49,24 +16,41 @@ fn mixed_manifest_filters_scripted_tasks_out_of_the_one_shot_recipe() { .unwrap(); let one_shot = build_dispatch_task(&base_opts()).unwrap(); - let manifest = build_manifest( - "foo", - Mode::NewSkill, - None, - 1, - "2026-01-01T00:00:00Z", - &[scripted, one_shot], - ManifestContext { - harness: Harness::resolve("codex").unwrap(), - guard: false, - agent_model: None, - agent_env: &Default::default(), - }, - ); - - assert!(manifest.contains("eval-magic dispatch-task")); - assert!( - manifest.contains(".tasks[] | select(.turns == null)"), - "one-shot parallel recipe must exclude scripted tasks: {manifest}" - ); + let manifest = |tasks: &[DispatchTask]| { + build_manifest( + "foo", + Mode::NewSkill, + None, + 1, + "2026-01-01T00:00:00Z", + tasks, + ManifestContext { + harness: Harness::resolve("codex").unwrap(), + guard: false, + agent_model: None, + agent_env: &Default::default(), + }, + ) + }; + for tasks in [ + vec![scripted.clone()], + vec![one_shot.clone()], + vec![scripted, one_shot], + ] { + let rendered = manifest(&tasks); + assert_eq!( + rendered.matches("eval-magic dispatch --iteration").count(), + 1, + "one command, whatever the plan holds: {rendered}" + ); + assert!( + !rendered.contains("dispatch-task"), + "the per-task command is gone: {rendered}" + ); + assert!( + !rendered.contains("select(.turns == null)"), + "nothing filters scripted tasks out of a recipe any more: {rendered}" + ); + assert!(rendered.contains("conversation.json"), "{rendered}"); + } } diff --git a/src/cli/run/drive.rs b/src/cli/run/drive.rs new file mode 100644 index 0000000..8c99c70 --- /dev/null +++ b/src/cli/run/drive.rs @@ -0,0 +1,280 @@ +//! The batch dispatch driver behind `eval-magic dispatch`. +//! +//! `run` prepares a plan; this executes it. One command owns the whole batch — +//! concurrency, per-task failure accounting, and deciding what a rerun may skip +//! — so no operator drives tasks one at a time from a generated recipe. + +pub mod judges; + +use std::collections::BTreeMap; +use std::fs; +use std::path::Path; +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use anyhow::{Context, anyhow}; +use serde::Deserialize; + +use crate::adapters::descriptor::finalize_descriptor; +use crate::adapters::descriptor_adapter::DescriptorAdapter; +use crate::cli::run::conversation::{TaskOutcome, run_task}; +use crate::cli::run::dispatch::DispatchTask; +use crate::core::{posix_shell, validate_agent_environment_entry}; + +/// The run-time half of `dispatch.json`: everything the driver needs to execute +/// a task, frozen at plan time so a dispatch is reproducible from the workspace +/// alone. +#[derive(Debug, Deserialize)] +pub struct DispatchEnvelope { + #[serde(default)] + pub guard: bool, + #[serde(default)] + pub agent_model: Option, + #[serde(default)] + pub agent_env: BTreeMap, + pub harness_descriptor: serde_json::Value, + pub tasks: Vec, +} + +impl DispatchEnvelope { + /// Read and validate a plan. The `agent_env` check happens here rather than + /// per task so a malformed environment fails before anything is dispatched. + pub fn load(dispatch_path: &Path) -> anyhow::Result { + let raw = fs::read_to_string(dispatch_path) + .with_context(|| format!("failed to read {}", dispatch_path.display()))?; + let envelope: Self = serde_json::from_str(&raw) + .with_context(|| format!("failed to parse {}", dispatch_path.display()))?; + for (name, value) in &envelope.agent_env { + validate_agent_environment_entry(name, value).map_err(|message| { + anyhow!( + "invalid agent_env in {}: {message}", + dispatch_path.display() + ) + })?; + } + Ok(envelope) + } + + /// Build an adapter over the frozen descriptor. Each dispatch worker calls + /// this for itself: the adapter is cheap to build from a value both threads + /// only read, which keeps the pool from having to share one. + pub fn adapter(&self, dispatch_path: &Path) -> anyhow::Result { + let descriptor = finalize_descriptor( + &self.harness_descriptor, + &format!("{}#harness_descriptor", dispatch_path.display()), + )?; + Ok(DescriptorAdapter::from_descriptor(descriptor)) + } +} + +/// What one task did, paired with the identity to report it under. Every task +/// produces one of these, including the ones that failed, which is what lets a +/// batch finish rather than abort on the first bad dispatch. +#[derive(Debug)] +pub struct TaskReport { + pub description: String, + pub result: Result, +} + +/// The tally a dispatch prints when it finishes, in task order. +#[derive(Debug, Default)] +pub struct DispatchSummary { + pub reports: Vec, +} + +impl DispatchSummary { + fn count(&self, matching: impl Fn(&TaskOutcome) -> bool) -> usize { + self.reports + .iter() + .filter(|report| report.result.as_ref().is_ok_and(&matching)) + .count() + } + + pub fn completed(&self) -> usize { + self.count(|outcome| matches!(outcome, TaskOutcome::Completed { .. })) + } + + pub fn stopped(&self) -> usize { + self.count(|outcome| matches!(outcome, TaskOutcome::Stopped { .. })) + } + + pub fn skipped(&self) -> usize { + self.count(|outcome| matches!(outcome, TaskOutcome::SkippedExisting)) + } + + pub fn timed_out(&self) -> usize { + self.count(|outcome| matches!(outcome, TaskOutcome::TimedOut { .. })) + } + + pub fn failed(&self) -> usize { + self.reports + .iter() + .filter(|report| report.result.is_err()) + .count() + } + + /// Warnings naming every task that did not produce usable eval data. A + /// gate stop is not one: it is a valid, recorded result. + pub fn warnings(&self) -> Vec { + self.reports + .iter() + .filter_map(|report| match &report.result { + Err(reason) => Some(format!("{} failed: {reason}", report.description)), + Ok(TaskOutcome::TimedOut { round }) => Some(format!( + "{} timed out in round {round}; its environment holds whatever the \ + agent finished before the deadline", + report.description + )), + Ok(_) => None, + }) + .collect() + } + + /// Tasks that produced no usable result. Drives the exit status: a script + /// has to be able to tell a clean batch from one that needs a rerun. + pub fn unusable(&self) -> usize { + self.failed() + self.timed_out() + } + + /// The headline tally line. + pub fn tally(&self) -> String { + format!( + "{} completed, {} stopped, {} timed out, {} failed, {} skipped", + self.completed(), + self.stopped(), + self.timed_out(), + self.failed(), + self.skipped() + ) + } +} + +/// Execute every task in `dispatch_path`, or just the selected indices. +pub fn command_dispatch( + dispatch_path: &Path, + task_indices: &[usize], + overwrite: bool, + timeout: Option, + jobs: usize, +) -> anyhow::Result { + let envelope = DispatchEnvelope::load(dispatch_path)?; + let selected = select_tasks(&envelope, task_indices)?; + // Fail before spawning anything if this host has no shell, and warm the + // process-wide cache so workers only ever read it. + posix_shell().map_err(|message| anyhow!("{message}"))?; + // Every worker builds its own adapter, so a bad descriptor must fail once, + // here, rather than identically in each thread. + envelope.adapter(dispatch_path)?; + + let reports = run_pool(jobs, selected.len(), |slot| { + let adapter = envelope.adapter(dispatch_path)?; + let task = &envelope.tasks[selected[slot]]; + let result = run_task( + &adapter, + task, + envelope.guard, + envelope.agent_model.as_deref(), + &envelope.agent_env, + overwrite, + timeout, + ) + // A failed task is data about the campaign, not a reason to abandon the + // rest of it. `{:#}` keeps anyhow's context chain in the reported reason. + .map_err(|error| format!("{error:#}")); + Ok(TaskReport { + description: task.agent_description.clone(), + result, + }) + })?; + Ok(DispatchSummary { reports }) +} + +/// Run `total` units of work across at most `jobs` threads, returning what each +/// produced in slot order — which worker finished first is not something a +/// report should depend on. +/// +/// `work` is called once per slot and may run on any thread. An `Err` from it +/// aborts the batch: it means the runner itself could not proceed (a bad +/// descriptor, an unreachable shell), as distinct from a task that failed, +/// which `work` reports inside its own return value. +pub(crate) fn run_pool(jobs: usize, total: usize, work: F) -> anyhow::Result> +where + T: Send + Describe, + F: Fn(usize) -> anyhow::Result + Sync, +{ + let cursor = AtomicUsize::new(0); + let done = AtomicUsize::new(0); + let slots: Mutex>> = Mutex::new((0..total).map(|_| None).collect()); + + std::thread::scope(|scope| -> anyhow::Result<()> { + // Never more threads than there is work for them to do. + let workers: Vec<_> = (0..jobs.min(total).max(1)) + .map(|_| { + let (cursor, done, slots, work) = (&cursor, &done, &slots, &work); + scope.spawn(move || -> anyhow::Result<()> { + loop { + let slot = cursor.fetch_add(1, Ordering::Relaxed); + if slot >= total { + return Ok(()); + } + let produced = work(slot)?; + // One lock covers the progress line and the slot write, + // so concurrent workers cannot interleave mid-line. + let mut slots = slots.lock().expect("dispatch pool lock"); + let finished = done.fetch_add(1, Ordering::Relaxed) + 1; + println!("[{finished}/{total}] {}", produced.describe()); + slots[slot] = Some(produced); + } + }) + }) + .collect(); + // Joined explicitly: dropping the handles would discard a worker's + // error, leaving a short batch that looks like a successful one. + for worker in workers { + worker + .join() + .map_err(|_| anyhow!("a dispatch worker panicked"))??; + } + Ok(()) + })?; + + Ok(slots + .into_inner() + .expect("dispatch pool lock") + .into_iter() + .flatten() + .collect()) +} + +/// The one-line progress text a pooled unit of work reports when it finishes. +pub(crate) trait Describe { + fn describe(&self) -> String; +} + +impl Describe for TaskReport { + fn describe(&self) -> String { + match &self.result { + Ok(outcome) => format!("{}: {}", self.description, outcome.summary()), + // The reason follows as a ⚠ line, so this stays scannable. + Err(_) => format!("{}: failed", self.description), + } + } +} + +/// The task indices to run: every task by default, or exactly those requested. +/// An out-of-range index is an error before anything is dispatched, so a typo +/// cannot half-run a batch. +fn select_tasks(envelope: &DispatchEnvelope, requested: &[usize]) -> anyhow::Result> { + if requested.is_empty() { + return Ok((0..envelope.tasks.len()).collect()); + } + let total = envelope.tasks.len(); + for index in requested { + anyhow::ensure!( + *index < total, + "--task-index {index} is out of range for {total} task(s)" + ); + } + Ok(requested.to_vec()) +} diff --git a/src/cli/run/drive/judges.rs b/src/cli/run/drive/judges.rs new file mode 100644 index 0000000..11924c7 --- /dev/null +++ b/src/cli/run/drive/judges.rs @@ -0,0 +1,199 @@ +//! Runner-driven judge dispatch: the `--judges` half of `eval-magic dispatch`. +//! +//! A judge task is a one-shot dispatch like any other, so it reuses the +//! harness's `exec_template` with its placeholders bound differently — the +//! iteration directory instead of a private task env, the judge prompt instead +//! of the task prompt, and a per-task capture directory derived from the +//! response path. A harness therefore needs no judge-specific template: what +//! makes a judge a judge is the prompt `grade` wrote, not the command line. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::Context; +use serde::Deserialize; + +use crate::adapters::descriptor_adapter::DescriptorAdapter; +use crate::adapters::harness::HarnessAdapter; +use crate::cli::run::conversation::render_dispatch_command; +use crate::cli::run::drive::DispatchEnvelope; +use crate::core::{ShellOutcome, posix_shell, run_in_posix_shell}; + +use super::{Describe, run_pool}; + +/// The parts of `judge-tasks.json` a dispatch needs. +#[derive(Debug, Deserialize)] +struct JudgeTasksFile { + #[serde(default)] + tasks: Vec, +} + +#[derive(Debug, Deserialize)] +struct JudgeTask { + eval_id: String, + condition: String, + assertion_id: String, + #[serde(default)] + model: Option, + response_path: String, + dispatch_prompt_path: String, +} + +impl JudgeTask { + fn description(&self) -> String { + format!("{}:{}:{}", self.eval_id, self.condition, self.assertion_id) + } + + /// A verdict is present once its response file exists and is non-empty — + /// the same test the shipped recipe applied with `[ -s "$response_path" ]`. + fn verdict_present(&self) -> bool { + std::fs::metadata(&self.response_path).is_ok_and(|meta| meta.len() > 0) + } + + /// This task's private capture directory: the response path minus its + /// `.json` suffix. Several assertions share one `judge-responses/` + /// directory, so binding captures there would have each judge overwrite the + /// previous one's transcript. + fn capture_dir(&self) -> PathBuf { + Path::new(&self.response_path).with_extension("") + } +} + +/// One judge dispatch's result, reported under the assertion it graded. +#[derive(Debug)] +struct JudgeReport { + description: String, + result: Result<(), String>, +} + +impl Describe for JudgeReport { + fn describe(&self) -> String { + match &self.result { + Ok(()) => format!("{}: dispatched", self.description), + Err(_) => format!("{}: failed", self.description), + } + } +} + +/// What a judge batch did. The verdict counts drive the exit status: a judge +/// batch is finished only once every verdict is on disk. +#[derive(Debug, Default)] +pub struct JudgeSummary { + pub total: usize, + pub present: usize, + pub dispatched: usize, + pub skipped: usize, + pub failures: Vec, +} + +impl JudgeSummary { + /// `N/M verdicts present` — the sentence the shipped recipe printed, kept + /// word for word so an operator reads the same thing either way. + pub fn verdict_line(&self) -> String { + format!("{}/{} verdicts present", self.present, self.total) + } + + /// Whether every judge task has a verdict and nothing failed on the way. + pub fn complete(&self) -> bool { + self.present == self.total && self.failures.is_empty() + } +} + +/// Dispatch every judge task that has no verdict yet. +pub fn command_dispatch_judges( + iteration_dir: &Path, + overwrite: bool, + timeout: Option, + jobs: usize, +) -> anyhow::Result { + let judge_path = iteration_dir.join("judge-tasks.json"); + let raw = std::fs::read_to_string(&judge_path).with_context(|| { + format!( + "failed to read {} — run `eval-magic ingest` to emit judge tasks first", + judge_path.display() + ) + })?; + let file: JudgeTasksFile = serde_json::from_str(&raw) + .with_context(|| format!("failed to parse {}", judge_path.display()))?; + + // A judge runs against the harness the campaign was planned with, so the + // descriptor comes from the same frozen envelope the eval tasks used. + let dispatch_path = iteration_dir.join("dispatch.json"); + let envelope = DispatchEnvelope::load(&dispatch_path)?; + // Both fail once, here, rather than identically inside every worker. + posix_shell().map_err(|message| anyhow::anyhow!("{message}"))?; + envelope.adapter(&dispatch_path)?; + + let pending: Vec<&JudgeTask> = file + .tasks + .iter() + .filter(|task| overwrite || !task.verdict_present()) + .collect(); + let mut summary = JudgeSummary { + total: file.tasks.len(), + skipped: file.tasks.len() - pending.len(), + ..JudgeSummary::default() + }; + + let reports = run_pool(jobs, pending.len(), |slot| { + let task = pending[slot]; + let adapter = envelope.adapter(&dispatch_path)?; + Ok(JudgeReport { + description: task.description(), + result: dispatch_judge(&adapter, task, iteration_dir, &envelope.agent_env, timeout), + }) + })?; + + summary.dispatched = reports.len(); + summary.failures = reports + .into_iter() + .filter_map(|report| { + report + .result + .err() + .map(|reason| format!("{}: {reason}", report.description)) + }) + .collect(); + // Counted from disk rather than from what was dispatched: a judge that ran + // without writing its verdict has not produced one. + summary.present = file + .tasks + .iter() + .filter(|task| task.verdict_present()) + .count(); + Ok(summary) +} + +/// Run one judge task through the harness's exec template. +fn dispatch_judge( + adapter: &DescriptorAdapter, + task: &JudgeTask, + iteration_dir: &Path, + agent_env: &BTreeMap, + timeout: Option, +) -> Result<(), String> { + // Guard arguments are deliberately off: a judge runs outside every guarded + // task env, and the hook-trust bypass exists only for eval-agent dispatches + // whose cwd actually contains the vetted guard hook. + let template = adapter + .cli_exec_command(false, task.model.as_deref(), agent_env) + .ok_or_else(|| "harness declares no dispatch exec command".to_string())?; + + let capture_dir = task.capture_dir(); + std::fs::create_dir_all(&capture_dir) + .map_err(|error| format!("failed to create {}: {error}", capture_dir.display()))?; + + let command = render_dispatch_command( + &template, + &iteration_dir.to_string_lossy(), + &task.dispatch_prompt_path, + &capture_dir, + ); + match run_in_posix_shell(&command, iteration_dir, agent_env, timeout) { + Ok(ShellOutcome::Exited(status)) if status.success() => Ok(()), + Ok(ShellOutcome::Exited(status)) => Err(format!("judge command exited with {status}")), + Ok(ShellOutcome::TimedOut) => Err("judge command timed out".to_string()), + Err(message) => Err(message), + } +} diff --git a/src/cli/run/golden_tests.rs b/src/cli/run/golden_tests.rs index 09cef57..ae62aeb 100644 --- a/src/cli/run/golden_tests.rs +++ b/src/cli/run/golden_tests.rs @@ -13,7 +13,7 @@ use std::fs; use std::path::{Path, PathBuf}; use std::sync::LazyLock; -use crate::adapters::{CliDispatchContext, CliJudgeContext, adapter_for}; +use crate::adapters::{CliDispatchContext, adapter_for}; use crate::core::{AvailableSkill, Harness, Mode}; use super::dispatch::{DispatchTaskOpts, ManifestContext, build_dispatch_task, build_manifest}; @@ -98,7 +98,7 @@ fn bare_task(harness: Harness) -> DispatchTaskOpts<'static> { } } -fn render_manifest(harness: Harness, guard: bool, agent_model: Option<&str>) -> String { +fn render_manifest(harness: Harness) -> String { let slug = adapter_for(harness).staged_slug("slow-powers-eval-", 2, "with_skill", "widget-skill"); let mut staged = staged_task(harness); @@ -116,8 +116,8 @@ fn render_manifest(harness: Harness, guard: bool, agent_model: Option<&str>) -> &tasks, ManifestContext { harness, - guard, - agent_model, + guard: true, + agent_model: Some("model-x"), agent_env: empty_env(), }, ) @@ -137,39 +137,27 @@ fn golden_runbook_per_harness() { cond_a: "old_skill", cond_b: "new_skill", num_tasks: 6, - multi_turn_tasks: 0, target_args: " --skill-dir /tmp/skills --skill widget-skill", - guard: true, - agent_model: Some("model-x"), - agent_env: empty_env(), }); assert_golden(&format!("{label}/runbook.golden.md"), &book); } } +/// One golden per harness: the manifest quotes the exec command but renders no +/// conditional recipe, so guard state and model selection cannot change it. +/// What stays harness-specific is the dispatch prompt each task carries, pinned +/// by `golden_dispatch_prompt_per_harness`. #[test] fn golden_manifest_per_harness() { for harness in Harness::known() { let label = adapter_for(harness).label(); - let manifest = render_manifest(harness, true, Some("model-x")); - assert_golden(&format!("{label}/manifest.golden.md"), &manifest); + assert_golden( + &format!("{label}/manifest.golden.md"), + &render_manifest(harness), + ); } } -#[test] -fn golden_manifest_codex_without_guard() { - // Pins the hook-trust conditional: no --dangerously-bypass-hook-trust. - let manifest = render_manifest(Harness::resolve("codex").unwrap(), false, Some("model-x")); - assert_golden("codex/manifest-noguard.golden.md", &manifest); -} - -#[test] -fn golden_manifest_claude_without_model() { - // Pins the empty model-arg rendering. - let manifest = render_manifest(Harness::resolve("claude-code").unwrap(), true, None); - assert_golden("claude-code/manifest-nomodel.golden.md", &manifest); -} - #[test] fn golden_dispatch_prompt_per_harness() { for harness in Harness::known() { @@ -210,80 +198,28 @@ fn golden_guard_armed_message_per_harness() { } } +/// The post-run hand-off names one runner command, so the agent model cannot +/// change it. Pinned per harness, and asserted invariant across model +/// selection. #[test] -fn golden_judge_recipe_per_harness() { - for (harness, guard, rel) in [ - ( - Harness::resolve("claude-code").unwrap(), - true, - "claude-code/judge-recipe.golden.md", - ), - ( - Harness::resolve("codex").unwrap(), - true, - "codex/judge-recipe.golden.md", - ), - // Cline has no guard, so one variant covers both guard states. - ( - Harness::resolve("cline").unwrap(), - false, - "cline/judge-recipe.golden.md", - ), - // Pins the hook-trust conditional in the judge command line. - ( - Harness::resolve("codex").unwrap(), - false, - "codex/judge-recipe-noguard.golden.md", - ), - // OpenCode has no guard args, so one variant covers both guard states. - ( - Harness::resolve("opencode").unwrap(), - false, - "opencode/judge-recipe.golden.md", - ), - ] { - let recipe = adapter_for(harness) - .cli_judge_next_steps(CliJudgeContext { - guard, - iteration_dir: Path::new("/work/iter-1"), - }) - .expect("judge recipe is wired for this harness"); - assert_golden(rel, &recipe); - } -} - -#[test] -fn golden_cline_next_steps_with_and_without_model() { - for (agent_model, rel) in [ - (Some("model-x"), "cline/next-steps-model.golden.txt"), - (None, "cline/next-steps-nomodel.golden.txt"), - ] { - let steps = - adapter_for(Harness::resolve("cline").unwrap()).cli_next_steps(CliDispatchContext { +fn golden_next_steps_per_harness_do_not_vary_with_the_model() { + for harness in ["cline", "opencode"] { + let adapter = adapter_for(Harness::resolve(harness).unwrap()); + let steps = |agent_model| { + adapter.cli_next_steps(CliDispatchContext { guard: false, target_args: " --skill-dir /tmp/skills --skill widget-skill", iteration: 2, agent_model, agent_env: empty_env(), - }); - assert_golden(rel, &steps); - } -} - -#[test] -fn golden_opencode_next_steps_with_and_without_model() { - for (agent_model, rel) in [ - (Some("model-x"), "opencode/next-steps-model.golden.txt"), - (None, "opencode/next-steps-nomodel.golden.txt"), - ] { - let steps = - adapter_for(Harness::resolve("opencode").unwrap()).cli_next_steps(CliDispatchContext { - guard: false, - target_args: " --skill-dir /tmp/skills --skill widget-skill", - iteration: 2, - agent_model, - agent_env: empty_env(), - }); - assert_golden(rel, &steps); + }) + }; + let with_model = steps(Some("model-x")); + assert_eq!( + with_model, + steps(None), + "{harness}: dispatch guidance must not depend on the model" + ); + assert_golden(&format!("{harness}/next-steps.golden.txt"), &with_model); } } diff --git a/src/cli/run/mod.rs b/src/cli/run/mod.rs index 1200c20..e9ecc85 100644 --- a/src/cli/run/mod.rs +++ b/src/cli/run/mod.rs @@ -3,7 +3,7 @@ //! Split into focused sub-orchestrators: //! //! - [`staging`] — staged-skill lifecycle (install/cleanup + sibling manifest). -//! - [`dispatch`] — dispatch-task and prompt assembly (`dispatch.json`). +//! - [`dispatch`] — dispatch task and prompt assembly (`dispatch.json`). //! - [`steps`] — the `ingest` / `finalize` fixed-order chains. //! - [`orchestrate`] — `command_run`, the top-level orchestrator. //! @@ -12,6 +12,7 @@ pub mod conversation; pub mod dispatch; +pub mod drive; pub mod fixtures; #[cfg(test)] mod golden_tests; diff --git a/src/cli/run/orchestrate/build.rs b/src/cli/run/orchestrate/build.rs index e9a71f8..43dd66f 100644 --- a/src/cli/run/orchestrate/build.rs +++ b/src/cli/run/orchestrate/build.rs @@ -288,7 +288,10 @@ pub(super) fn write_dispatch( .expect("dispatch envelope is an object") .insert("agent_env".to_string(), json!(conditions.agent_env)); } - if r.selected_evals.iter().any(|eval| eval.turns.is_some()) { + // Unconditional: `dispatch` drives every task from this envelope, so the + // descriptor it freezes and the guard state it dispatches under are needed + // whether or not any eval declares scripted turns. + { let descriptor = crate::adapters::registry::descriptor_value_for(ctx.harness); let envelope = dispatch_json .as_object_mut() @@ -356,11 +359,7 @@ pub(super) fn write_dispatch( cond_a: r.cond_a, cond_b: r.cond_b, num_tasks: tasks.len(), - multi_turn_tasks: tasks.iter().filter(|task| task.turns.is_some()).count(), target_args: &target_args, - guard: opts.guard_armed(), - agent_model: opts.agent_model, - agent_env: &opts.agent_env, }); fs::write(r.iteration_dir.join("RUNBOOK.md"), runbook)?; diff --git a/src/cli/run/orchestrate/mod.rs b/src/cli/run/orchestrate/mod.rs index 08f79ec..0a707f2 100644 --- a/src/cli/run/orchestrate/mod.rs +++ b/src/cli/run/orchestrate/mod.rs @@ -409,23 +409,8 @@ fn print_next_steps(ctx: &RunContext, opts: &RunOptions, r: &Resolved, num_tasks return; } let target_args = command_target_args(ctx); - if r.selected_evals.iter().any(|eval| eval.turns.is_some()) { - let mix = r.selected_evals.iter().any(|eval| eval.turns.is_none()); - println!( - "\nNext: read RUNBOOK.md and run every task with scripted `turns` through \ - `eval-magic dispatch-task` so follow-ups resume the same native session.{} \ - Then run `eval-magic ingest{target_args} --iteration {} --harness {}`.", - if mix { - " Use its harness recipe only for the remaining one-shot tasks." - } else { - "" - }, - r.iteration, - adapter_for(ctx.harness).label() - ); - return; - } - // One-shot CLI dispatch; the exact command is harness-specific. + // One command whatever the plan holds: scripted and one-shot tasks are both + // runner-driven. println!( "{}", adapter_for(ctx.harness).cli_next_steps(CliDispatchContext { diff --git a/src/cli/run/orchestrate/shell.rs b/src/cli/run/orchestrate/shell.rs index 621c60c..45598ab 100644 --- a/src/cli/run/orchestrate/shell.rs +++ b/src/cli/run/orchestrate/shell.rs @@ -1,57 +1,36 @@ -//! Host-tooling preflight for `run`: does this machine have what the generated -//! recipes ask an operator to paste? +//! Host-shell preflight for `run`: can this machine dispatch what it is about +//! to prepare? //! //! Unlike [`super::git`], a missing shell is a warning rather than an error. //! `run` never dispatches — it prepares a workspace, and that workspace is //! correct whatever shell prepared it, so this reports the gap and lets the run -//! finish. +//! finish. `dispatch` is where the absence becomes fatal. //! -//! The shell that eventually dispatches still has to resolve the paths this host -//! wrote into the recipes, which keeps the gap host-local: Git Bash shares the -//! Windows filesystem, WSL resolves its own. See [`POSIX_TOOLING_REQUIREMENT`] -//! for the declared rule the warnings below defer to. +//! The gap is host-local: `dispatch` spawns each harness command line with the +//! workspace's own absolute paths, so the shell it resolves has to resolve +//! those. Git Bash shares the Windows filesystem; WSL resolves its own. See +//! [`POSIX_TOOLING_REQUIREMENT`] for the declared rule the warning defers to. use std::path::Path; -use crate::core::{ - POSIX_RECIPE_TOOLS, POSIX_TOOLING_REQUIREMENT, posix_shell, require_posix_toolchain, -}; +use crate::core::posix_shell; -/// Warnings for a host that cannot run the recipes this run is about to -/// generate. Empty on a complete host. +/// Warnings for a host that cannot dispatch the run it is about to prepare. +/// Empty on a complete host. pub(super) fn preflight_posix_tooling() -> Vec { - let shell = posix_shell(); - // Only probe for tools once a shell exists: `require_posix_toolchain` - // resolves the same shell first, and reporting one failure beats reporting - // the same absence twice in different words. - let missing = match shell { - Ok(_) => require_posix_toolchain(POSIX_RECIPE_TOOLS).err(), - Err(_) => None, - }; - tooling_warning(shell, missing.as_deref()) - .into_iter() - .collect() + tooling_warning(posix_shell()).into_iter().collect() } -/// The operator warning for a host that cannot run the generated recipes, or -/// `None` when it can — a healthy run stays silent. +/// The operator warning for a host with no POSIX shell, or `None` when one +/// resolves — a healthy run stays silent. /// -/// Two distinguishable failures. `shell` is `Err` when no `sh` exists at all, -/// and its message already carries [`POSIX_TOOLING_REQUIREMENT`], so the warning -/// only adds that the prepared workspace survives. `missing` is the tool absent -/// from a shell that *was* found — the Git for Windows case, which resolves a -/// shell but bundles no `jq` — and needs the requirement appended. -fn tooling_warning(shell: Result<&Path, &str>, missing: Option<&str>) -> Option { - if let Err(reason) = shell { - return Some(format!( - "{reason} The workspace and recipes below are still correct — dispatch them from a \ - POSIX shell on this host." - )); - } - let reason = missing?; +/// `posix_shell`'s own message already carries [`POSIX_TOOLING_REQUIREMENT`], +/// so the warning only adds that the prepared workspace survives the gap. +fn tooling_warning(shell: Result<&Path, &str>) -> Option { + let reason = shell.err()?; Some(format!( - "{reason}. The parallel-dispatch and judge recipes in RUNBOOK.md are pipelines over that \ - toolchain and cannot run without it. {POSIX_TOOLING_REQUIREMENT}" + "{reason} The workspace below is still correct — dispatch it from a POSIX shell on \ + this host." )) } @@ -64,7 +43,7 @@ mod tests { /// A complete host stays silent — the preflight must not nag the common case. #[test] fn a_complete_host_produces_no_warning() { - assert_eq!(tooling_warning(Ok(Path::new("/bin/sh")), None), None); + assert_eq!(tooling_warning(Ok(Path::new("/bin/sh"))), None); } /// No shell at all: `posix_shell`'s own message already carries the declared @@ -72,47 +51,22 @@ mod tests { /// workspace is still correct. #[test] fn a_missing_shell_warns_with_the_declared_requirement() { - let warning = tooling_warning(Err("no POSIX shell found. Use Git Bash or WSL."), None) + let warning = tooling_warning(Err("no POSIX shell found. Use Git Bash or WSL.")) .expect("a host with no POSIX shell must be told"); assert!(warning.contains("no POSIX shell found"), "{warning}"); assert!(warning.contains("Git Bash"), "{warning}"); } - /// An unqualified "dispatch them from a POSIX shell" reads as an invitation - /// to prepare here and dispatch from WSL — the one split + /// An unqualified "dispatch it from a POSIX shell" reads as an invitation to + /// prepare here and dispatch from WSL — the one split /// [`POSIX_TOOLING_REQUIREMENT`] rules out, and the one that fails quietly. #[test] fn a_missing_shell_confines_dispatch_to_the_host_that_prepared_the_workspace() { - let warning = tooling_warning(Err("no POSIX shell found. Use Git Bash."), None) + let warning = tooling_warning(Err("no POSIX shell found. Use Git Bash.")) .expect("a host with no POSIX shell must be told"); assert!( warning.contains("this host"), "the warning must keep dispatch on the preparing host: {warning}" ); } - - /// A shell that is missing a recipe tool names the tool and the shell whose - /// PATH was searched, and still points at the declared requirement — Git for - /// Windows resolves a shell but bundles no `jq`, which is exactly this case. - #[test] - fn a_missing_recipe_tool_warns_naming_the_tool_and_the_requirement() { - let warning = tooling_warning( - Ok(Path::new("/opt/Git/usr/bin/sh.exe")), - Some("jq is not on the PATH of /opt/Git/usr/bin/sh.exe"), - ) - .expect("a shell without `jq` must be told"); - assert!(warning.contains("jq is not on the PATH"), "{warning}"); - assert!(warning.contains("/opt/Git/usr/bin/sh.exe"), "{warning}"); - assert!(warning.contains("RUNBOOK.md"), "{warning}"); - assert!(warning.contains("Git Bash"), "{warning}"); - } - - /// A resolved shell wins over a stale tool reason: the shell branch is only - /// reachable when discovery itself failed. - #[test] - fn the_shell_failure_takes_precedence() { - let warning = tooling_warning(Err("no POSIX shell found."), Some("jq is missing")) - .expect("a missing shell warns"); - assert!(!warning.contains("jq is missing"), "{warning}"); - } } diff --git a/src/cli/run/runbook.rs b/src/cli/run/runbook.rs index 93fd024..cad23bc 100644 --- a/src/cli/run/runbook.rs +++ b/src/cli/run/runbook.rs @@ -10,10 +10,9 @@ //! placeholders the renderer fills with run-specific values. The generated //! `RUNBOOK.md` itself is a workspace artifact and is not version controlled. -use std::collections::BTreeMap; use std::path::Path; -use crate::adapters::{CliDispatchContext, CliJudgeContext, RUNBOOK_TEMPLATE, adapter_for}; +use crate::adapters::RUNBOOK_TEMPLATE; use crate::core::fs::artifact_path; use crate::core::{Harness, Mode, POSIX_TOOLING_REQUIREMENT}; @@ -32,13 +31,9 @@ pub(crate) struct RunbookContext<'a> { pub cond_a: &'a str, pub cond_b: &'a str, pub num_tasks: usize, - pub multi_turn_tasks: usize, /// The self-sufficient `--skill-dir … --skill …` selector (leading space), /// from [`command_target_args`](crate::cli::command_target_args). pub target_args: &'a str, - pub guard: bool, - pub agent_model: Option<&'a str>, - pub agent_env: &'a BTreeMap, } /// Render `RUNBOOK.md` for a run: fill the shared runbook template's @@ -47,7 +42,6 @@ pub(crate) struct RunbookContext<'a> { /// runbook stays in lockstep with `dispatch-manifest.md` and the printed next /// steps; pipeline commands carry `--harness`. pub(crate) fn build_runbook(ctx: &RunbookContext) -> String { - let adapter = adapter_for(ctx.harness); let template = RUNBOOK_TEMPLATE; let iteration = ctx.iteration.to_string(); @@ -70,64 +64,30 @@ pub(crate) fn build_runbook(ctx: &RunbookContext) -> String { ("POSIX_REQUIREMENT", POSIX_TOOLING_REQUIREMENT), ]; - // A human pastes commands. The harness-specific dispatch + judge recipes come - // from the adapter's CLI generators, so the runbook stays in lockstep with - // `dispatch-manifest.md` and the printed next steps; pipeline commands carry - // `--harness`. Owners outlive the `render` call below. + // One command per phase: the runner drives every dispatch, so nothing here + // varies by harness beyond the `--harness` selector itself. let label = harness_label(ctx.harness); - let one_shot_recipe = adapter.cli_next_steps(CliDispatchContext { - guard: ctx.guard, - target_args: ctx.target_args, - iteration: ctx.iteration, - agent_model: ctx.agent_model, - agent_env: ctx.agent_env, - }); - let dispatch_recipe = if ctx.multi_turn_tasks == 0 { - one_shot_recipe - } else { - let driver = format!( - "Scripted tasks must run through eval-magic's conversation driver so every follow-up \ - resumes the same native session and produces a schema-validated \ - `conversation.json`. From this iteration directory:\n\n```bash\n\ - JOBS=${{JOBS:-4}}\n\ - jq -r '.tasks | to_entries[] | select(.value.turns != null) | .key' \ - \"{dispatch_json}\" | \\\n xargs -P \"$JOBS\" -n 1 eval-magic dispatch-task \ - --dispatch \"{dispatch_json}\" --task-index\n```\n\n\ - A normal guardrail stop (`agent_did_not_ask` or `agent_response_mismatch`) is valid \ - completed eval data; an interrupted task has no `conversation.json` and ingest \ - skips it." - ); - if ctx.multi_turn_tasks == ctx.num_tasks { - format!( - "{driver}\n\nThen run `eval-magic ingest{} --iteration {} --harness {label}`.", - ctx.target_args, ctx.iteration - ) - } else { - format!( - "{driver}\n\nFor the remaining task entries whose `turns` field is absent, use \ - the one-shot harness recipe below (do not use it for scripted tasks):\n\ - {one_shot_recipe}" - ) - } - }; - let judge_recipe = adapter - .cli_judge_next_steps(CliJudgeContext { - guard: ctx.guard, - iteration_dir: ctx.iteration_dir, - }) - .unwrap_or_else(|| { - "Dispatch each judge task `ingest` listed through the same harness CLI, \ - capturing its transcript output, then finalize." - .to_string() - }); + let dispatch_cmd = format!( + "eval-magic dispatch{} --iteration {} --harness {label}", + ctx.target_args, ctx.iteration + ); + let ingest_cmd = format!( + "eval-magic ingest{} --iteration {} --harness {label}", + ctx.target_args, ctx.iteration + ); + let judge_cmd = format!( + "eval-magic dispatch --judges{} --iteration {} --harness {label}", + ctx.target_args, ctx.iteration + ); let finalize_cmd = format!( "eval-magic finalize{} --iteration {} --harness {label}", ctx.target_args, ctx.iteration ); let teardown_cmd = format!("eval-magic teardown{} --harness {label}", ctx.target_args); vars.push(("HARNESS", &label)); - vars.push(("DISPATCH_RECIPE", &dispatch_recipe)); - vars.push(("JUDGE_RECIPE", &judge_recipe)); + vars.push(("DISPATCH_CMD", &dispatch_cmd)); + vars.push(("INGEST_CMD", &ingest_cmd)); + vars.push(("JUDGE_CMD", &judge_cmd)); vars.push(("FINALIZE_CMD", &finalize_cmd)); vars.push(("TEARDOWN_CMD", &teardown_cmd)); @@ -174,12 +134,6 @@ fn render(template: &str, vars: &[(&str, &str)]) -> String { mod tests { use super::*; use std::path::PathBuf; - use std::sync::LazyLock; - - fn empty_env() -> &'static BTreeMap { - static EMPTY: LazyLock> = LazyLock::new(BTreeMap::new); - &EMPTY - } #[test] fn runbook_is_human_followed_cli_recipe() { @@ -193,11 +147,7 @@ mod tests { cond_a: "old_skill", cond_b: "new_skill", num_tasks: 6, - multi_turn_tasks: 0, target_args: " --skill-dir /tmp/skills --skill widget-skill", - guard: false, - agent_model: Some("gpt-5-mini"), - agent_env: empty_env(), }; let book = build_runbook(&ctx); @@ -215,11 +165,15 @@ mod tests { "frames the run for a human at a terminal: {book}" ); - // The CLI dispatch recipe comes from the Codex adapter; pipeline commands - // carry --harness codex so they are copy-pasteable. + // Every phase is a runner command carrying --harness codex, so the whole + // runbook is copy-pasteable without knowing the harness's own CLI. assert!( - book.contains("codex --ask-for-approval never exec"), - "carries the Codex CLI dispatch recipe: {book}" + book.contains("eval-magic dispatch --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness codex"), + "carries the dispatch command: {book}" + ); + assert!( + book.contains("eval-magic dispatch --judges --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness codex"), + "carries the judge dispatch command: {book}" ); assert!( book.contains("eval-magic finalize --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness codex"), @@ -267,10 +221,12 @@ mod tests { assert_eq!(out, "value-with-{{B}}-inside second"); } + /// The runbook reads the same whether or not the plan holds scripted turns: + /// the runner drives both, so there is nothing to branch on. #[test] - fn all_scripted_runbook_uses_dispatch_task_instead_of_one_shot_recipe() { + fn a_scripted_plan_reads_the_same_as_a_one_shot_plan() { let dir = PathBuf::from("/work/.eval-magic/widget-skill/iteration-2"); - let book = build_runbook(&RunbookContext { + let context = |num_tasks: usize| RunbookContext { harness: Harness::resolve("codex").unwrap(), skill_name: "widget-skill", iteration: 2, @@ -278,18 +234,20 @@ mod tests { mode: Mode::NewSkill, cond_a: "with_skill", cond_b: "without_skill", - num_tasks: 4, - multi_turn_tasks: 4, + num_tasks, target_args: " --skill /tmp/widget-skill", - guard: false, - agent_model: None, - agent_env: empty_env(), - }); - assert!(book.contains("eval-magic dispatch-task")); + }; + let book = build_runbook(&context(4)); + assert!(book.contains("eval-magic dispatch --skill /tmp/widget-skill")); assert!(book.contains("conversation.json")); assert!( !book.contains("--output-last-message /final-message.md"), - "all-scripted runs must not advertise the one-shot eval command" + "the runbook no longer carries a harness CLI recipe: {book}" + ); + // Only the dispatch count differs between plans. + assert_eq!( + build_runbook(&context(4)).replace("**Dispatches:** 4", "**Dispatches:** 6"), + build_runbook(&context(6)) ); } } diff --git a/src/cli/run/util.rs b/src/cli/run/util.rs index 89f4a95..1e2dcf4 100644 --- a/src/cli/run/util.rs +++ b/src/cli/run/util.rs @@ -215,9 +215,9 @@ pub(crate) fn harness_run_preflight<'a>( } if !adapter.has_dispatch_recipes() { warnings.push(format!( - "--harness {label} declares no dispatch exec recipe — RUNBOOK.md and \ - dispatch-manifest.md carry handoff guidance without a copy-pasteable per-task \ - command; construct each dispatch through the harness's one-shot CLI yourself." + "--harness {label} declares no dispatch exec template — `eval-magic dispatch` \ + has no command to run for these tasks and will fail. Add \ + `[dispatch].exec_template` to the descriptor (see `eval-magic docs byoh`)." )); } Ok(HarnessPreflight { opts, warnings }) diff --git a/src/core/mod.rs b/src/core/mod.rs index aaf3830..9f3a381 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -21,8 +21,8 @@ pub use context::{ContextError, DetectInput, Harness, RunContext, detect_run_con pub use git::BASELINE_REF; pub(crate) use git::IsolatedGit; pub(crate) use runtime::{ - GIT_ROUTING_ENV_VARS, POSIX_RECIPE_TOOLS, POSIX_TOOLING_REQUIREMENT, clear_git_environment, - posix_shell, require_posix_toolchain, validate_agent_environment_entry, + GIT_ROUTING_ENV_VARS, POSIX_TOOLING_REQUIREMENT, ShellOutcome, clear_git_environment, + posix_shell, run_in_posix_shell, validate_agent_environment_entry, }; pub use runtime::{GitOutput, run_git}; pub use types::*; diff --git a/src/core/runtime.rs b/src/core/runtime.rs index 630e444..8e266e8 100644 --- a/src/core/runtime.rs +++ b/src/core/runtime.rs @@ -5,10 +5,13 @@ //! `clap` owns argument parsing, and the `error: ` + exit(1) contract //! lives in `src/main.rs`. +use std::collections::BTreeMap; use std::ffi::OsStr; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Command, ExitStatus, Stdio}; use std::sync::OnceLock; +use std::thread::sleep; +use std::time::{Duration, Instant}; /// Inherited Git routing variables that can redirect repository discovery or /// object/index access away from a command's current working directory. @@ -99,25 +102,21 @@ pub fn run_git(args: &[&str], cwd: &Path) -> GitOutput { /// `--help` restates it in `cli::help::AFTER_HELP` instead, hard-wrapped and /// without backticks, because clap renders into a terminal rather than Markdown. /// -/// It names `jq` as well as the shell deliberately. Harness `exec_template`s ship -/// as POSIX command lines (`/mingw64/libexec/git-core` — hence three levels up). @@ -212,6 +211,78 @@ pub(crate) fn posix_shell() -> Result<&'static Path, &'static str> { } } +/// How a command run through the shell ended. +#[derive(Debug)] +pub(crate) enum ShellOutcome { + /// The child finished on its own, with this status. + Exited(ExitStatus), + /// The child outran its deadline and was killed. + TimedOut, +} + +/// How often [`run_in_posix_shell`] re-checks a child it is timing. Short +/// enough that a deadline is honored promptly, long enough that waiting on a +/// half-hour dispatch costs nothing measurable. +const CHILD_POLL_INTERVAL: Duration = Duration::from_millis(10); + +/// Run `command` through the resolved POSIX shell with `cwd` as the child's +/// working directory and `env` layered onto the inherited environment. With a +/// `timeout`, a child that outruns it is killed and reported as +/// [`ShellOutcome::TimedOut`] rather than blocking the caller; without one, the +/// call blocks until the child exits. +/// +/// Both the working directory and the environment travel through the spawn +/// rather than through process-global state, which is what lets several +/// dispatches run concurrently in their own task environments. +/// +/// All three standard streams are `null`. stdin, because harness command lines +/// detach it themselves so a permission prompt cannot block on a TTY, and +/// concurrent children must not contend for the terminal. stdout and stderr, +/// because every harness `exec_template` redirects them into the task's outputs +/// directory itself — and because inheriting them would defeat the timeout: +/// only the direct child is killed, so a shell that has already forked leaves +/// a grandchild holding the inherited pipe open, and the caller stays blocked +/// on it long past the deadline it just enforced. +pub(crate) fn run_in_posix_shell( + command: &str, + cwd: &Path, + env: &BTreeMap, + timeout: Option, +) -> Result { + let shell = posix_shell().map_err(str::to_string)?; + let mut child = Command::new(shell) + .arg("-c") + .arg(command) + .current_dir(cwd) + .envs(env) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| format!("failed to start {}: {error}", shell.display()))?; + let Some(timeout) = timeout else { + return child + .wait() + .map(ShellOutcome::Exited) + .map_err(|error| error.to_string()); + }; + let deadline = Instant::now() + timeout; + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(ShellOutcome::Exited(status)), + Ok(None) => { + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + return Ok(ShellOutcome::TimedOut); + } + sleep(CHILD_POLL_INTERVAL); + } + Err(error) => return Err(error.to_string()), + } + } +} + /// Announce that `test` is being skipped, `reason` explaining what the host /// lacks. Returns `true` so a caller can `return` on it. /// @@ -230,37 +301,11 @@ pub(crate) fn report_skip(test: &str, reason: &str) -> bool { true } -/// The toolchain a shipped recipe shells out to: the parallel-dispatch and judge -/// recipes are `jq` pipelines over `xargs`, `tr`, and `wc`. Both the `run` -/// preflight that warns about a gap and the tests that execute a rendered recipe -/// check this same list, so neither can drift from what the recipes actually use. -pub(crate) const POSIX_RECIPE_TOOLS: &[&str] = &["jq", "xargs", "tr", "wc"]; - -/// The resolved shell, once every tool in `tools` is reachable from inside it, -/// otherwise an error naming the first one that is not. -/// -/// The shipped parallel and judge recipes are POSIX pipelines over `jq`, -/// `xargs`, `tr`, and `wc`, so both a test that executes one and the `run` -/// preflight that warns about one need all of them. They are checked through the -/// shell rather than on the host `PATH` because that is where the recipe will -/// look: Git for Windows carries its own `/usr/bin`. -pub(crate) fn require_posix_toolchain(tools: &[&str]) -> Result<&'static Path, String> { - let shell = posix_shell().map_err(str::to_string)?; - for tool in tools { - let found = Command::new(shell) - .arg("-c") - .arg(format!("command -v {tool}")) - .output() - .is_ok_and(|output| output.status.success()); - if !found { - return Err(format!("{tool} is not on the PATH of {}", shell.display())); - } - } - Ok(shell) -} - #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use std::time::Duration; + use super::*; /// A successful git command returns exit status 0 and writes to stdout. @@ -349,10 +394,10 @@ mod tests { assert!(error.contains("/nonexistent-shell-for-tests"), "{error}"); assert!(error.contains("Git Bash"), "{error}"); assert!(error.contains("WSL"), "{error}"); - // The declared requirement is a POSIX shell *and* `jq`: Git for Windows - // supplies the shell but not `jq`, so naming only the shell would send - // an operator to a setup that still cannot run the judge recipe. - assert!(error.contains("jq"), "{error}"); + // A POSIX shell is the whole requirement now. `jq` was only ever needed + // by the generated recipes an operator pasted, and the runner dispatches + // directly instead. + assert!(!error.contains("jq"), "{error}"); } /// A POSIX shell is a declared development requirement, not a capability the @@ -388,27 +433,103 @@ mod tests { } #[test] - fn require_posix_toolchain_names_the_tool_that_is_missing() { - let error = require_posix_toolchain(&["eval-magic-not-a-real-tool"]) - .expect_err("an uninstalled tool should be reported"); - assert!(error.contains("eval-magic-not-a-real-tool"), "{error}"); + fn report_skip_panics_only_when_coverage_is_enforced() { + // The unenforced path is the one this suite runs under; the enforced + // path is covered by CI setting the variable. + assert!( + std::env::var_os("EVAL_MAGIC_REQUIRE_POSIX_TOOLS").is_some() + || report_skip("demo", "a demo capability") + ); + } + + /// Build a `__fixture` command line. The string is handed to a shell, so it + /// has to parse identically under `sh -c` and `cmd /C`: a double-quoted + /// program path followed by double-quoted arguments does, because both + /// shells strip the quotes and hand the tokens over unchanged. + fn fixture(args: &[&str]) -> String { + let exe = assert_cmd::cargo::cargo_bin("eval-magic"); + assert!( + exe.is_file(), + "the __fixture command needs the eval-magic binary at {}; \ + run `cargo test`, which builds bins, or `cargo build` first", + exe.display() + ); + let mut command = format!("\"{}\" __fixture", exe.display()); + for arg in args { + command.push_str(&format!(" \"{arg}\"")); + } + command } - /// With nothing to look for, the check reduces to locating the shell — which - /// is required, so this succeeds wherever the suite is allowed to run. + /// The child's own exit status reaches the caller, so a dispatch can tell a + /// harness failure from a runner failure. #[test] - fn require_posix_toolchain_with_no_tools_reduces_to_finding_the_shell() { - let shell = require_posix_toolchain(&[]).expect("the required POSIX shell resolves"); - assert!(shell.is_file()); + fn a_shell_command_reports_the_child_exit_status() { + let outcome = run_in_posix_shell( + &fixture(&["--exit", "3"]), + Path::new("."), + &BTreeMap::new(), + None, + ) + .expect("the fixture runs"); + match outcome { + ShellOutcome::Exited(status) => assert_eq!(status.code(), Some(3)), + ShellOutcome::TimedOut => panic!("an untimed command cannot time out"), + } } + /// A child that outruns its deadline is killed and reported, rather than + /// blocking the caller forever. This is the whole reason a campaign can + /// survive one hung dispatch. #[test] - fn report_skip_panics_only_when_coverage_is_enforced() { - // The unenforced path is the one this suite runs under; the enforced - // path is covered by CI setting the variable. + fn a_shell_command_that_outruns_its_timeout_is_killed_and_reported() { + let outcome = run_in_posix_shell( + &fixture(&["--sleep-ms", "10000"]), + Path::new("."), + &BTreeMap::new(), + Some(Duration::from_millis(100)), + ) + .expect("the fixture spawns"); assert!( - std::env::var_os("EVAL_MAGIC_REQUIRE_POSIX_TOOLS").is_some() - || report_skip("demo", "a demo capability") + matches!(outcome, ShellOutcome::TimedOut), + "expected a timeout, got {outcome:?}" + ); + } + + /// Per-task environment reaches the child. Concurrent dispatches each carry + /// their own map, so this must travel through the spawn rather than through + /// the parent's environment. + #[test] + fn a_shell_command_carries_the_supplied_environment() { + let env = BTreeMap::from([("EVAL_MAGIC_PROBE".to_string(), "carried".to_string())]); + let outcome = run_in_posix_shell( + &fixture(&["--require-env", "EVAL_MAGIC_PROBE=carried"]), + Path::new("."), + &env, + None, + ) + .expect("the fixture runs"); + match outcome { + ShellOutcome::Exited(status) => assert!(status.success(), "{status}"), + ShellOutcome::TimedOut => panic!("an untimed command cannot time out"), + } + } + + /// The child runs in the directory it was given, not the runner's cwd — + /// what keeps concurrent dispatches in their own private task envs. + #[test] + fn a_shell_command_runs_in_the_directory_it_was_given() { + let dir = tempfile::TempDir::new().unwrap(); + run_in_posix_shell( + &fixture(&["--text", "here", "--write", "marker.txt"]), + dir.path(), + &BTreeMap::new(), + None, + ) + .expect("the fixture runs"); + assert_eq!( + std::fs::read_to_string(dir.path().join("marker.txt")).unwrap(), + "here" ); } diff --git a/src/core/types.rs b/src/core/types.rs index b233580..92278cb 100644 --- a/src/core/types.rs +++ b/src/core/types.rs @@ -383,7 +383,7 @@ pub struct RunRecord { pub skill_source: Option, } -/// The completed outcome of one scripted conversation. +/// The completed outcome of one dispatched task's conversation. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ConversationRecord { pub status: ConversationStatus, @@ -392,6 +392,9 @@ pub struct ConversationRecord { pub stop_reason: Option, #[serde(skip_serializing_if = "Option::is_none")] pub stopped_before_followup: Option, + /// The round the dispatch was killed in, when it outran its deadline. + #[serde(skip_serializing_if = "Option::is_none")] + pub timed_out_in_round: Option, pub events: Vec, } @@ -399,7 +402,11 @@ pub struct ConversationRecord { #[serde(rename_all = "snake_case")] pub enum ConversationStatus { Completed, + /// Halted at a scripted gate — a normal, recorded result. Stopped, + /// Killed at its deadline. Recorded rather than lost, so the campaign shows + /// what hung instead of silently missing a cell. + TimedOut, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] diff --git a/src/pipeline/grade/transcript_check.rs b/src/pipeline/grade/transcript_check.rs index 24d0647..dd060aa 100644 --- a/src/pipeline/grade/transcript_check.rs +++ b/src/pipeline/grade/transcript_check.rs @@ -313,6 +313,7 @@ mod tests { delivered_followups: 1, stop_reason: None, stopped_before_followup: None, + timed_out_in_round: None, events: vec![ ConversationEvent::UserMessage { ordinal: 0, diff --git a/src/pipeline/record_runs.rs b/src/pipeline/record_runs.rs index 17724d6..1abf49a 100644 --- a/src/pipeline/record_runs.rs +++ b/src/pipeline/record_runs.rs @@ -68,6 +68,11 @@ struct DispatchTask { dispatch_prompt_path: String, #[serde(default)] conversation_path: Option, + /// Present only for a scripted task. Every task carries a + /// `conversation_path`, so this is what tells a task whose rounds are + /// unknown-without-the-artifact from a one-shot task. + #[serde(default)] + turns: Option, /// Group this task belongs to; absent for a single-group run. Carried so the /// session-surface report can be joined back to the comparison cells a /// shadow finding names. @@ -182,7 +187,8 @@ impl RecordRunsResult { Some(format!( "⚠ {n} scripted conversation{plural} skipped — conversation.json is missing, so \ eval-magic cannot distinguish a completed/stopped scenario from an interrupted \ - dispatch. Re-run the corresponding `dispatch-task` command." + dispatch. Re-run `eval-magic dispatch` — it retries exactly the tasks with no \ + completion artifact." )) } } @@ -215,7 +221,11 @@ pub fn record_runs( let mut surface_tasks: Vec = Vec::new(); for task in &tasks { let conversation = conversation::for_task(task)?; - if task.conversation_path.is_some() && conversation.is_none() { + // Keyed on `turns`, not on `conversation_path`: every task declares a + // conversation artifact, so its presence does not distinguish a scripted + // one. A scripted task without the artifact is genuinely incomplete — + // which rounds ran is unknown. + if task.turns.is_some() && conversation.is_none() { result.skipped_incomplete_conversation += 1; continue; } diff --git a/tests/cli/basics.rs b/tests/cli/basics.rs index 8168b7c..49637ef 100644 --- a/tests/cli/basics.rs +++ b/tests/cli/basics.rs @@ -86,7 +86,7 @@ fn help_uses_published_binary_name() { fn every_visible_command_and_harness_subcommand_renders_help() { for args in [ "run --help", - "dispatch-task --help", + "dispatch --help", "snapshot --help", "teardown --help", "teardown-guard --help", @@ -152,9 +152,9 @@ fn top_level_examples_stop_after_orientation_and_handoffs() { } #[test] -fn dispatch_task_help_documents_conversation_verification() { +fn dispatch_help_documents_conversation_verification() { skill_eval() - .args(["dispatch-task", "--help"]) + .args(["dispatch", "--help"]) .assert() .success() .stdout(contains("delivered_followups")) @@ -246,12 +246,14 @@ fn grade_and_ingest_help_document_runner_owned_command_checks() { } #[test] -fn ingest_help_documents_judge_batch_completion() { +fn dispatch_judges_help_documents_batch_completion() { + // `dispatch --judges` owns the batch-completion contract: it is what counts + // verdicts and decides the exit status. skill_eval() - .args(["ingest", "--help"]) + .args(["dispatch", "--help"]) .assert() .success() - .stdout(contains("skips existing nonempty responses")) + .stdout(contains("Skips existing nonempty responses")) .stdout(contains("verdicts present")) .stdout(contains("exits nonzero while any are missing")); } diff --git a/tests/cli/docs.rs b/tests/cli/docs.rs index f7c206d..6cc7b23 100644 --- a/tests/cli/docs.rs +++ b/tests/cli/docs.rs @@ -232,7 +232,7 @@ fn every_guide_reference_in_shipped_help_resolves() { for help_args in [ "--help", "run --help", - "dispatch-task --help", + "dispatch --help", "snapshot --help", "teardown --help", "teardown-guard --help", @@ -308,13 +308,13 @@ fn repository_documentation_map_names_each_surface() { assert!(!agents.contains("docs/README.md")); // A POSIX shell is a development requirement, not a probed capability: the - // scripted-turn tests spawn a `#!/bin/sh` stub through it and cannot skip. - // Both contributor-facing docs have to say so, or the next contributor on + // dispatch tests spawn a `#!/bin/sh` stub through it and cannot skip. Both + // contributor-facing docs have to say so, or the next contributor on // Windows rediscovers it as a test failure (issue #248). for (name, text) in [("AGENTS.md", &agents), ("developer overview", &overview)] { assert!( - text.contains("POSIX shell") && text.contains("jq"), - "{name} should record the POSIX shell + jq development requirement" + text.contains("POSIX shell"), + "{name} should record the POSIX shell development requirement" ); } } @@ -329,9 +329,11 @@ fn help_states_the_posix_tooling_requirement() { .success() .stdout(contains("REQUIREMENTS:")) .stdout(contains("POSIX shell")) - .stdout(contains("jq")) .stdout(contains("Git Bash")) - .stdout(contains("WSL")); + .stdout(contains("WSL")) + // `jq` was a requirement only while operators pasted the generated + // recipes; the runner dispatches directly and needs no such toolchain. + .stdout(contains("jq").not()); } #[test] @@ -351,11 +353,12 @@ fn readme_is_a_concise_first_run_path() { "eval-magic docs isolation", "docs/developer_overview.md", // The declared host requirement, stated for both audiences the README - // serves: installing the tool, and developing it (issue #248). + // serves: installing the tool, and developing it (issue #248). `jq` is + // deliberately absent — it was a requirement only while operators + // pasted the generated recipes. "POSIX shell", "Git Bash", "WSL", - "jq", ] { assert!(readme.contains(expected), "README is missing {expected}"); } diff --git a/tests/cli/grade_models.rs b/tests/cli/grade_models.rs index da883ca..e5f244e 100644 --- a/tests/cli/grade_models.rs +++ b/tests/cli/grade_models.rs @@ -85,9 +85,11 @@ fn grade_defaults_judge_tasks_to_recorded_judge_model() { let assert = grade_cmd(&cwd, &skill_dir, Some("codex")) .assert() .success(); + // The hand-off is the runner's own command now, not a harness recipe with + // a `$model_arg` slot: each judge task carries its resolved model in + // judge-tasks.json, and `dispatch --judges` reads it from there. let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); - assert!(stdout.contains("codex --ask-for-approval never exec")); - assert!(stdout.contains("model_arg=\"-m $model\"")); + assert!(stdout.contains("eval-magic dispatch --judges"), "{stdout}"); let tasks: serde_json::Value = serde_json::from_str(&fs::read_to_string(iteration_dir.join("judge-tasks.json")).unwrap()) diff --git a/tests/cli/harness.rs b/tests/cli/harness.rs index 88258ae..c10cee3 100644 --- a/tests/cli/harness.rs +++ b/tests/cli/harness.rs @@ -521,36 +521,6 @@ fn harness_lint_probe_fails_when_final_message_missing() { .stderr(contains("✗").and(contains("final-message.md"))); } -#[test] -fn harness_lint_probe_renders_parallel_and_judge_templates() { - let tmp = TempDir::new().unwrap(); - let file = tmp.path().join("probe-full.toml"); - fs::write( - &file, - "label = \"probe-full\"\n\n\ - [model]\nflag = \"-m\"\n\n\ - [dispatch]\n\ - exec_template = 'printf \"ok\\n\" > /final-message.md'\n\ - capture_prefix = \"out\"\n\ - parallel_command_template = \"agent --cd {cwd} run\"\n\ - judge_command_template = \"judge --cd {cwd} $model_arg \\\\\"\n", - ) - .unwrap(); - - skill_eval() - .current_dir(tmp.path()) - .args(["harness", "lint"]) - .arg(&file) - .args(["--probe", "--yes"]) - .assert() - .success() - .stdout( - contains("✓ live exec template") - .and(contains("✓ render: parallel_command_template")) - .and(contains("✓ render: judge_command_template")), - ); -} - #[test] fn harness_lint_probe_aborts_without_yes_on_non_yes_stdin() { let tmp = TempDir::new().unwrap(); diff --git a/tests/golden/claude-code/judge-recipe.golden.md b/tests/golden/claude-code/judge-recipe.golden.md deleted file mode 100644 index c4f66ad..0000000 --- a/tests/golden/claude-code/judge-recipe.golden.md +++ /dev/null @@ -1,37 +0,0 @@ -Dispatch each judge task from judge-tasks.json with: -Existing nonempty response files are skipped; delete one to dispatch that judge again. -The final `N/M verdicts present` summary exits nonzero until every task has one. - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - prompt_path="$1" - response_path="$2" - model="${3#model=}" - if [ -s "$response_path" ]; then exit 0; fi - response_base="${response_path%.json}" - mkdir -p "$(dirname "$response_path")" - model_arg=""; [ -n "$model" ] && model_arg="--model $model" - cd "/work/iter-1" && claude -p --output-format stream-json --verbose --permission-mode bypassPermissions $model_arg \ - "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ - "$response_base.claude-events.jsonl" \ - 2> "$response_base.claude-stderr.log" - ' sh -judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') -judge_present=$( - jq -r '.tasks[].response_path' judge-tasks.json \ - | tr -d '\r' \ - | while IFS= read -r response_path; do - if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi - done \ - | wc -l \ - | tr -d '[:space:]' -) -printf '%s/%s verdicts present\n' "$judge_present" "$judge_total" -[ "$judge_dispatch_status" -eq 0 ] && [ "$judge_present" -eq "$judge_total" ] -``` \ No newline at end of file diff --git a/tests/golden/claude-code/manifest-nomodel.golden.md b/tests/golden/claude-code/manifest-nomodel.golden.md deleted file mode 100644 index 9f0ea4a..0000000 --- a/tests/golden/claude-code/manifest-nomodel.golden.md +++ /dev/null @@ -1,127 +0,0 @@ -# Dispatch manifest — widget-skill iteration-2 - -Mode: revision (baseline: iteration-1) -Generated: 2026-01-01T00:00:00Z -Total dispatches: 2 - -## How to use this manifest - -In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the task with a short "read this file and follow it" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`. - -**Requires:** eval-magic's dispatch and judge recipes are POSIX command lines built on `jq`, `xargs`, `tr`, and `wc`. Run them in a POSIX shell with `jq` installed that resolves the same paths this workspace was prepared with — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. - -After all dispatches (Claude Code): - -Run one fresh `claude -p` per task from the env dir (`cd ` — `claude` has no --cd flag). `--output-format stream-json` requires `--verbose`; detach stdin with ` && claude -p --output-format stream-json --verbose --permission-mode bypassPermissions \ - "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ - /claude-events.jsonl \ - 2> /claude-stderr.log -``` - -Parallel dispatch from this iteration directory: - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - eval_root="$1" - prompt_path="$2" - outputs_dir="$3" - mkdir -p "$outputs_dir" - unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES - cd "$eval_root" && claude -p --output-format stream-json --verbose --permission-mode bypassPermissions \ - "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response your closing summary." \ - "$outputs_dir/claude-events.jsonl" \ - 2> "$outputs_dir/claude-stderr.log" - ' sh -``` - -Then run `eval-magic ingest --harness claude-code`; ingest reads each task's `outputs/claude-events.jsonl`. - -After all dispatches: - -1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. -2. Dispatch the judge tasks ingest lists, then run `eval-magic finalize` for the benchmark. - -On a harness without persisted transcripts, instead write each task's `run.json` (matching `skills/evaluating-skills/schema/run-record.schema.json`, enforced at runtime by grade/fill-transcripts/detect-stray-writes) and `timing.json` by hand when its subagent returns: carry over `eval_id`, `condition`, `skill_path` (`null` on the without_skill arm), `prompt`, and `files` from the task; populate `final_message` from the subagent's reply; leave `tool_invocations` as `[]`; capture `total_tokens`/`duration_ms` from the task completion event immediately — they may not be persisted anywhere else. - -## Dispatches -### demo-eval / with_skill - -- run.json: /work/cond/run.json -- timing.json: /work/cond/timing.json - -``` - -The following guidelines were loaded at session start by the slow-powers plugin -(equivalent to the SessionStart hook firing in a real user's environment): - -Session guidelines: be concise. - -The following skills are available for use with the Skill tool: - -- aux-helper: Assists with auxiliary chores. -- widget-skill: Builds widgets the house way. - - -PLAN STEP - - -You are executing a single test case for a skill evaluation framework. -Treat this as a real user request — do NOT optimize behavior for the eval. - -The `widget-skill` skill is registered under the identifier `slow-powers-eval-2-with_skill__widget-skill` and is discoverable via the Skill tool. If you invoke it, use that identifier. -If the Skill tool cannot resolve that identifier, read the skill from `/work/staged/widget-skill/SKILL.md` instead. - -Available fixture files: - - /work/fixtures/input.txt -Task environment: /work/task -Task-local scratch directory: /work/task/tmp -Framework output directory: /work/outputs - -Instructions: -- Work normally on the task: you may edit existing files and create new files inside the task environment. -- Keep temporary and scratch files in the task-local scratch directory, not in a host temp directory. -- Use the framework output directory only for framework artifacts. -- After completing the task, write your final user-facing response to /work/outputs/final-message.md. -- Do not write outside the task environment. - -User request: -Build me a widget. -``` - -### demo-eval / without_skill - -- run.json: /work/cond-b/run.json -- timing.json: /work/cond-b/timing.json - -``` -You are executing a single test case for a skill evaluation framework. -Treat this as a real user request — do NOT optimize behavior for the eval. - -No skill is loaded. Respond as you naturally would. - -Available fixture files: - - /work/fixtures/input.txt -Task environment: /work/task-b -Task-local scratch directory: /work/task-b/tmp -Framework output directory: /work/outputs-b - -Instructions: -- Work normally on the task: you may edit existing files and create new files inside the task environment. -- Keep temporary and scratch files in the task-local scratch directory, not in a host temp directory. -- Use the framework output directory only for framework artifacts. -- After completing the task, write your final user-facing response to /work/outputs-b/final-message.md. -- Do not write outside the task environment. - -User request: -Build me a widget. -``` diff --git a/tests/golden/claude-code/manifest.golden.md b/tests/golden/claude-code/manifest.golden.md index 83499d7..fca698f 100644 --- a/tests/golden/claude-code/manifest.golden.md +++ b/tests/golden/claude-code/manifest.golden.md @@ -8,11 +8,19 @@ Total dispatches: 2 In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the task with a short "read this file and follow it" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`. -**Requires:** eval-magic's dispatch and judge recipes are POSIX command lines built on `jq`, `xargs`, `tr`, and `wc`. Run them in a POSIX shell with `jq` installed that resolves the same paths this workspace was prepared with — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. +**Requires:** harness dispatch commands are POSIX command lines, and `eval-magic dispatch` runs them itself, so the host it runs on needs a POSIX shell — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. -After all dispatches (Claude Code): +## Dispatch -Run one fresh `claude -p` per task from the env dir (`cd ` — `claude` has no --cd flag). `--output-format stream-json` requires `--verbose`; detach stdin with ` --harness + +It runs `--jobs` tasks at a time, each in its own private environment, and writes each task's conversation.json. A task that already has one is skipped, so rerunning retries only what did not finish. A task exceeding `--timeout` is recorded as timed out, and a failing task is recorded while the rest of the batch continues. A conversation that stops at a scripted gate is valid eval data; a task with no conversation.json is incomplete and ingest skips it. + +Harness dispatch (Claude Code): + +`eval-magic dispatch` runs one fresh `claude -p` per task from the env dir (`cd ` — `claude` has no --cd flag). `--output-format stream-json` requires `--verbose`; detach stdin with `/claude-events.jsonl` and stderr as `outputs/turn-/claude-stderr.log`. ```bash unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES @@ -23,33 +31,12 @@ cd && claude -p --output-format stream-json --verbose --permission-m 2> /claude-stderr.log ``` -Parallel dispatch from this iteration directory: - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - eval_root="$1" - prompt_path="$2" - outputs_dir="$3" - mkdir -p "$outputs_dir" - unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES - cd "$eval_root" && claude -p --output-format stream-json --verbose --permission-mode bypassPermissions --model model-x \ - "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response your closing summary." \ - "$outputs_dir/claude-events.jsonl" \ - 2> "$outputs_dir/claude-stderr.log" - ' sh -``` - -Then run `eval-magic ingest --harness claude-code`; ingest reads each task's `outputs/claude-events.jsonl`. +Then run `eval-magic ingest --harness claude-code`; ingest reads each task's `outputs/turn-/claude-events.jsonl`. After all dispatches: -1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. -2. Dispatch the judge tasks ingest lists, then run `eval-magic finalize` for the benchmark. +1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/turn-/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. +2. Run `eval-magic dispatch --judges --harness ` to grade the judge tasks ingest listed, then `eval-magic finalize` for the benchmark. On a harness without persisted transcripts, instead write each task's `run.json` (matching `skills/evaluating-skills/schema/run-record.schema.json`, enforced at runtime by grade/fill-transcripts/detect-stray-writes) and `timing.json` by hand when its subagent returns: carry over `eval_id`, `condition`, `skill_path` (`null` on the without_skill arm), `prompt`, and `files` from the task; populate `final_message` from the subagent's reply; leave `tool_invocations` as `[]`; capture `total_tokens`/`duration_ms` from the task completion event immediately — they may not be persisted anywhere else. @@ -58,6 +45,7 @@ On a harness without persisted transcripts, instead write each task's `run.json` - run.json: /work/cond/run.json - timing.json: /work/cond/timing.json +- conversation.json: /work/cond/conversation.json ``` @@ -102,6 +90,7 @@ Build me a widget. - run.json: /work/cond-b/run.json - timing.json: /work/cond-b/timing.json +- conversation.json: /work/cond-b/conversation.json ``` You are executing a single test case for a skill evaluation framework. diff --git a/tests/golden/claude-code/runbook.golden.md b/tests/golden/claude-code/runbook.golden.md index 713eace..43350cb 100644 --- a/tests/golden/claude-code/runbook.golden.md +++ b/tests/golden/claude-code/runbook.golden.md @@ -4,7 +4,7 @@ This runbook is for a human driving the run from a terminal. Work from this iter and copy-paste each step. The workspace is self-contained — you should not need the surrounding repo. -> **Requires:** eval-magic's dispatch and judge recipes are POSIX command lines built on `jq`, `xargs`, `tr`, and `wc`. Run them in a POSIX shell with `jq` installed that resolves the same paths this workspace was prepared with — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. +> **Requires:** harness dispatch commands are POSIX command lines, and `eval-magic dispatch` runs them itself, so the host it runs on needs a POSIX shell — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. - **Skill under test:** widget-skill - **Mode:** revision — comparing `old_skill` vs `new_skill` @@ -12,14 +12,20 @@ repo. ## 1. Dispatch the eval agents, then ingest -Next: iterate the tasks[] array in dispatch.json and dispatch each task (from the env dir — `claude` has no --cd flag) with: -unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES -cd && claude -p --output-format stream-json --verbose --permission-mode bypassPermissions --model model-x \ - "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ - /claude-events.jsonl \ - 2> /claude-stderr.log -Then run `ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness claude-code`. +``` +eval-magic dispatch --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness claude-code +``` + +`dispatch` runs every task in its own private environment, `--jobs` of them at a time, and writes +each task's `conversation.json`. A task that already has one is skipped, so rerunning the same +command retries only what did not finish. A task that exceeds `--timeout` is recorded as timed out +rather than left to stall the campaign, and a task that fails is recorded and named while the rest +of the batch continues. A conversation that stops at a scripted gate is valid eval data, not a +failure. + +``` +eval-magic ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness claude-code +``` `ingest` records each run, backfills transcripts, scans for stray writes, collects guarded-task blocks into `guard-denials.json`, and grades every mechanical assertion. Inspect any denial @@ -27,43 +33,13 @@ warning before trusting the affected task. It then prints any `llm_judge` tasks grade itself. ## 2. Dispatch the judge agents, then finalize -Dispatch each judge task from judge-tasks.json with: -Existing nonempty response files are skipped; delete one to dispatch that judge again. -The final `N/M verdicts present` summary exits nonzero until every task has one. - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - prompt_path="$1" - response_path="$2" - model="${3#model=}" - if [ -s "$response_path" ]; then exit 0; fi - response_base="${response_path%.json}" - mkdir -p "$(dirname "$response_path")" - model_arg=""; [ -n "$model" ] && model_arg="--model $model" - cd "/work/.eval-magic/widget-skill/iteration-2" && claude -p --output-format stream-json --verbose --permission-mode bypassPermissions $model_arg \ - "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ - "$response_base.claude-events.jsonl" \ - 2> "$response_base.claude-stderr.log" - ' sh -judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') -judge_present=$( - jq -r '.tasks[].response_path' judge-tasks.json \ - | tr -d '\r' \ - | while IFS= read -r response_path; do - if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi - done \ - | wc -l \ - | tr -d '[:space:]' -) -printf '%s/%s verdicts present\n' "$judge_present" "$judge_total" -[ "$judge_dispatch_status" -eq 0 ] && [ "$judge_present" -eq "$judge_total" ] + ``` +eval-magic dispatch --judges --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness claude-code +``` + +Verdicts that are already present are skipped; the summary prints `N/M verdicts present` and exits +nonzero until every task has one, so rerun the same command to fill the gaps. Then merge the verdicts and aggregate: diff --git a/tests/golden/cline/judge-recipe.golden.md b/tests/golden/cline/judge-recipe.golden.md deleted file mode 100644 index 355fb2d..0000000 --- a/tests/golden/cline/judge-recipe.golden.md +++ /dev/null @@ -1,37 +0,0 @@ -Dispatch each judge task from judge-tasks.json with: -Existing nonempty response files are skipped; delete one to dispatch that judge again. -The final `N/M verdicts present` summary exits nonzero until every task has one. - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - prompt_path="$1" - response_path="$2" - model="${3#model=}" - if [ -s "$response_path" ]; then exit 0; fi - response_base="${response_path%.json}" - mkdir -p "$(dirname "$response_path")" - model_arg=""; [ -n "$model" ] && model_arg="-m $model" - cline --cwd "/work/iter-1" --act --json --auto-approve true $model_arg \ - "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ - "$response_base.cline-events.jsonl" \ - 2> "$response_base.cline-stderr.log" - ' sh -judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') -judge_present=$( - jq -r '.tasks[].response_path' judge-tasks.json \ - | tr -d '\r' \ - | while IFS= read -r response_path; do - if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi - done \ - | wc -l \ - | tr -d '[:space:]' -) -printf '%s/%s verdicts present\n' "$judge_present" "$judge_total" -[ "$judge_dispatch_status" -eq 0 ] && [ "$judge_present" -eq "$judge_total" ] -``` \ No newline at end of file diff --git a/tests/golden/cline/manifest.golden.md b/tests/golden/cline/manifest.golden.md index 5103519..44605c1 100644 --- a/tests/golden/cline/manifest.golden.md +++ b/tests/golden/cline/manifest.golden.md @@ -8,11 +8,19 @@ Total dispatches: 2 In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the task with a short "read this file and follow it" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`. -**Requires:** eval-magic's dispatch and judge recipes are POSIX command lines built on `jq`, `xargs`, `tr`, and `wc`. Run them in a POSIX shell with `jq` installed that resolves the same paths this workspace was prepared with — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. +**Requires:** harness dispatch commands are POSIX command lines, and `eval-magic dispatch` runs them itself, so the host it runs on needs a POSIX shell — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. -After all dispatches (Cline): +## Dispatch -Run one fresh `cline --cwd --act --json --auto-approve true` per task. Detach stdin with `` so piped task data cannot become extra prompt context; capture stdout as `outputs/cline-events.jsonl` and stderr as `outputs/cline-stderr.log`. The trailing jq step recovers `outputs/final-message.md` from the terminal `run_result` event. +Every task is runner-driven — one-shot and scripted alike — so one command runs the whole plan from this iteration directory: + +eval-magic dispatch --iteration --harness + +It runs `--jobs` tasks at a time, each in its own private environment, and writes each task's conversation.json. A task that already has one is skipped, so rerunning retries only what did not finish. A task exceeding `--timeout` is recorded as timed out, and a failing task is recorded while the rest of the batch continues. A conversation that stops at a scripted gate is valid eval data; a task with no conversation.json is incomplete and ingest skips it. + +Harness dispatch (Cline): + +`eval-magic dispatch` runs one fresh `cline --cwd --act --json --auto-approve true` per task. Detach stdin with `` so piped task data cannot become extra prompt context; capture stdout as `outputs/turn-/cline-events.jsonl` and stderr as `outputs/turn-/cline-stderr.log`. `eval-magic dispatch` writes `outputs/final-message.md` itself from the parsed transcript; the template's trailing jq step is a belt-and-braces copy of the terminal `run_result` event. ```bash unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES @@ -25,35 +33,12 @@ cline --cwd --act --json --auto-approve true -m model-x \ > /final-message.md ``` -Parallel dispatch from this iteration directory: - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - eval_root="$1" - prompt_path="$2" - outputs_dir="$3" - mkdir -p "$outputs_dir" - unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES - cline --cwd "$eval_root" --act --json --auto-approve true -m model-x \ - "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response your closing summary." \ - "$outputs_dir/cline-events.jsonl" \ - 2> "$outputs_dir/cline-stderr.log"; \ - jq -rj "select(.type == \"run_result\") | .text" "$outputs_dir/cline-events.jsonl" \ - > "$outputs_dir/final-message.md" - ' sh -``` - -Then run `eval-magic ingest --harness cline`; ingest reads each task's `outputs/cline-events.jsonl`. +Then run `eval-magic ingest --harness cline`; ingest reads each task's `outputs/turn-/cline-events.jsonl`. After all dispatches: -1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. -2. Dispatch the judge tasks ingest lists, then run `eval-magic finalize` for the benchmark. +1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/turn-/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. +2. Run `eval-magic dispatch --judges --harness ` to grade the judge tasks ingest listed, then `eval-magic finalize` for the benchmark. On a harness without persisted transcripts, instead write each task's `run.json` (matching `skills/evaluating-skills/schema/run-record.schema.json`, enforced at runtime by grade/fill-transcripts/detect-stray-writes) and `timing.json` by hand when its subagent returns: carry over `eval_id`, `condition`, `skill_path` (`null` on the without_skill arm), `prompt`, and `files` from the task; populate `final_message` from the subagent's reply; leave `tool_invocations` as `[]`; capture `total_tokens`/`duration_ms` from the task completion event immediately — they may not be persisted anywhere else. @@ -62,6 +47,7 @@ On a harness without persisted transcripts, instead write each task's `run.json` - run.json: /work/cond/run.json - timing.json: /work/cond/timing.json +- conversation.json: /work/cond/conversation.json ``` @@ -106,6 +92,7 @@ Build me a widget. - run.json: /work/cond-b/run.json - timing.json: /work/cond-b/timing.json +- conversation.json: /work/cond-b/conversation.json ``` You are executing a single test case for a skill evaluation framework. diff --git a/tests/golden/cline/next-steps-model.golden.txt b/tests/golden/cline/next-steps-model.golden.txt deleted file mode 100644 index 673bf35..0000000 --- a/tests/golden/cline/next-steps-model.golden.txt +++ /dev/null @@ -1,11 +0,0 @@ - -Next: iterate the tasks[] array in dispatch.json and dispatch each task with: -unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES -cline --cwd --act --json --auto-approve true -m model-x \ - "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ - /cline-events.jsonl \ - 2> /cline-stderr.log; \ - jq -rj 'select(.type == "run_result") | .text' /cline-events.jsonl \ - > /final-message.md -Then run `ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness cline`. \ No newline at end of file diff --git a/tests/golden/cline/next-steps-nomodel.golden.txt b/tests/golden/cline/next-steps-nomodel.golden.txt deleted file mode 100644 index 55aee1f..0000000 --- a/tests/golden/cline/next-steps-nomodel.golden.txt +++ /dev/null @@ -1,11 +0,0 @@ - -Next: iterate the tasks[] array in dispatch.json and dispatch each task with: -unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES -cline --cwd --act --json --auto-approve true \ - "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ - /cline-events.jsonl \ - 2> /cline-stderr.log; \ - jq -rj 'select(.type == "run_result") | .text' /cline-events.jsonl \ - > /final-message.md -Then run `ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness cline`. \ No newline at end of file diff --git a/tests/golden/cline/next-steps.golden.txt b/tests/golden/cline/next-steps.golden.txt new file mode 100644 index 0000000..e1a6bbc --- /dev/null +++ b/tests/golden/cline/next-steps.golden.txt @@ -0,0 +1,3 @@ + +Next: eval-magic dispatch --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness cline +Then run `eval-magic ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness cline`. \ No newline at end of file diff --git a/tests/golden/cline/runbook.golden.md b/tests/golden/cline/runbook.golden.md index f6ce60e..8998d00 100644 --- a/tests/golden/cline/runbook.golden.md +++ b/tests/golden/cline/runbook.golden.md @@ -4,7 +4,7 @@ This runbook is for a human driving the run from a terminal. Work from this iter and copy-paste each step. The workspace is self-contained — you should not need the surrounding repo. -> **Requires:** eval-magic's dispatch and judge recipes are POSIX command lines built on `jq`, `xargs`, `tr`, and `wc`. Run them in a POSIX shell with `jq` installed that resolves the same paths this workspace was prepared with — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. +> **Requires:** harness dispatch commands are POSIX command lines, and `eval-magic dispatch` runs them itself, so the host it runs on needs a POSIX shell — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. - **Skill under test:** widget-skill - **Mode:** revision — comparing `old_skill` vs `new_skill` @@ -12,16 +12,20 @@ repo. ## 1. Dispatch the eval agents, then ingest -Next: iterate the tasks[] array in dispatch.json and dispatch each task with: -unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES -cline --cwd --act --json --auto-approve true -m model-x \ - "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ - /cline-events.jsonl \ - 2> /cline-stderr.log; \ - jq -rj 'select(.type == "run_result") | .text' /cline-events.jsonl \ - > /final-message.md -Then run `ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness cline`. +``` +eval-magic dispatch --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness cline +``` + +`dispatch` runs every task in its own private environment, `--jobs` of them at a time, and writes +each task's `conversation.json`. A task that already has one is skipped, so rerunning the same +command retries only what did not finish. A task that exceeds `--timeout` is recorded as timed out +rather than left to stall the campaign, and a task that fails is recorded and named while the rest +of the batch continues. A conversation that stops at a scripted gate is valid eval data, not a +failure. + +``` +eval-magic ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness cline +``` `ingest` records each run, backfills transcripts, scans for stray writes, collects guarded-task blocks into `guard-denials.json`, and grades every mechanical assertion. Inspect any denial @@ -29,43 +33,13 @@ warning before trusting the affected task. It then prints any `llm_judge` tasks grade itself. ## 2. Dispatch the judge agents, then finalize -Dispatch each judge task from judge-tasks.json with: -Existing nonempty response files are skipped; delete one to dispatch that judge again. -The final `N/M verdicts present` summary exits nonzero until every task has one. - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - prompt_path="$1" - response_path="$2" - model="${3#model=}" - if [ -s "$response_path" ]; then exit 0; fi - response_base="${response_path%.json}" - mkdir -p "$(dirname "$response_path")" - model_arg=""; [ -n "$model" ] && model_arg="-m $model" - cline --cwd "/work/.eval-magic/widget-skill/iteration-2" --act --json --auto-approve true $model_arg \ - "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ - "$response_base.cline-events.jsonl" \ - 2> "$response_base.cline-stderr.log" - ' sh -judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') -judge_present=$( - jq -r '.tasks[].response_path' judge-tasks.json \ - | tr -d '\r' \ - | while IFS= read -r response_path; do - if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi - done \ - | wc -l \ - | tr -d '[:space:]' -) -printf '%s/%s verdicts present\n' "$judge_present" "$judge_total" -[ "$judge_dispatch_status" -eq 0 ] && [ "$judge_present" -eq "$judge_total" ] + ``` +eval-magic dispatch --judges --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness cline +``` + +Verdicts that are already present are skipped; the summary prints `N/M verdicts present` and exits +nonzero until every task has one, so rerun the same command to fill the gaps. Then merge the verdicts and aggregate: diff --git a/tests/golden/codex/judge-recipe-noguard.golden.md b/tests/golden/codex/judge-recipe-noguard.golden.md deleted file mode 100644 index ee5233c..0000000 --- a/tests/golden/codex/judge-recipe-noguard.golden.md +++ /dev/null @@ -1,37 +0,0 @@ -Dispatch each judge task from judge-tasks.json with: -Existing nonempty response files are skipped; delete one to dispatch that judge again. -The final `N/M verdicts present` summary exits nonzero until every task has one. - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - prompt_path="$1" - response_path="$2" - model="${3#model=}" - if [ -s "$response_path" ]; then exit 0; fi - response_base="${response_path%.json}" - mkdir -p "$(dirname "$response_path")" - model_arg=""; [ -n "$model" ] && model_arg="-m $model" - codex --ask-for-approval never exec --cd "/work/iter-1" --sandbox workspace-write $model_arg --json \ - "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ - "$response_base.codex-events.jsonl" \ - 2> "$response_base.codex-stderr.log" - ' sh -judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') -judge_present=$( - jq -r '.tasks[].response_path' judge-tasks.json \ - | tr -d '\r' \ - | while IFS= read -r response_path; do - if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi - done \ - | wc -l \ - | tr -d '[:space:]' -) -printf '%s/%s verdicts present\n' "$judge_present" "$judge_total" -[ "$judge_dispatch_status" -eq 0 ] && [ "$judge_present" -eq "$judge_total" ] -``` \ No newline at end of file diff --git a/tests/golden/codex/judge-recipe.golden.md b/tests/golden/codex/judge-recipe.golden.md deleted file mode 100644 index ee5233c..0000000 --- a/tests/golden/codex/judge-recipe.golden.md +++ /dev/null @@ -1,37 +0,0 @@ -Dispatch each judge task from judge-tasks.json with: -Existing nonempty response files are skipped; delete one to dispatch that judge again. -The final `N/M verdicts present` summary exits nonzero until every task has one. - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - prompt_path="$1" - response_path="$2" - model="${3#model=}" - if [ -s "$response_path" ]; then exit 0; fi - response_base="${response_path%.json}" - mkdir -p "$(dirname "$response_path")" - model_arg=""; [ -n "$model" ] && model_arg="-m $model" - codex --ask-for-approval never exec --cd "/work/iter-1" --sandbox workspace-write $model_arg --json \ - "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ - "$response_base.codex-events.jsonl" \ - 2> "$response_base.codex-stderr.log" - ' sh -judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') -judge_present=$( - jq -r '.tasks[].response_path' judge-tasks.json \ - | tr -d '\r' \ - | while IFS= read -r response_path; do - if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi - done \ - | wc -l \ - | tr -d '[:space:]' -) -printf '%s/%s verdicts present\n' "$judge_present" "$judge_total" -[ "$judge_dispatch_status" -eq 0 ] && [ "$judge_present" -eq "$judge_total" ] -``` \ No newline at end of file diff --git a/tests/golden/codex/manifest-noguard.golden.md b/tests/golden/codex/manifest-noguard.golden.md deleted file mode 100644 index 8312d24..0000000 --- a/tests/golden/codex/manifest-noguard.golden.md +++ /dev/null @@ -1,129 +0,0 @@ -# Dispatch manifest — widget-skill iteration-2 - -Mode: revision (baseline: iteration-1) -Generated: 2026-01-01T00:00:00Z -Total dispatches: 2 - -## How to use this manifest - -In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the task with a short "read this file and follow it" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`. - -**Requires:** eval-magic's dispatch and judge recipes are POSIX command lines built on `jq`, `xargs`, `tr`, and `wc`. Run them in a POSIX shell with `jq` installed that resolves the same paths this workspace was prepared with — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. - -After all dispatches (Codex): - -Run one fresh `codex --ask-for-approval never exec --json` per task. Detach stdin with ` --sandbox workspace-write -m model-x --json \ - --output-last-message /final-message.md \ - "Read the file at and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to /final-message.md." \ - /codex-events.jsonl \ - 2> /codex-stderr.log -``` - -Parallel dispatch from this iteration directory: - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - eval_root="$1" - prompt_path="$2" - outputs_dir="$3" - mkdir -p "$outputs_dir" - unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES - codex --ask-for-approval never exec --cd "$eval_root" --sandbox workspace-write -m model-x --json \ - --output-last-message "$outputs_dir/final-message.md" \ - "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to $outputs_dir/final-message.md." \ - "$outputs_dir/codex-events.jsonl" \ - 2> "$outputs_dir/codex-stderr.log" - ' sh -``` - -Then run `eval-magic ingest --harness codex`; Codex transcript ingest reads each task's `outputs/codex-events.jsonl`. - -After all dispatches: - -1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. -2. Dispatch the judge tasks ingest lists, then run `eval-magic finalize` for the benchmark. - -On a harness without persisted transcripts, instead write each task's `run.json` (matching `skills/evaluating-skills/schema/run-record.schema.json`, enforced at runtime by grade/fill-transcripts/detect-stray-writes) and `timing.json` by hand when its subagent returns: carry over `eval_id`, `condition`, `skill_path` (`null` on the without_skill arm), `prompt`, and `files` from the task; populate `final_message` from the subagent's reply; leave `tool_invocations` as `[]`; capture `total_tokens`/`duration_ms` from the task completion event immediately — they may not be persisted anywhere else. - -## Dispatches -### demo-eval / with_skill - -- run.json: /work/cond/run.json -- timing.json: /work/cond/timing.json - -``` - -The following guidelines were loaded at session start by the slow-powers plugin -(equivalent to the SessionStart hook firing in a real user's environment): - -Session guidelines: be concise. - -## Skills - -- aux-helper: Assists with auxiliary chores. (file: /work/staged/aux-helper/SKILL.md) -- widget-skill: Builds widgets the house way. (file: /work/staged/widget-skill/SKILL.md) - - -PLAN STEP - - -You are executing a single test case for a skill evaluation framework. -Treat this as a real user request — do NOT optimize behavior for the eval. - -The `widget-skill` skill is registered under the identifier `slow-powers-eval-2-with_skill__widget-skill` and is discoverable as a Codex skill. If you invoke it, use that identifier. -If it does not load as a Codex skill, read the skill from `/work/staged/widget-skill/SKILL.md` instead. - -Available fixture files: - - /work/fixtures/input.txt -Task environment: /work/task -Task-local scratch directory: /work/task/tmp -Framework output directory: /work/outputs - -Instructions: -- Work normally on the task: you may edit existing files and create new files inside the task environment. -- Keep temporary and scratch files in the task-local scratch directory, not in a host temp directory. -- Use the framework output directory only for framework artifacts. -- After completing the task, write your final user-facing response to /work/outputs/final-message.md. -- Do not write outside the task environment. - -User request: -Build me a widget. -``` - -### demo-eval / without_skill - -- run.json: /work/cond-b/run.json -- timing.json: /work/cond-b/timing.json - -``` -You are executing a single test case for a skill evaluation framework. -Treat this as a real user request — do NOT optimize behavior for the eval. - -No skill is loaded. Respond as you naturally would. - -Available fixture files: - - /work/fixtures/input.txt -Task environment: /work/task-b -Task-local scratch directory: /work/task-b/tmp -Framework output directory: /work/outputs-b - -Instructions: -- Work normally on the task: you may edit existing files and create new files inside the task environment. -- Keep temporary and scratch files in the task-local scratch directory, not in a host temp directory. -- Use the framework output directory only for framework artifacts. -- After completing the task, write your final user-facing response to /work/outputs-b/final-message.md. -- Do not write outside the task environment. - -User request: -Build me a widget. -``` diff --git a/tests/golden/codex/manifest.golden.md b/tests/golden/codex/manifest.golden.md index 75b3ee4..202a2a2 100644 --- a/tests/golden/codex/manifest.golden.md +++ b/tests/golden/codex/manifest.golden.md @@ -8,11 +8,19 @@ Total dispatches: 2 In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the task with a short "read this file and follow it" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`. -**Requires:** eval-magic's dispatch and judge recipes are POSIX command lines built on `jq`, `xargs`, `tr`, and `wc`. Run them in a POSIX shell with `jq` installed that resolves the same paths this workspace was prepared with — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. +**Requires:** harness dispatch commands are POSIX command lines, and `eval-magic dispatch` runs them itself, so the host it runs on needs a POSIX shell — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. -After all dispatches (Codex): +## Dispatch -Run one fresh `codex --ask-for-approval never exec --json` per task. Detach stdin with ` --harness + +It runs `--jobs` tasks at a time, each in its own private environment, and writes each task's conversation.json. A task that already has one is skipped, so rerunning retries only what did not finish. A task exceeding `--timeout` is recorded as timed out, and a failing task is recorded while the rest of the batch continues. A conversation that stops at a scripted gate is valid eval data; a task with no conversation.json is incomplete and ingest skips it. + +Harness dispatch (Codex): + +`eval-magic dispatch` runs one fresh `codex --ask-for-approval never exec --json` per task. Detach stdin with `/codex-events.jsonl` and stderr as `outputs/turn-/codex-stderr.log`. ```bash unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES @@ -24,34 +32,12 @@ codex --ask-for-approval never exec --cd --sandbox workspace-write - 2> /codex-stderr.log ``` -Parallel dispatch from this iteration directory: - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - eval_root="$1" - prompt_path="$2" - outputs_dir="$3" - mkdir -p "$outputs_dir" - unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES - codex --ask-for-approval never exec --cd "$eval_root" --sandbox workspace-write --dangerously-bypass-hook-trust -m model-x --json \ - --output-last-message "$outputs_dir/final-message.md" \ - "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to $outputs_dir/final-message.md." \ - "$outputs_dir/codex-events.jsonl" \ - 2> "$outputs_dir/codex-stderr.log" - ' sh -``` - -Then run `eval-magic ingest --harness codex`; Codex transcript ingest reads each task's `outputs/codex-events.jsonl`. +Then run `eval-magic ingest --harness codex`; Codex transcript ingest reads each task's `outputs/turn-/codex-events.jsonl`. After all dispatches: -1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. -2. Dispatch the judge tasks ingest lists, then run `eval-magic finalize` for the benchmark. +1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/turn-/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. +2. Run `eval-magic dispatch --judges --harness ` to grade the judge tasks ingest listed, then `eval-magic finalize` for the benchmark. On a harness without persisted transcripts, instead write each task's `run.json` (matching `skills/evaluating-skills/schema/run-record.schema.json`, enforced at runtime by grade/fill-transcripts/detect-stray-writes) and `timing.json` by hand when its subagent returns: carry over `eval_id`, `condition`, `skill_path` (`null` on the without_skill arm), `prompt`, and `files` from the task; populate `final_message` from the subagent's reply; leave `tool_invocations` as `[]`; capture `total_tokens`/`duration_ms` from the task completion event immediately — they may not be persisted anywhere else. @@ -60,6 +46,7 @@ On a harness without persisted transcripts, instead write each task's `run.json` - run.json: /work/cond/run.json - timing.json: /work/cond/timing.json +- conversation.json: /work/cond/conversation.json ``` @@ -104,6 +91,7 @@ Build me a widget. - run.json: /work/cond-b/run.json - timing.json: /work/cond-b/timing.json +- conversation.json: /work/cond-b/conversation.json ``` You are executing a single test case for a skill evaluation framework. diff --git a/tests/golden/codex/runbook.golden.md b/tests/golden/codex/runbook.golden.md index 6bbb1b9..c1ec038 100644 --- a/tests/golden/codex/runbook.golden.md +++ b/tests/golden/codex/runbook.golden.md @@ -4,7 +4,7 @@ This runbook is for a human driving the run from a terminal. Work from this iter and copy-paste each step. The workspace is self-contained — you should not need the surrounding repo. -> **Requires:** eval-magic's dispatch and judge recipes are POSIX command lines built on `jq`, `xargs`, `tr`, and `wc`. Run them in a POSIX shell with `jq` installed that resolves the same paths this workspace was prepared with — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. +> **Requires:** harness dispatch commands are POSIX command lines, and `eval-magic dispatch` runs them itself, so the host it runs on needs a POSIX shell — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. - **Skill under test:** widget-skill - **Mode:** revision — comparing `old_skill` vs `new_skill` @@ -12,15 +12,20 @@ repo. ## 1. Dispatch the eval agents, then ingest -Next: iterate the tasks[] array in dispatch.json and dispatch each task with: -unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES -codex --ask-for-approval never exec --cd --sandbox workspace-write --dangerously-bypass-hook-trust -m model-x --json \ - --output-last-message /final-message.md \ - "Read the file at and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to /final-message.md." \ - /codex-events.jsonl \ - 2> /codex-stderr.log -Then run `ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness codex`. +``` +eval-magic dispatch --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness codex +``` + +`dispatch` runs every task in its own private environment, `--jobs` of them at a time, and writes +each task's `conversation.json`. A task that already has one is skipped, so rerunning the same +command retries only what did not finish. A task that exceeds `--timeout` is recorded as timed out +rather than left to stall the campaign, and a task that fails is recorded and named while the rest +of the batch continues. A conversation that stops at a scripted gate is valid eval data, not a +failure. + +``` +eval-magic ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness codex +``` `ingest` records each run, backfills transcripts, scans for stray writes, collects guarded-task blocks into `guard-denials.json`, and grades every mechanical assertion. Inspect any denial @@ -28,43 +33,13 @@ warning before trusting the affected task. It then prints any `llm_judge` tasks grade itself. ## 2. Dispatch the judge agents, then finalize -Dispatch each judge task from judge-tasks.json with: -Existing nonempty response files are skipped; delete one to dispatch that judge again. -The final `N/M verdicts present` summary exits nonzero until every task has one. - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - prompt_path="$1" - response_path="$2" - model="${3#model=}" - if [ -s "$response_path" ]; then exit 0; fi - response_base="${response_path%.json}" - mkdir -p "$(dirname "$response_path")" - model_arg=""; [ -n "$model" ] && model_arg="-m $model" - codex --ask-for-approval never exec --cd "/work/.eval-magic/widget-skill/iteration-2" --sandbox workspace-write $model_arg --json \ - "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ - "$response_base.codex-events.jsonl" \ - 2> "$response_base.codex-stderr.log" - ' sh -judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') -judge_present=$( - jq -r '.tasks[].response_path' judge-tasks.json \ - | tr -d '\r' \ - | while IFS= read -r response_path; do - if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi - done \ - | wc -l \ - | tr -d '[:space:]' -) -printf '%s/%s verdicts present\n' "$judge_present" "$judge_total" -[ "$judge_dispatch_status" -eq 0 ] && [ "$judge_present" -eq "$judge_total" ] + ``` +eval-magic dispatch --judges --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness codex +``` + +Verdicts that are already present are skipped; the summary prints `N/M verdicts present` and exits +nonzero until every task has one, so rerun the same command to fill the gaps. Then merge the verdicts and aggregate: diff --git a/tests/golden/opencode/judge-recipe.golden.md b/tests/golden/opencode/judge-recipe.golden.md deleted file mode 100644 index d7ce316..0000000 --- a/tests/golden/opencode/judge-recipe.golden.md +++ /dev/null @@ -1,37 +0,0 @@ -Dispatch each judge task from judge-tasks.json with: -Existing nonempty response files are skipped; delete one to dispatch that judge again. -The final `N/M verdicts present` summary exits nonzero until every task has one. - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - prompt_path="$1" - response_path="$2" - model="${3#model=}" - if [ -s "$response_path" ]; then exit 0; fi - response_base="${response_path%.json}" - mkdir -p "$(dirname "$response_path")" - model_arg=""; [ -n "$model" ] && model_arg="-m $model" - opencode run --dir "/work/iter-1" --format json --auto $model_arg \ - "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ - "$response_base.opencode-events.jsonl" \ - 2> "$response_base.opencode-stderr.log" - ' sh -judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') -judge_present=$( - jq -r '.tasks[].response_path' judge-tasks.json \ - | tr -d '\r' \ - | while IFS= read -r response_path; do - if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi - done \ - | wc -l \ - | tr -d '[:space:]' -) -printf '%s/%s verdicts present\n' "$judge_present" "$judge_total" -[ "$judge_dispatch_status" -eq 0 ] && [ "$judge_present" -eq "$judge_total" ] -``` \ No newline at end of file diff --git a/tests/golden/opencode/manifest.golden.md b/tests/golden/opencode/manifest.golden.md index 9acfc20..66110d4 100644 --- a/tests/golden/opencode/manifest.golden.md +++ b/tests/golden/opencode/manifest.golden.md @@ -8,11 +8,19 @@ Total dispatches: 2 In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the task with a short "read this file and follow it" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`. -**Requires:** eval-magic's dispatch and judge recipes are POSIX command lines built on `jq`, `xargs`, `tr`, and `wc`. Run them in a POSIX shell with `jq` installed that resolves the same paths this workspace was prepared with — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. +**Requires:** harness dispatch commands are POSIX command lines, and `eval-magic dispatch` runs them itself, so the host it runs on needs a POSIX shell — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. -After all dispatches (OpenCode): +## Dispatch -Run one fresh `opencode run --format json --auto` per task. Detach stdin with ` --harness + +It runs `--jobs` tasks at a time, each in its own private environment, and writes each task's conversation.json. A task that already has one is skipped, so rerunning retries only what did not finish. A task exceeding `--timeout` is recorded as timed out, and a failing task is recorded while the rest of the batch continues. A conversation that stops at a scripted gate is valid eval data; a task with no conversation.json is incomplete and ingest skips it. + +Harness dispatch (OpenCode): + +`eval-magic dispatch` runs one fresh `opencode run --format json --auto` per task. Detach stdin with `/opencode-events.jsonl` and stderr as `outputs/turn-/opencode-stderr.log`. ```bash unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES @@ -23,33 +31,12 @@ opencode run --dir --format json --auto -m model-x \ 2> /opencode-stderr.log ``` -Parallel dispatch from this iteration directory: - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - eval_root="$1" - prompt_path="$2" - outputs_dir="$3" - mkdir -p "$outputs_dir" - unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES - opencode run --dir "$eval_root" --format json --auto -m model-x \ - "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response your closing summary." \ - "$outputs_dir/opencode-events.jsonl" \ - 2> "$outputs_dir/opencode-stderr.log" - ' sh -``` - -Then run `eval-magic ingest --harness opencode`; OpenCode transcript ingest reads each task's `outputs/opencode-events.jsonl`. +Then run `eval-magic ingest --harness opencode`; OpenCode transcript ingest reads each task's `outputs/turn-/opencode-events.jsonl`. After all dispatches: -1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. -2. Dispatch the judge tasks ingest lists, then run `eval-magic finalize` for the benchmark. +1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/turn-/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. +2. Run `eval-magic dispatch --judges --harness ` to grade the judge tasks ingest listed, then `eval-magic finalize` for the benchmark. On a harness without persisted transcripts, instead write each task's `run.json` (matching `skills/evaluating-skills/schema/run-record.schema.json`, enforced at runtime by grade/fill-transcripts/detect-stray-writes) and `timing.json` by hand when its subagent returns: carry over `eval_id`, `condition`, `skill_path` (`null` on the without_skill arm), `prompt`, and `files` from the task; populate `final_message` from the subagent's reply; leave `tool_invocations` as `[]`; capture `total_tokens`/`duration_ms` from the task completion event immediately — they may not be persisted anywhere else. @@ -58,6 +45,7 @@ On a harness without persisted transcripts, instead write each task's `run.json` - run.json: /work/cond/run.json - timing.json: /work/cond/timing.json +- conversation.json: /work/cond/conversation.json ``` @@ -108,6 +96,7 @@ Build me a widget. - run.json: /work/cond-b/run.json - timing.json: /work/cond-b/timing.json +- conversation.json: /work/cond-b/conversation.json ``` You are executing a single test case for a skill evaluation framework. diff --git a/tests/golden/opencode/next-steps-model.golden.txt b/tests/golden/opencode/next-steps-model.golden.txt deleted file mode 100644 index b19427a..0000000 --- a/tests/golden/opencode/next-steps-model.golden.txt +++ /dev/null @@ -1,9 +0,0 @@ - -Next: iterate the tasks[] array in dispatch.json and dispatch each task with: -unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES -opencode run --dir --format json --auto -m model-x \ - "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ - /opencode-events.jsonl \ - 2> /opencode-stderr.log -Then run `ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness opencode`. \ No newline at end of file diff --git a/tests/golden/opencode/next-steps-nomodel.golden.txt b/tests/golden/opencode/next-steps-nomodel.golden.txt deleted file mode 100644 index 4f32ce6..0000000 --- a/tests/golden/opencode/next-steps-nomodel.golden.txt +++ /dev/null @@ -1,9 +0,0 @@ - -Next: iterate the tasks[] array in dispatch.json and dispatch each task with: -unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES -opencode run --dir --format json --auto \ - "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ - /opencode-events.jsonl \ - 2> /opencode-stderr.log -Then run `ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness opencode`. \ No newline at end of file diff --git a/tests/golden/opencode/next-steps.golden.txt b/tests/golden/opencode/next-steps.golden.txt new file mode 100644 index 0000000..6fff1ec --- /dev/null +++ b/tests/golden/opencode/next-steps.golden.txt @@ -0,0 +1,3 @@ + +Next: eval-magic dispatch --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness opencode +Then run `eval-magic ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness opencode`. \ No newline at end of file diff --git a/tests/golden/opencode/runbook.golden.md b/tests/golden/opencode/runbook.golden.md index bdd0107..b86f7eb 100644 --- a/tests/golden/opencode/runbook.golden.md +++ b/tests/golden/opencode/runbook.golden.md @@ -4,7 +4,7 @@ This runbook is for a human driving the run from a terminal. Work from this iter and copy-paste each step. The workspace is self-contained — you should not need the surrounding repo. -> **Requires:** eval-magic's dispatch and judge recipes are POSIX command lines built on `jq`, `xargs`, `tr`, and `wc`. Run them in a POSIX shell with `jq` installed that resolves the same paths this workspace was prepared with — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. +> **Requires:** harness dispatch commands are POSIX command lines, and `eval-magic dispatch` runs them itself, so the host it runs on needs a POSIX shell — on Windows, Git Bash (Git for Windows). WSL resolves a different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. Set EVAL_MAGIC_SH to select a specific `sh`. - **Skill under test:** widget-skill - **Mode:** revision — comparing `old_skill` vs `new_skill` @@ -12,14 +12,20 @@ repo. ## 1. Dispatch the eval agents, then ingest -Next: iterate the tasks[] array in dispatch.json and dispatch each task with: -unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_COMMON_DIR GIT_CEILING_DIRECTORIES -opencode run --dir --format json --auto -m model-x \ - "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ - /opencode-events.jsonl \ - 2> /opencode-stderr.log -Then run `ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness opencode`. +``` +eval-magic dispatch --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness opencode +``` + +`dispatch` runs every task in its own private environment, `--jobs` of them at a time, and writes +each task's `conversation.json`. A task that already has one is skipped, so rerunning the same +command retries only what did not finish. A task that exceeds `--timeout` is recorded as timed out +rather than left to stall the campaign, and a task that fails is recorded and named while the rest +of the batch continues. A conversation that stops at a scripted gate is valid eval data, not a +failure. + +``` +eval-magic ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness opencode +``` `ingest` records each run, backfills transcripts, scans for stray writes, collects guarded-task blocks into `guard-denials.json`, and grades every mechanical assertion. Inspect any denial @@ -27,43 +33,13 @@ warning before trusting the affected task. It then prints any `llm_judge` tasks grade itself. ## 2. Dispatch the judge agents, then finalize -Dispatch each judge task from judge-tasks.json with: -Existing nonempty response files are skipped; delete one to dispatch that judge again. -The final `N/M verdicts present` summary exits nonzero until every task has one. - -```bash -JOBS=${JOBS:-4} -jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ - | tr -d '\r' \ - | tr '\n' '\0' \ - | xargs -0 -P "$JOBS" -n 3 sh -c ' - prompt_path="$1" - response_path="$2" - model="${3#model=}" - if [ -s "$response_path" ]; then exit 0; fi - response_base="${response_path%.json}" - mkdir -p "$(dirname "$response_path")" - model_arg=""; [ -n "$model" ] && model_arg="-m $model" - opencode run --dir "/work/.eval-magic/widget-skill/iteration-2" --format json --auto $model_arg \ - "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ - "$response_base.opencode-events.jsonl" \ - 2> "$response_base.opencode-stderr.log" - ' sh -judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') -judge_present=$( - jq -r '.tasks[].response_path' judge-tasks.json \ - | tr -d '\r' \ - | while IFS= read -r response_path; do - if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi - done \ - | wc -l \ - | tr -d '[:space:]' -) -printf '%s/%s verdicts present\n' "$judge_present" "$judge_total" -[ "$judge_dispatch_status" -eq 0 ] && [ "$judge_present" -eq "$judge_total" ] + ``` +eval-magic dispatch --judges --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness opencode +``` + +Verdicts that are already present are skipped; the summary prints `N/M verdicts present` and exits +nonzero until every task has one, so rerun the same command to fill the gaps. Then merge the verdicts and aggregate: diff --git a/tests/run/agent_env.rs b/tests/run/agent_env.rs index c3b7d43..2781859 100644 --- a/tests/run/agent_env.rs +++ b/tests/run/agent_env.rs @@ -53,17 +53,11 @@ fn descriptor_defaults_and_cli_overrides_are_recorded_and_rendered() { expected ); - let manifest = read_str(&iteration.join("dispatch-manifest.md")); - assert!( - manifest.contains("export EMPTY=\nexport MODE=cli\nexport TZ=UTC"), - "{manifest}" - ); + // The environment travels in the dispatch envelope, not in pasted `export` + // lines: the runner applies it per task when it spawns the harness, so + // neither the manifest nor the runbook restates it. let runbook = read_str(&iteration.join("RUNBOOK.md")); - let judge = runbook - .split_once("## 2. Dispatch the judge agents, then finalize") - .unwrap() - .1; - assert!(!judge.contains("export TZ=UTC"), "{judge}"); + assert!(!runbook.contains("export TZ=UTC"), "{runbook}"); } #[test] @@ -91,8 +85,10 @@ fn cli_agent_environment_renders_for_every_builtin_harness() { .assert() .success(); - let manifest = read_str(&iteration_dir(&cwd).join("dispatch-manifest.md")); - assert!(manifest.contains("export TZ=UTC"), "{harness}: {manifest}"); + // Recorded once, in the envelope the runner dispatches from, for every + // built-in harness. + let envelope = read_json(&iteration_dir(&cwd).join("dispatch.json")); + assert_eq!(envelope["agent_env"]["TZ"], "UTC", "{harness}: {envelope}"); } } @@ -148,9 +144,18 @@ fn dispatch_task_revalidates_persisted_agent_environment() { skill_eval() .current_dir(&cwd) - .args(["dispatch-task", "--dispatch"]) - .arg(&dispatch_path) - .args(["--task-index", "0"]) + .args(["dispatch", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + "--task-index", + "0", + ]) .assert() .failure() .stderr(predicates::str::contains("BAD-NAME")); diff --git a/tests/run/byoh.rs b/tests/run/byoh.rs index 16a5c71..f456fa5 100644 --- a/tests/run/byoh.rs +++ b/tests/run/byoh.rs @@ -82,11 +82,12 @@ fn descriptor_alone_carries_a_complete_run() { .and(contains("provenance")), ); - // The exec recipe reached both human-facing artifacts. + // The runbook drives through the runner, so it names the command rather + // than the harness CLI; the manifest still shows what the runner spawns. let runbook = read_str(&iteration_dir(&cwd).join("RUNBOOK.md")); - assert!(runbook.contains("cool-cli run"), "{runbook}"); + assert!(runbook.contains("eval-magic dispatch"), "{runbook}"); let manifest = read_str(&iteration_dir(&cwd).join("dispatch-manifest.md")); - assert!(manifest.contains("## Dispatch recipe"), "{manifest}"); + assert!(manifest.contains("## Dispatch"), "{manifest}"); assert!(manifest.contains("cool-cli run"), "{manifest}"); // Forced --no-stage: nothing was staged, so no task carries a staged slug. @@ -137,12 +138,13 @@ fn descriptor_alone_carries_a_complete_run() { } } -/// A descriptor without an exec_template warns naming the generic handoff: -/// RUNBOOK.md and dispatch-manifest.md carry guidance, not a copy-pasteable -/// per-task command. (The built-in-harness half of this pin — wired harnesses -/// stay quiet — lives in src/cli/run/util.rs.) +/// A descriptor without an exec_template warns at prep time, because the runner +/// will have nothing to spawn: `eval-magic dispatch` fails outright for such a +/// harness, so the gap is worth naming before the workspace is built. (The +/// built-in-harness half of this pin — wired harnesses stay quiet — lives in +/// src/cli/run/util.rs.) #[test] -fn dispatchless_descriptor_warns_naming_the_generic_handoff() { +fn dispatchless_descriptor_warns_that_dispatch_has_nothing_to_run() { let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); write_project_descriptor(&cwd, "label = \"cool-custom-harness\"\n"); @@ -161,7 +163,11 @@ fn dispatchless_descriptor_warns_naming_the_generic_handoff() { ]) .assert() .success() - .stderr(contains("declares no dispatch exec recipe").and(contains("RUNBOOK.md"))); + .stderr( + contains("declares no dispatch exec template") + .and(contains("eval-magic dispatch")) + .and(contains("eval-magic docs byoh")), + ); } /// `--guard` with a harness that exists only in user-supplied descriptors is diff --git a/tests/run/claude_cli.rs b/tests/run/claude_cli.rs index e487b95..6a69a2e 100644 --- a/tests/run/claude_cli.rs +++ b/tests/run/claude_cli.rs @@ -26,16 +26,19 @@ fn claude_dispatch_guidance_uses_claude_p() { .success(); let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); - assert!(stdout.contains("claude -p --output-format stream-json")); - assert!(stdout.contains("--verbose")); - assert!(stdout.contains("cd ")); - assert!(stdout.contains("claude-events.jsonl")); - assert!(!stdout.contains("--output-last-message")); + // The post-run hand-off names the runner command; the harness CLI it will + // spawn is documented in the manifest, not pasted at the operator. + assert!(stdout.contains("eval-magic dispatch"), "{stdout}"); + assert!(stdout.contains("--harness claude-code"), "{stdout}"); let manifest = read_str(&iteration_dir(&cwd).join("dispatch-manifest.md")); assert!(manifest.contains("claude -p --output-format stream-json")); + assert!(manifest.contains("--verbose")); + assert!(manifest.contains("cd ")); assert!(manifest.contains("claude-events.jsonl")); - assert!(manifest.contains("xargs -0 -P")); + assert!(!manifest.contains("--output-last-message")); + // Concurrency is the runner's `--jobs`, not a pasted `xargs -P` pipeline. + assert!(manifest.contains("eval-magic dispatch")); let conditions = read_json(&iteration_dir(&cwd).join("conditions.json")); assert_eq!(conditions["harness"], "claude-code"); @@ -59,9 +62,10 @@ fn claude_dispatch_guidance_includes_agent_model_when_provided() { ]) .assert() .success(); - let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); - assert!(stdout.contains("claude -p --output-format stream-json")); - assert!(stdout.contains("--model opus")); + assert.success(); + let manifest = read_str(&iteration_dir(&cwd).join("dispatch-manifest.md")); + assert!(manifest.contains("claude -p --output-format stream-json")); + assert!(manifest.contains("--model opus"), "{manifest}"); } #[test] @@ -87,15 +91,15 @@ fn claude_run_writes_human_followed_runbook() { // Each task dispatches from its own per-(group, condition) env, so the shared // human-followed runbook lives in the iteration dir, above those envs, and - // carries the claude -p recipe plus the --harness-threaded pipeline commands. + // carries the runner commands with --harness threaded through them. let runbook = read_str(&iteration_dir(&cwd).join("RUNBOOK.md")); assert!( runbook.contains("human driving"), "uses the human-followed template: {runbook}" ); assert!( - runbook.contains("claude -p"), - "carries the claude -p dispatch recipe: {runbook}" + runbook.contains("eval-magic dispatch"), + "carries the dispatch command: {runbook}" ); assert!( runbook.contains("--harness claude-code"), diff --git a/tests/run/codex.rs b/tests/run/codex.rs index 188d22f..db0f5d1 100644 --- a/tests/run/codex.rs +++ b/tests/run/codex.rs @@ -228,13 +228,11 @@ fn codex_dispatch_guidance_detaches_stdin_and_logs_stderr() { ]) .assert() .success(); + // The post-run hand-off names the runner command; the harness CLI it will + // spawn is documented in the manifest, not pasted at the operator. let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); - - assert!(stdout.contains("codex --ask-for-approval never exec --cd ")); - assert!(stdout.contains("--dangerously-bypass-hook-trust")); - assert!(stdout.contains("")); @@ -242,7 +240,8 @@ fn codex_dispatch_guidance_detaches_stdin_and_logs_stderr() { assert!(manifest.contains("")); - assert!(stdout.contains("-m gpt-5-mini")); - assert!(stdout.contains("")); assert!(manifest.contains("-m gpt-5-mini")); - assert!(manifest.contains("xargs -0 -P")); + assert!(manifest.contains("")); - assert!(stdout.contains("")); + assert!(manifest.contains(" assert_cmd::Command { + let mut command = skill_eval(); + command + .current_dir(cwd) + .args(["dispatch", "--skill-dir"]) + .arg(skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + harness, + "--task-index", + &index.to_string(), + ]); + if overwrite { + command.arg("--overwrite"); + } + command +} + +/// Swap the frozen descriptor's `exec_template` for a stub, the way every +/// driver test here substitutes a real harness CLI. +fn stub_exec_template(cwd: &Path, template: &str) { + let dispatch_path = iteration_dir(cwd).join("dispatch.json"); + let mut dispatch = read_json(&dispatch_path); + dispatch["harness_descriptor"]["dispatch"]["exec_template"] = serde_json::json!(template); + fs::write( + &dispatch_path, + format!("{}\n", serde_json::to_string_pretty(&dispatch).unwrap()), + ) + .unwrap(); +} + +/// A one-shot harness stub: emits a session id, one agent message, and a usage +/// event, which is the minimum a transcript needs to parse. Written as a POSIX +/// script and invoked through `sh` for the same reason the scripted stub below +/// is — that is the shape of a real `exec_template`, and it needs no executable +/// bit on any host. +fn one_shot_stub(dir: &Path, message: &str) -> String { + let script = dir.join("fake-one-shot.sh"); + fs::write( + &script, + r#"#!/bin/sh +outputs=$1 +message=$2 +printf '%s\n' '{"type":"thread.started","thread_id":"session-1"}' > "$outputs/codex-events.jsonl" +printf '%s\n' "{\"type\":\"item.completed\",\"item\":{\"id\":\"m1\",\"type\":\"agent_message\",\"text\":\"$message\"}}" >> "$outputs/codex-events.jsonl" +printf '%s\n' '{"type":"turn.completed","usage":{"input_tokens":2,"output_tokens":3}}' >> "$outputs/codex-events.jsonl" +"#, + ) + .unwrap(); + format!( + "sh \"{}\" \"{message}\"", + script.to_string_lossy() + ) +} + // The harness stub below is a POSIX shell script, because that is what a real `exec_template` is — // every descriptor in `harnesses/` ships one (`` redirection). // The driver resolves an `sh` on every host, and the template invokes the stub *through* that `sh` @@ -155,11 +363,7 @@ printf '%s\n' '{"type":"turn.completed","usage":{"input_tokens":2,"output_tokens ) .unwrap(); - skill_eval() - .current_dir(&cwd) - .args(["dispatch-task", "--dispatch"]) - .arg(&dispatch_path) - .args(["--task-index", "0"]) + dispatch_one(&skill_dir, &cwd, "codex", 0, false) .assert() .success(); @@ -185,11 +389,7 @@ printf '%s\n' '{"type":"turn.completed","usage":{"input_tokens":2,"output_tokens "Updated the date handling.\n" ); - skill_eval() - .current_dir(&cwd) - .args(["dispatch-task", "--dispatch"]) - .arg(&dispatch_path) - .args(["--task-index", "1"]) + dispatch_one(&skill_dir, &cwd, "codex", 1, false) .assert() .success(); @@ -218,11 +418,7 @@ printf '%s\n' '{"type":"turn.completed","usage":{"input_tokens":2,"output_tokens ) .unwrap(); - skill_eval() - .current_dir(&cwd) - .args(["dispatch-task", "--dispatch"]) - .arg(&dispatch_path) - .args(["--task-index", "0", "--overwrite"]) + dispatch_one(&skill_dir, &cwd, "codex", 0, true) .assert() .failure(); assert!( diff --git a/tests/run/conversation/dispatch.rs b/tests/run/conversation/dispatch.rs new file mode 100644 index 0000000..710d668 --- /dev/null +++ b/tests/run/conversation/dispatch.rs @@ -0,0 +1,401 @@ +//! The batch behaviors `eval-magic dispatch` owns: running a whole plan, what a +//! rerun may skip, how a failure and a timeout are recorded, and concurrency. +//! +//! The single-task driver those batches call lives beside this module, in +//! [`super`]. + +use super::{ + ONE_SHOT_EVALS, dispatch_one, one_shot_stub, prepare_one_shot_run, stub_exec_template, +}; +use crate::helpers::*; +use predicates::str::contains; +use std::fs; +use std::path::Path; + +/// `dispatch` drives every task in the plan from one command, which is the +/// whole point of the ticket: no operator pastes a per-task recipe any more. +#[test] +fn dispatch_drives_every_task_in_the_plan() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), ONE_SHOT_EVALS); + prepare_one_shot_run(&skill_dir, &cwd, "codex"); + stub_exec_template( + &cwd, + &one_shot_stub(tmp.path(), "Updated the date handling."), + ); + + skill_eval() + .current_dir(&cwd) + .args(["dispatch", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + ]) + .assert() + .success(); + + let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json")); + let tasks = dispatch["tasks"].as_array().unwrap(); + assert_eq!(tasks.len(), 2, "both conditions dispatch"); + for task in tasks { + let conversation = read_json(Path::new(task["conversation_path"].as_str().unwrap())); + assert_eq!(conversation["status"], "completed", "{conversation}"); + assert_eq!(conversation["delivered_followups"], 0); + } +} + +/// Rerunning a dispatch must not redo finished work: the completion artifact is +/// the marker, so a second run skips what completed and retries only what did +/// not. Proven by a counter the stub appends to once per invocation. +#[test] +fn rerunning_dispatch_skips_completed_tasks_and_retries_the_rest() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), ONE_SHOT_EVALS); + prepare_one_shot_run(&skill_dir, &cwd, "codex"); + let counter = tmp.path().join("dispatch-count.log"); + stub_exec_template( + &cwd, + &counting_stub(tmp.path(), &counter, "Updated the date handling."), + ); + + // Dispatch one task, leaving the other without a completion artifact. + dispatch_one(&skill_dir, &cwd, "codex", 0, false) + .assert() + .success(); + assert_eq!(dispatch_count(&counter), 1); + + // The whole batch: the finished task is skipped, the other one runs. + skill_eval() + .current_dir(&cwd) + .args(["dispatch", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + ]) + .assert() + .success() + .stdout(contains( + "1 completed, 0 stopped, 0 timed out, 0 failed, 1 skipped", + )); + assert_eq!( + dispatch_count(&counter), + 2, + "the completed task must not be dispatched twice" + ); +} + +/// A failing task is campaign data, not a reason to abandon the batch: the rest +/// still runs, the failure is named, and the command exits nonzero so a script +/// notices. +#[test] +fn a_failing_task_is_recorded_and_the_rest_of_the_batch_still_runs() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), ONE_SHOT_EVALS); + prepare_one_shot_run(&skill_dir, &cwd, "codex"); + + // Fails for the first condition's env, succeeds for the second. + let script = tmp.path().join("fail-one.sh"); + fs::write( + &script, + r#"#!/bin/sh +outputs=$1 +eval_root=$2 +case "$eval_root" in + *-with_skill) exit 9 ;; +esac +printf '%s +' '{"type":"thread.started","thread_id":"session-1"}' > "$outputs/codex-events.jsonl" +printf '%s +' '{"type":"item.completed","item":{"id":"m1","type":"agent_message","text":"Done."}}' >> "$outputs/codex-events.jsonl" +printf '%s +' '{"type":"turn.completed","usage":{"input_tokens":2,"output_tokens":3}}' >> "$outputs/codex-events.jsonl" +"#, + ) + .unwrap(); + stub_exec_template( + &cwd, + &format!( + "sh \"{}\" ", + script.to_string_lossy() + ), + ); + + skill_eval() + .current_dir(&cwd) + .args(["dispatch", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + ]) + .assert() + .failure() + .stdout(contains( + "1 completed, 0 stopped, 0 timed out, 1 failed, 0 skipped", + )) + .stderr(contains("one-shot:with_skill")); + + let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json")); + let tasks = dispatch["tasks"].as_array().unwrap(); + let failed = Path::new(tasks[0]["conversation_path"].as_str().unwrap()); + let succeeded = Path::new(tasks[1]["conversation_path"].as_str().unwrap()); + assert!( + !failed.exists(), + "a failed task writes no completion artifact, so a rerun retries it" + ); + assert!(succeeded.is_file(), "the healthy task still completed"); +} + +/// A hung dispatch must not hang the campaign: it is killed at the deadline, +/// recorded as timed out, and every other task still finishes. Without this, +/// `execute_round` ran to completion however long that took. +#[test] +fn a_task_that_outruns_the_timeout_is_recorded_and_the_batch_finishes() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), ONE_SHOT_EVALS); + prepare_one_shot_run(&skill_dir, &cwd, "codex"); + + // `with_skill` hangs well past the deadline; the other arm answers at once. + let script = tmp.path().join("hang-one.sh"); + fs::write( + &script, + r#"#!/bin/sh +outputs=$1 +eval_root=$2 +exe=$3 +case "$eval_root" in + *-with_skill) "$exe" __fixture --sleep-ms 5000 ;; +esac +printf '%s +' '{"type":"thread.started","thread_id":"session-1"}' > "$outputs/codex-events.jsonl" +printf '%s +' '{"type":"item.completed","item":{"id":"m1","type":"agent_message","text":"Done."}}' >> "$outputs/codex-events.jsonl" +printf '%s +' '{"type":"turn.completed","usage":{"input_tokens":2,"output_tokens":3}}' >> "$outputs/codex-events.jsonl" +"#, + ) + .unwrap(); + stub_exec_template( + &cwd, + &format!( + "sh \"{}\" \"{}\"", + script.to_string_lossy(), + env!("CARGO_BIN_EXE_eval-magic") + ), + ); + + skill_eval() + .current_dir(&cwd) + .args(["dispatch", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + "--timeout", + "1", + ]) + .assert() + .failure() + .stdout(contains( + "1 completed, 0 stopped, 1 timed out, 0 failed, 0 skipped", + )); + + let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json")); + let tasks = dispatch["tasks"].as_array().unwrap(); + let hung = read_json(Path::new(tasks[0]["conversation_path"].as_str().unwrap())); + assert_eq!(hung["status"], "timed_out", "{hung}"); + assert_eq!(hung["timed_out_in_round"], 1); + let healthy = read_json(Path::new(tasks[1]["conversation_path"].as_str().unwrap())); + assert_eq!(healthy["status"], "completed", "{healthy}"); +} + +/// `--jobs` runs tasks concurrently. Each task is a private environment, so +/// they are independent; four one-second dispatches must therefore finish in +/// well under the four seconds they would take in sequence. +#[test] +fn jobs_runs_tasks_concurrently() { + let tmp = tempfile::TempDir::new().unwrap(); + let evals = r#"{ + "skill_name": "mr-review", + "evals": [ + {"id": "a", "prompt": "Fix the date.", "expected_output": "fixed"}, + {"id": "b", "prompt": "Fix the time.", "expected_output": "fixed"} + ] + }"#; + let (skill_dir, cwd) = setup(tmp.path(), evals); + prepare_one_shot_run(&skill_dir, &cwd, "codex"); + + let script = tmp.path().join("slow-stub.sh"); + fs::write( + &script, + r#"#!/bin/sh +outputs=$1 +exe=$2 +"$exe" __fixture --sleep-ms 1000 +printf '%s +' '{"type":"thread.started","thread_id":"session-1"}' > "$outputs/codex-events.jsonl" +printf '%s +' '{"type":"item.completed","item":{"id":"m1","type":"agent_message","text":"Done."}}' >> "$outputs/codex-events.jsonl" +printf '%s +' '{"type":"turn.completed","usage":{"input_tokens":2,"output_tokens":3}}' >> "$outputs/codex-events.jsonl" +"#, + ) + .unwrap(); + stub_exec_template( + &cwd, + &format!( + "sh \"{}\" \"{}\"", + script.to_string_lossy(), + env!("CARGO_BIN_EXE_eval-magic") + ), + ); + + let started = std::time::Instant::now(); + skill_eval() + .current_dir(&cwd) + .args(["dispatch", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + "--jobs", + "4", + ]) + .assert() + .success() + .stdout(contains("4 completed")); + let elapsed = started.elapsed(); + + // Four sequential one-second dispatches take at least four seconds. The + // ceiling is deliberately loose — this asserts concurrency happened, not + // how fast a loaded CI runner schedules four processes. + assert!( + elapsed < std::time::Duration::from_millis(2500), + "four concurrent 1s dispatches took {elapsed:?}, which is serial" + ); +} + +/// Mode B dispatches through the same command Mode A does. Its conditions are +/// two skill revisions rather than skill-versus-none, and nothing about how a +/// task is driven may depend on which mode produced it. +#[test] +fn revision_mode_dispatches_through_the_same_command() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), ONE_SHOT_EVALS); + + skill_eval() + .current_dir(&cwd) + .args(["snapshot", "--skill-dir"]) + .arg(&skill_dir) + .args(["--skill", "mr-review", "--label", "baseline"]) + .assert() + .success(); + skill_eval() + .current_dir(&cwd) + .args(["run", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--mode", + "revision", + "--harness", + "codex", + "--no-guard", + ]) + .assert() + .success(); + stub_exec_template( + &cwd, + &one_shot_stub(tmp.path(), "Updated the date handling."), + ); + + skill_eval() + .current_dir(&cwd) + .args(["dispatch", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + ]) + .assert() + .success() + .stdout(contains("2 completed")); + + let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json")); + let tasks = dispatch["tasks"].as_array().unwrap(); + let mut conditions: Vec<&str> = tasks + .iter() + .map(|task| task["condition"].as_str().unwrap()) + .collect(); + conditions.sort_unstable(); + assert_eq!( + conditions, + ["new_skill", "old_skill"], + "revision mode compares two skill revisions" + ); + for task in tasks { + let conversation = read_json(Path::new(task["conversation_path"].as_str().unwrap())); + assert_eq!(conversation["status"], "completed", "{conversation}"); + } +} + +/// A stub that records each invocation, so a test can prove how many dispatches +/// actually happened. +fn counting_stub(dir: &Path, counter: &Path, message: &str) -> String { + let script = dir.join("counting-stub.sh"); + fs::write( + &script, + r#"#!/bin/sh +outputs=$1 +counter=$2 +message=$3 +printf 'x +' >> "$counter" +printf '%s +' '{"type":"thread.started","thread_id":"session-1"}' > "$outputs/codex-events.jsonl" +printf '%s +' "{\"type\":\"item.completed\",\"item\":{\"id\":\"m1\",\"type\":\"agent_message\",\"text\":\"$message\"}}" >> "$outputs/codex-events.jsonl" +printf '%s +' '{"type":"turn.completed","usage":{"input_tokens":2,"output_tokens":3}}' >> "$outputs/codex-events.jsonl" +"#, + ) + .unwrap(); + format!( + "sh \"{}\" \"{}\" \"{message}\"", + script.to_string_lossy(), + counter.to_string_lossy() + ) +} + +fn dispatch_count(counter: &Path) -> usize { + fs::read_to_string(counter) + .map(|body| body.lines().count()) + .unwrap_or(0) +} diff --git a/tests/run/judges.rs b/tests/run/judges.rs new file mode 100644 index 0000000..3c5b915 --- /dev/null +++ b/tests/run/judges.rs @@ -0,0 +1,313 @@ +//! Runner-driven judge dispatch: `eval-magic dispatch --judges`. + +use crate::helpers::*; +use predicates::str::contains; +use std::fs; +use std::path::Path; + +/// Two `llm_judge` assertions, so both arms emit judge tasks and one condition +/// carries more than one — which is what makes a shared capture path collide. +const JUDGED_EVALS: &str = r#"{ + "skill_name": "mr-review", + "evals": [{ + "id": "reviewed", + "prompt": "Review this MR.", + "expected_output": "a clear review", + "assertions": [ + {"id": "clear", "type": "llm_judge", "rubric": "Was the review clear?"}, + {"id": "concise", "type": "llm_judge", "rubric": "Was the review concise?"} + ] + }] +}"#; + +/// The runner dispatches judge tasks the same way it dispatches eval tasks: it +/// skips verdicts that already exist, runs the ones that do not, and reports +/// how many are present. Before this, an operator pasted a `jq`/`xargs` +/// pipeline out of `RUNBOOK.md` to do it. +#[test] +fn dispatch_judges_runs_missing_verdicts_and_skips_present_ones() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), JUDGED_EVALS); + prepare_and_dispatch(tmp.path(), &skill_dir, &cwd); + + let judge_tasks = read_json(&iteration_dir(&cwd).join("judge-tasks.json")); + let tasks = judge_tasks["tasks"].as_array().unwrap().clone(); + assert!(tasks.len() >= 2, "both conditions judge: {judge_tasks}"); + + // Pre-answer the first task; the runner must leave it alone. + let answered = Path::new(tasks[0]["response_path"].as_str().unwrap()); + fs::create_dir_all(answered.parent().unwrap()).unwrap(); + fs::write( + answered, + r#"{"passed":true,"evidence":"pre-existing","confidence":0.9}"#, + ) + .unwrap(); + + stub_judge_template(&cwd, &judge_stub(tmp.path())); + + skill_eval() + .current_dir(&cwd) + .args(["dispatch", "--judges", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + ]) + .assert() + .success() + .stdout(contains(format!( + "{}/{} verdicts present", + tasks.len(), + tasks.len() + ))); + + assert_eq!( + fs::read_to_string(answered).unwrap(), + r#"{"passed":true,"evidence":"pre-existing","confidence":0.9}"#, + "an existing verdict is never redispatched" + ); + for task in &tasks[1..] { + let response = read_json(Path::new(task["response_path"].as_str().unwrap())); + assert_eq!(response["evidence"], "stub verdict", "{response}"); + } +} + +/// Every judge task captures its transcript in its own directory. Several +/// assertions share one `judge-responses/` directory, so binding the capture to +/// that directory would have them overwrite each other's events file. +#[test] +fn each_judge_task_captures_its_transcript_separately() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), JUDGED_EVALS); + prepare_and_dispatch(tmp.path(), &skill_dir, &cwd); + stub_judge_template(&cwd, &judge_stub(tmp.path())); + + skill_eval() + .current_dir(&cwd) + .args(["dispatch", "--judges", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + ]) + .assert() + .success(); + + let judge_tasks = read_json(&iteration_dir(&cwd).join("judge-tasks.json")); + let captures: Vec<_> = judge_tasks["tasks"] + .as_array() + .unwrap() + .iter() + .map(|task| { + let response = Path::new(task["response_path"].as_str().unwrap()); + response.with_extension("").join("codex-events.jsonl") + }) + .collect(); + for capture in &captures { + assert!(capture.is_file(), "missing judge transcript {capture:?}"); + } + let distinct: std::collections::BTreeSet<_> = captures.iter().collect(); + assert_eq!( + distinct.len(), + captures.len(), + "each judge task needs its own capture path" + ); +} + +/// A missing verdict is reported and exits nonzero, so a script can tell a +/// finished judge batch from one that still needs a rerun. +#[test] +fn dispatch_judges_exits_nonzero_while_a_verdict_is_missing() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), JUDGED_EVALS); + prepare_and_dispatch(tmp.path(), &skill_dir, &cwd); + + // A judge that answers nothing: the batch runs, but no verdict lands. + let script = tmp.path().join("silent-judge.sh"); + fs::write(&script, "#!/bin/sh\nexit 0\n").unwrap(); + stub_judge_template(&cwd, &format!("sh \"{}\"", script.to_string_lossy())); + + skill_eval() + .current_dir(&cwd) + .args(["dispatch", "--judges", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + ]) + .assert() + .failure() + .stdout(contains("0/")) + .stderr(contains("verdict")); +} + +/// Prepare an iteration, dispatch its eval tasks through a stub, and ingest, so +/// `judge-tasks.json` exists to dispatch judges from. +fn prepare_and_dispatch(tmp: &Path, skill_dir: &Path, cwd: &Path) { + skill_eval() + .current_dir(cwd) + .args(["run", "--skill-dir"]) + .arg(skill_dir) + .args([ + "--skill", + "mr-review", + "--mode", + "new-skill", + "--harness", + "codex", + "--no-guard", + ]) + .assert() + .success(); + + let script = tmp.join("eval-stub.sh"); + fs::write( + &script, + r#"#!/bin/sh +outputs=$1 +printf '%s\n' '{"type":"thread.started","thread_id":"session-1"}' > "$outputs/codex-events.jsonl" +printf '%s\n' '{"type":"item.completed","item":{"id":"m1","type":"agent_message","text":"I reviewed the MR."}}' >> "$outputs/codex-events.jsonl" +printf '%s\n' '{"type":"turn.completed","usage":{"input_tokens":2,"output_tokens":3}}' >> "$outputs/codex-events.jsonl" +"#, + ) + .unwrap(); + set_descriptor_template( + cwd, + "exec_template", + &format!("sh \"{}\" ", script.to_string_lossy()), + ); + + skill_eval() + .current_dir(cwd) + .args(["dispatch", "--skill-dir"]) + .arg(skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + ]) + .assert() + .success(); + + // Judge tasks are emitted here. The exit status is deliberately discarded: + // ingest exits nonzero while verdicts are outstanding, which is exactly the + // state this fixture wants to hand to the judge dispatcher. + let _ = skill_eval() + .current_dir(cwd) + .args(["ingest", "--skill-dir"]) + .arg(skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + ]) + .assert(); +} + +/// A judge stub: derives the verdict path from its capture directory the way a +/// real judge reads it out of its prompt, and writes a verdict there. +fn judge_stub(dir: &Path) -> String { + let script = dir.join("judge-stub.sh"); + fs::write( + &script, + r#"#!/bin/sh +outputs=$1 +printf '%s\n' '{"type":"thread.started","thread_id":"judge-1"}' > "$outputs/codex-events.jsonl" +printf '%s\n' '{"passed":true,"evidence":"stub verdict","confidence":0.8}' > "${outputs}.json" +"#, + ) + .unwrap(); + format!("sh \"{}\" ", script.to_string_lossy()) +} + +fn stub_judge_template(cwd: &Path, template: &str) { + set_descriptor_template(cwd, "exec_template", template); +} + +/// Swap one template in the frozen descriptor `dispatch.json` carries. +fn set_descriptor_template(cwd: &Path, field: &str, template: &str) { + let dispatch_path = iteration_dir(cwd).join("dispatch.json"); + let mut dispatch = read_json(&dispatch_path); + dispatch["harness_descriptor"]["dispatch"][field] = serde_json::json!(template); + fs::write( + &dispatch_path, + format!("{}\n", serde_json::to_string_pretty(&dispatch).unwrap()), + ) + .unwrap(); +} + +/// A judge runs from the iteration directory, outside every guarded task env, +/// so it must not inherit the eval dispatch's hook-trust bypass. The stub +/// records the guard fragment it was handed. +#[test] +fn a_judge_dispatch_carries_no_guard_arguments() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), JUDGED_EVALS); + prepare_and_dispatch(tmp.path(), &skill_dir, &cwd); + + let seen = tmp.path().join("guard-args.log"); + let script = tmp.path().join("guard-probe.sh"); + fs::write( + &script, + r#"#!/bin/sh +outputs=$1 +guard=$2 +printf '[%s]\n' "$guard" >> "$3" +printf '%s\n' '{"passed":true,"evidence":"stub verdict","confidence":0.8}' > "${outputs}.json" +"#, + ) + .unwrap(); + // `{guard_args}` renders as the descriptor's fragment when the guard is on + // and as the empty string when it is off. + set_descriptor_template( + &cwd, + "exec_template", + &format!( + "sh \"{}\" \"{{guard_args}}\" \"{}\"", + script.to_string_lossy(), + seen.to_string_lossy() + ), + ); + + skill_eval() + .current_dir(&cwd) + .args(["dispatch", "--judges", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + ]) + .assert() + .success(); + + let recorded = fs::read_to_string(&seen).unwrap(); + assert!(!recorded.trim().is_empty(), "the judge stub ran"); + for line in recorded.lines() { + assert_eq!( + line, "[]", + "a judge must get no guard arguments: {recorded}" + ); + } +} diff --git a/tests/run/main.rs b/tests/run/main.rs index d611f94..0f8ed08 100644 --- a/tests/run/main.rs +++ b/tests/run/main.rs @@ -24,6 +24,7 @@ mod diff_scope; mod env_layout; mod git_isolation; mod grouping; +mod judges; mod lifecycle; mod opencode; mod opencode_permission_denials; diff --git a/tests/run/runbook.rs b/tests/run/runbook.rs index 230f371..3395334 100644 --- a/tests/run/runbook.rs +++ b/tests/run/runbook.rs @@ -5,6 +5,48 @@ use crate::helpers::*; use predicates::prelude::PredicateBooleanExt; use predicates::str::contains; +/// One dispatch command, whatever the plan holds — a mixed plan of scripted and +/// one-shot evals included. The runner drives every task, so the runbook has no +/// per-plan-shape branch to render. +#[test] +fn the_runbook_names_exactly_one_task_dispatch_command() { + let tmp = tempfile::TempDir::new().unwrap(); + let evals = r#"{ + "skill_name": "mr-review", + "evals": [ + {"id": "one-shot", "prompt": "Fix it.", "expected_output": "fixed"}, + {"id": "scripted", "prompt": "Fix it.", "expected_output": "asks first", + "turns": [{"prompt": "Use UTC.", "deliver_when": "always"}]} + ] + }"#; + let (skill_dir, cwd) = setup(tmp.path(), evals); + skill_eval() + .current_dir(&cwd) + .args(["run", "--skill-dir"]) + .arg(&skill_dir) + .args(["--skill", "mr-review", "--harness", "codex", "--dry-run"]) + .assert() + .success(); + + let book = read_str(&iteration_dir(&cwd).join("RUNBOOK.md")); + assert_eq!( + book.matches("eval-magic dispatch --").count(), + 2, + "one command for the eval tasks and one for the judges: {book}" + ); + assert!( + book.contains("eval-magic dispatch --judges"), + "judges dispatch through the runner too: {book}" + ); + for recipe_tool in ["xargs", "jq ", "tr -d"] { + assert!( + !book.contains(recipe_tool), + "no pasted shell pipeline survives ({recipe_tool}): {book}" + ); + } + assert!(!book.contains("{{"), "no unsubstituted tokens: {book}"); +} + #[test] fn run_writes_headless_runbook_for_codex() { let tmp = tempfile::TempDir::new().unwrap(); @@ -30,8 +72,8 @@ fn run_writes_headless_runbook_for_codex() { "frames the run for a human at a terminal: {book}" ); assert!( - book.contains("codex --ask-for-approval never exec"), - "carries the Codex CLI dispatch recipe: {book}" + book.contains("eval-magic dispatch --skill-dir"), + "carries the runner-driven dispatch command: {book}" ); assert!( book.contains("--harness codex"), @@ -59,16 +101,17 @@ fn run_writes_headless_runbook_for_claude() { .success(); let book = read_str(&iteration_dir(&cwd).join("RUNBOOK.md")); - // A Claude Code run uses the shared human-followed template carrying the - // `claude -p` recipe. Each task dispatches from its own per-(group, condition) - // env, so the runbook lives in the iteration dir, above those envs. + // Every harness now uses the same shared template: the runner drives the + // dispatch, so the command differs only by its `--harness` selector. Each + // task still runs in its own per-(group, condition) env, so the runbook + // lives in the iteration dir, above those envs. assert!( book.contains("human driving"), "frames the run for a human at a terminal: {book}" ); assert!( - book.contains("claude -p"), - "carries the claude -p dispatch recipe: {book}" + book.contains("--harness claude-code"), + "pipeline commands carry --harness claude-code: {book}" ); assert!( !book.contains("switch-condition"), @@ -82,10 +125,11 @@ fn run_writes_headless_runbook_for_claude() { let requirement = book .find("Git Bash") .expect("the runbook states the POSIX shell requirement"); - assert!(book.contains("jq"), "the requirement names jq too: {book}"); assert!(book.contains("WSL"), "{book}"); + // Anchored at a line start: the requirement prose names the command too, + // and what this pins is the order of the *pasteable* line against it. assert!( - requirement < book.find("claude -p").unwrap(), + requirement < book.find("\neval-magic dispatch --skill-dir").unwrap(), "the requirement precedes the first pasteable command: {book}" ); } @@ -133,8 +177,8 @@ fn run_writes_headless_runbook_for_opencode() { let book = read_str(&iteration_dir(&cwd).join("RUNBOOK.md")); assert!( - book.contains("opencode run --dir"), - "carries the opencode CLI dispatch recipe: {book}" + book.contains("--harness opencode"), + "pipeline commands carry --harness opencode: {book}" ); assert!( book.contains("--harness opencode"), @@ -151,10 +195,10 @@ fn run_writes_headless_runbook_for_opencode() { !manifest.contains("{{"), "no unsubstituted tokens: {manifest}" ); - // The manifest carries the same POSIX recipes, so it carries the same - // requirement (issue #248 names both artifacts). + // Dispatch shells out to POSIX command lines, so the manifest states the + // same requirement the runbook does (issue #248 names both artifacts). assert!( - manifest.contains("Git Bash") && manifest.contains("jq"), - "the manifest states the POSIX tooling requirement: {manifest}" + manifest.contains("Git Bash"), + "the manifest states the POSIX shell requirement: {manifest}" ); }