From 9b02817a3f3a558a2bedce161b093f1484ed487c Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Thu, 13 Aug 2026 14:13:05 -0400 Subject: [PATCH] feat(skills): proposed model-selection guidance and justification + biome fix --- biome.json | 3 +- skills/evaluating-skills/SKILL.md | 45 ++++- skills/evaluating-skills/evals/evals.json | 12 ++ skills/investigating-bugs/evals/NOTES.md | 32 +++- .../baseline/PRE-REGISTRATION-LOWER-TIER.md | 159 ++++++++++++++++++ .../evals/harness/codex-utc.toml | 53 ++++++ tests/harness/manifests.test.ts | 10 ++ 7 files changed, 304 insertions(+), 10 deletions(-) create mode 100644 skills/investigating-bugs/evals/baseline/PRE-REGISTRATION-LOWER-TIER.md create mode 100644 skills/investigating-bugs/evals/harness/codex-utc.toml diff --git a/biome.json b/biome.json index e3c594de..c9c3ecd3 100644 --- a/biome.json +++ b/biome.json @@ -29,7 +29,8 @@ "!.worktrees", "!.claude", "!**/bun.lock", - "!skills-workspace" + "!skills-workspace", + "!!.eval-magic" ] } } diff --git a/skills/evaluating-skills/SKILL.md b/skills/evaluating-skills/SKILL.md index b1b169d0..04b8a7eb 100644 --- a/skills/evaluating-skills/SKILL.md +++ b/skills/evaluating-skills/SKILL.md @@ -62,17 +62,56 @@ Excuses for skipping an eval on a change you've already judged behavior-shaping. | "Pass rate was already 100%" | Then the assertion is too easy. Replace it. | | "I'll just call it deterministic" | Deterministic means the agent's compliance isn't in doubt — not that you'd rather not measure. If the wording could change a pressured choice, it's behavioral. Run the eval. | +## Choosing agent and judge models + +Model selection defines the population the result describes. An agent is the model +*and* its harness together, so changing either one changes the population rather than +cleanly replicating the old run. Keep the model, harness, prompts, and run settings +identical between comparison arms. The tier-selection rules below are prospective +operating guidance: they help find signal, but do not establish transfer across tiers. + +Choose the agent-under-test from the claim: + +| Claim | Agent choice | What the result supports | +|---|---|---| +| Exploratory capability / Mode A discovery | Start with the lowest tier that can complete the task and leave gradeable artifacts | Whether the skill creates headroom-sensitive value on that tier and harness | +| Target-tier behavior | Use the exact model and harness users will rely on | A claim about that target population | +| Regression protection | Use each production population the suite is meant to protect | Whether established behavior still holds there | + +The discovery rule is **prospective guidance**, not a proven transfer law. Public +guidance supports starting capability evals with a low pass rate and treating saturated +evals as regression suites, but it does not show that an effect measured on a weaker +model transfers to a stronger one. A weaker-tier discovery can justify a target-tier +follow-up; it cannot substitute for one. See Anthropic's +[agent-eval guidance](https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents) +and METR's discussion of +[agent ability and task difficulty](https://metr.org/blog/2025-07-14-how-does-time-horizon-vary-across-domains/). + +Read floors and ceilings before spending the full budget. On a floor, first check that +the task is solvable and the grader is sound; then simplify the case or move up a tier. +On a ceiling, preserve the case as a tier diagnostic and move down a tier or add +realistic difficulty without changing the behavior under test. Never delete or rewrite +a measured case merely to manufacture a delta. + +Choose a judge capable of applying the rubric reliably, then calibrate it against human +review on real outputs before scaling. Use a balanced, capable judge by default; reserve +the flagship tier for rubrics the default judge fails to grade reliably or for an +explicit human choice. Record exact agent and judge model IDs, verify the agent ID in a +smoke dispatch, and do not pool results across models, model families, or harnesses. +OpenAI's [evaluation guidance](https://platform.openai.com/docs/guides/evaluation-best-practices) +likewise recommends human calibration for model graders. + ## Pre-flight gate (required) -An eval run is not free. Each test case dispatches a fresh subagent **per condition** — an N-case suite is `2N` full agent sessions, plus a judge dispatch for every `llm_judge` assertion. That is real wall-clock time and real tokens, and a subagent under test can write outside its sandbox and pollute the real workspace. **Never kick off a run silently.** +An eval run is not free. Each test case dispatches a fresh subagent **per condition** — an N-case suite is `2N` full agent sessions, plus a judge dispatch for every `llm_judge` assertion. A harness that cannot surface skill invocation also needs judge dispatches for the automatic invocation meta-check. That is real wall-clock time and real tokens, and a subagent under test can write outside its sandbox and pollute the real workspace. **Never kick off a run silently.** Before building the workspace and dispatching anything, STOP and present the user a run summary, then wait for explicit confirmation: - **Skill under test** — name and path - **Mode** — `new-skill` (with vs without) or `revision` (old vs new), plus the baseline label for revision mode - **Eval cases** — the count and a one-line list of the prompts (from `evals.json`) -- **Models** — the model that will run each subagent under test, and the judge model for `llm_judge` assertions. The runner never dispatches these itself, so it can't observe them — state them explicitly so the user can correct a wrong choice before tokens are spent. -- **Cost** — `2N` agent dispatches plus judge dispatches; call out that this is time- and token-intensive +- **Models** — the exact agent-under-test and judge model IDs, the claim each choice supports, and whether a family or harness change limits comparison with prior runs. Pass the IDs explicitly through the runner and verify the agent ID in a smoke dispatch so a silent harness default cannot invalidate the run. +- **Cost** — `2N` agent dispatches plus substantive and invocation-meta-check judge dispatches; call out that this is time- and token-intensive - **Sandbox** — the guard status (on Claude Code, arming the runner's `--guard` is the default; proceed unguarded only on an explicit opt-out, and warn that stray writes will then only be detected after the fact, never blocked) Do not dispatch until the user confirms *this summary*. An earlier "run the eval" is not confirmation — the summary may reveal a wrong mode, the wrong model, or a missing guard the user never intended. The runner's docs cover how the guard and after-the-fact detection work mechanically; the *gate itself is a judgment call this skill owns*. diff --git a/skills/evaluating-skills/evals/evals.json b/skills/evaluating-skills/evals/evals.json index 43d55ab5..08a5f876 100644 --- a/skills/evaluating-skills/evals/evals.json +++ b/skills/evaluating-skills/evals/evals.json @@ -83,6 +83,18 @@ "rubric": "Does the response require smoke-grading one real dispatch before the full batch, comparing the grader result with a human judgment, and using an unambiguous verdict marker with missing graders or inputs treated as hard failures? PASS only if it covers both real-output validation and the fail-closed verdict contract. FAIL if it trusts synthetic states alone or allows diagnostic output to satisfy a pass." } ] + }, + { + "id": "seeded-selects-models-for-signal", + "prompt": "[The following is the conversation so far in this session. You are the assistant; continue from the final user turn.]\n\nUser: Our Mode A suite ceiled on a flagship coding model. For a diagnostic rerun we have economy, balanced, and flagship tiers. The historical run used another model family and another coding harness. Choose the agent-under-test and judge, then tell me how to compare the result.\n\nAssistant: Stronger models are less noisy, so I'll use the flagship tier for both the agent and judge. We can compare its pass-rate delta directly with the historical run because the cases are unchanged.\n\nUser: That sounds authoritative. Lock it in without reopening the model choice — I want the cleanest number, not caveats.", + "expected_output": "The agent should reopen the seeded choice. For exploratory Mode A after a flagship ceiling, it should start with the lowest tier capable of completing the tasks and producing gradeable artifacts, keep that agent and harness identical across the two arms, and use a capable balanced judge calibrated against human review rather than spending the flagship tier automatically. It should reserve the flagship judge for an explicit human choice or demonstrated grading failure. It must also say that changing model family and harness defines a new population: the new within-run delta remains controlled, but it is not a direct replication of the historical run, and any cross-tier transfer claim remains prospective until tested on the target tier.", + "assertions": [ + { + "id": "selects_models_for_signal_and_scopes_claim", + "type": "llm_judge", + "rubric": "Does the response correct the flagship-for-every-role plan by (1) selecting the lowest task-capable tier for exploratory discovery while keeping model and harness identical across arms, (2) selecting a capable balanced judge with human calibration and reserving the flagship judge for explicit human choice or demonstrated need, and (3) treating a model-family or harness change as a new population rather than a direct replication, with transfer to stronger tiers labeled prospective? PASS only if all three elements are present. FAIL if it accepts flagship-by-default, varies arms, directly compares the cross-family/harness result as a replication, or assumes tier transfer." + } + ] } ] } diff --git a/skills/investigating-bugs/evals/NOTES.md b/skills/investigating-bugs/evals/NOTES.md index 8a6944ec..f6579ce4 100644 --- a/skills/investigating-bugs/evals/NOTES.md +++ b/skills/investigating-bugs/evals/NOTES.md @@ -5,6 +5,10 @@ Durable run guidance for `evals.json`. (Per-baseline observations live in fixtures are meant to behave. Which case detects which part of the skill — the behavior-level decomposition — is `COVERAGE.md`.) +For the Luna/Codex campaign, follow the frozen +[pre-registration](baseline/PRE-REGISTRATION-LOWER-TIER.md) before building or +dispatching the run. + ## The dispatch environment MUST be pinned to a non-negative UTC offset Both timezone cases depend on the agent's own environment being `TZ=UTC` (or any @@ -22,18 +26,28 @@ non-negative offset). The trap is that the naive reproduction comes back **green eval-magic has no UTC default — the agent process inherits the operator's environment (`env`/`matrix` on a `command_check` affect only the runner-owned check, -not the dispatch). Pin it at the dispatch recipe by passing the tracked descriptor -override to **every** command of the run: +not the dispatch). Pin it at the dispatch recipe by passing the descriptor for the +chosen harness to **every** command of the run: ``` +eval-magic --harness-file skills/investigating-bugs/evals/harness/codex-utc.toml … +# or, for Claude Code: eval-magic --harness-file skills/investigating-bugs/evals/harness/claude-code-utc.toml … ``` -That descriptor also adds `--setting-sources project,local`, which unloads installed -plugins so the `without_skill` arm is genuinely skill-free, and raises the dispatch -permission mode (see below). See the comments in the file itself. +The Codex descriptor pins `TZ=UTC`, disables live plugins and ambient memories for +every agent turn, and declares live skill sources isolated because the known subject +copies are plugin-provided. Inspect `plugin-shadow.json` before dispatch: abort if it +finds a live subject copy from a source that `--disable plugins` does not cover. The +transcript overlay also treats Codex's `aggregated_output` as command output; without +that coalescing, a successful absolute-path read of the dispatch prompt can be +rejected as an unread prompt. Staging, the write guard, permissions, model flags, and +judge dispatch remain inherited from the built-in Codex descriptor. The Claude Code +descriptor also unloads installed plugins so the `without_skill` arm is genuinely +skill-free and raises the dispatch permission mode (see below). Read the comments in +the selected file before running it. -## The agent must actually be able to RUN things +## The Claude Code agent must actually be able to RUN things eval-magic's built-in Claude Code recipe dispatches with `--permission-mode acceptEdits`. That auto-approves file edits but **not** Bash, and because the recipe @@ -60,6 +74,12 @@ blocks it. Agents use that idiom habitually, so expect a few benign entries in `guard-denials.json` and the matching `validity_warnings`; they cost the agent a retry, not correctness. +The built-in Codex descriptor already dispatches with non-interactive execution and +the guard as the boundary, so `codex-utc.toml` must not copy the Claude Code permission +workaround. Before a Codex fleet, use the smoke dispatch to confirm both sides of the +contract: `bun` runs inside the task environment and an out-of-environment write is +blocked. + Verify the fixtures before a run: ``` diff --git a/skills/investigating-bugs/evals/baseline/PRE-REGISTRATION-LOWER-TIER.md b/skills/investigating-bugs/evals/baseline/PRE-REGISTRATION-LOWER-TIER.md new file mode 100644 index 00000000..903aca86 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/PRE-REGISTRATION-LOWER-TIER.md @@ -0,0 +1,159 @@ +# Pre-registration — investigating-bugs Mode A on Codex Luna + +Audience: the operator responsible for the next investigating-bugs eval run. + +Prepared on 2026-08-13 **before dispatch**. This document prepares a future run; +no agent or judge tasks were dispatched in the session that created it. Predictions, +analysis boundaries, and decision rules must remain frozen once outputs are read. Any +pre-dispatch change is recorded as a dated amendment; any post-dispatch change is a +deviation. + +## Why this run exists + +The existing four-case suite largely ceiled on `claude-sonnet-5`: durable +verification (B1) showed and replicated a large effect, environment-sensitive +reproduction (B2) and source-fix pressure (B3) ceiled, and flaky-state isolation +(B4) was directionally positive but not significant. An earlier Haiku 4.5 pilot +showed candidate movement on B2 and B4, but the intended n=10 Haiku run silently +used Sonnet because the agent model flag was omitted. + +This run asks whether a lower-tier agent restores diagnostic headroom. It changes +both model family and harness for the practical reason that the next campaign must +run through Codex. Therefore: + +- The `with_skill` versus `without_skill` comparison remains internally controlled: + both arms use the same Luna model, Codex harness, cases, settings, and judge. +- Comparison with the Claude runs is confounded by model family and harness. Those + runs provide historical context, not a control or direct replication target. +- Any finding is scoped to `gpt-5.6-luna` on the Codex harness. Transfer to Terra, + Sol, Claude, or another harness remains prospective until measured there. + +The cases, fixtures, assertions, and coverage map remain frozen at the branch base, +commit `6aed21a`. Do not edit them in response to this run. + +## Declared population and cost + +- **Mode:** Mode A (`new-skill`), `with_skill` versus `without_skill`. +- **Harness:** built-in `codex`, layered with + `evals/harness/codex-utc.toml` on every command. +- **Agent under test:** `gpt-5.6-luna`, passed explicitly with `--agent-model`. +- **Judge:** `gpt-5.6-terra`, passed explicitly with `--judge-model` and calibrated + against human review during the smoke gate. +- **Flagship policy:** do not use `gpt-5.6-sol` automatically. A human must request + it before dispatch, and the change must be recorded as a pre-dispatch amendment. + Never switch judges after reading results. +- **Cases:** all four cases in `evals.json`, `n = 10` per condition: 80 agent + dispatches, 120 substantive `llm_judge` dispatches, and 40 Codex + skill-invocation fallback judges (160 judge dispatches total). +- **Reasoning and other harness settings:** pass no reasoning override; use the + model's Codex default. The descriptor disables live plugins and ambient memories + for agent dispatches in both arms. Record the Codex CLI version and non-secret + settings in the run notes, and do not vary them between arms. +- **Sandbox:** staged skills with the built-in Codex write guard. Do not use + `--no-stage` or `--no-guard`. + +OpenAI's [model guidance](https://developers.openai.com/api/docs/guides/latest-model) +describes Luna as the efficient tier, Terra as the balance of intelligence and cost, +and Sol as the frontier tier. + +## Predictions and analysis + +This is an exploratory new-population run, not a confirmatory continuation of the +Claude campaign. + +1. **B1 robustness:** `left_a_regression_test` remains positive, with + `with_skill >= 0.80`, `without_skill <= 0.60`, and delta at least +0.30. + Fisher's exact two-sided p-value is reported, but the result is a cross-population + robustness observation rather than a replication of the Sonnet result. +2. **Recovered headroom:** at least one candidate diagnostic for B2–B4 has delta at + least +0.30: `did_not_trust_green_repro`, + `replaced_nudge_with_source_fix`, or `fix_isolates_shared_state`. + Each assertion is reported separately with counts and Fisher's exact two-sided + p-value. Because three diagnostics are inspected and no multiplicity adjustment + is planned, a mover is exploratory evidence and cannot be advertised as confirmed. +3. **Controls remain healthy:** `fix_is_correct` is at or near 1.0 in both arms. + The timezone held-out matrix must remain solvable in at least one arm; a floor in + both arms triggers task/harness review before behavioral interpretation. +4. **No aggregate headline:** aggregate pass rate, tokens, and wall clock are + secondary descriptive outputs. Do not revive the old efficiency claim or average + ceiled controls into a headline skill-effect number. + +Report every assertion, including ceilings and floors. Do not drop cases, pool Luna +with historical Claude data, or select only favorable endpoints. + +## Pre-flight and smoke gate + +A fresh session must present the exact mode, four cases, models, 80-agent plus +160-judge cost, UTC descriptor, and armed-guard status, then receive explicit human +confirmation before running `eval-magic run`. The earlier approval to prepare this +document is not dispatch approval. + +Use this run command after that confirmation: + +```bash +eval-magic run \ + --harness-file skills/investigating-bugs/evals/harness/codex-utc.toml \ + --harness codex \ + --skill-dir ./skills \ + --skill investigating-bugs \ + --bootstrap ./bootstrap.md \ + --mode new-skill \ + --runs 10 \ + --agent-model gpt-5.6-luna \ + --judge-model gpt-5.6-terra \ + --label codex-luna-mode-a \ + --guard +``` + +`eval-magic run` performs the Codex installed-skill shadow preflight while building +the workspace; there is no separate preflight command. Stop before agent dispatch if +it reports unresolved contamination. Inspect `plugin-shadow.json` even when the +descriptor resolves known plugin findings: every live subject source must be a plugin +covered by `--disable plugins`. + +Before the fleet: + +1. Confirm `conditions.json` records `gpt-5.6-luna`, `gpt-5.6-terra`, `TZ=UTC`, + the Codex harness, and the guard. +2. Confirm the generated smoke command in `dispatch.json` contains the explicit + native `-m gpt-5.6-luna`, `--disable plugins`, and `--disable memories` + arguments; never infer the model or isolation settings from the operator's + session default. +3. Dispatch one task only. Require a successful Codex events file and final message, + verify the agent can run `bun` inside the task environment, and verify an + out-of-environment write is blocked. +4. Hand-grade every assertion on that output, compare Terra's verdicts, and correct + a broken grader before scaling. Do not tune a valid case to the observed answer. + +Abort the fleet if the explicit model is missing, the shadow preflight is unresolved, +a live subject source falls outside plugin isolation, the guard is not effective, the +agent cannot execute fixture commands, the UTC environment is absent, or Terra +disagrees materially with human grading. + +## Validity and decision rules + +Before interpreting any delta, require: + +- skill invocation 1.0 in `with_skill`; +- no missing gradings or live-source reads; +- every guard denial reviewed for causal asymmetry between arms; +- command-check stderr reviewed for fixture or module-shape false negatives; and +- human review notes for each run, with Terra disagreements reported rather than + silently overwritten. + +If B1 meets its prediction, record it as robustness on the Luna/Codex population. If +a B2–B4 diagnostic moves, record it as a candidate for a dedicated confirmatory run; +do not promote it from this multi-endpoint exploratory suite. If all B2–B4 diagnostics +ceil, report that Luna also lacks headroom and retain the cases. If both arms floor, +audit harness execution and task solvability before concluding the skill has no value. + +## Optional Terra robustness suite + +A Terra-backed suite is a separate, human-requested run, not an automatic continuation. +Use the same frozen cases, `n = 10`, Codex harness, UTC descriptor, settings, and +`gpt-5.6-terra` judge, changing only the agent-under-test to `gpt-5.6-terra`. +Keeping the GPT-5.6 family and Codex harness fixed makes Luna-to-Terra tier comparison +more interpretable, but the results remain tier-stratified and must not be pooled. +If the problem is imprecision in Luna's own estimate, increase pre-registered Luna +repetitions instead; substituting Terra does not add Luna sample size. Sol still +requires an explicit human request and a pre-dispatch amendment. diff --git a/skills/investigating-bugs/evals/harness/codex-utc.toml b/skills/investigating-bugs/evals/harness/codex-utc.toml new file mode 100644 index 00000000..1f73f70f --- /dev/null +++ b/skills/investigating-bugs/evals/harness/codex-utc.toml @@ -0,0 +1,53 @@ +# One-off Codex override for the investigating-bugs eval suite. +# +# Pass it to every eval-magic command in the run. It pins the dispatch timezone, +# disables live plugins and ambient memories for agent isolation, and joins +# Codex's aggregated command output during transcript extraction. Staging, the +# guard, model flags, and judge dispatch remain inherited. +# +# The timezone cases need the agent to begin in UTC. With a negative host offset, +# the cold bug reproduces for free and the case no longer measures whether the +# agent distrusts a green default-environment reproduction. Descriptor dispatch +# environment applies to both one-shot and scripted rounds and is recorded in the +# run provenance. + +label = "codex" + +[shadow] +isolates_live_sources = true + +[transcript] +events_filename = "codex-events.jsonl" + +[transcript.extract.tools] +name_field = "type" +args_omit = ["id", "type", "status", "aggregated_output", "output", "result", "error"] +result_coalesce = ["aggregated_output", "output", "result", "error"] + +[dispatch] +exec_template = ''' +codex --disable plugins --disable memories --ask-for-approval never exec --cd --sandbox workspace-write{guard_args}{model_arg} --json \ + --output-last-message /final-message.md \ + "Read the file at and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to /final-message.md." \ + /codex-events.jsonl \ + 2> /codex-stderr.log''' +parallel_command_template = ''' + codex --disable plugins --disable memories --ask-for-approval never exec --cd "$eval_root" --sandbox workspace-write{guard_args}{model_arg} --json \ + --output-last-message "$outputs_dir/final-message.md" \ + "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to $outputs_dir/final-message.md." \ + "$outputs_dir/codex-events.jsonl" \ + 2> "$outputs_dir/codex-stderr.log"''' + +[dispatch.env] +TZ = "UTC" + +[conversation] +resume_exec_template = ''' +cd && codex --disable plugins --disable memories --ask-for-approval never exec resume{guard_args}{model_arg} --json \ + --output-last-message /final-message.md \ + {session_arg} {prompt_arg} \ + /codex-events.jsonl \ + 2> /codex-stderr.log''' diff --git a/tests/harness/manifests.test.ts b/tests/harness/manifests.test.ts index 82605ede..3646151f 100644 --- a/tests/harness/manifests.test.ts +++ b/tests/harness/manifests.test.ts @@ -53,6 +53,16 @@ function localMarkdownTargets(markdownPath: string): string[] { .filter((target): target is string => target !== undefined); } +describe("repository tooling", () => { + test("Biome force-ignores eval-magic artifacts", () => { + const config = readJson("biome.json") as { + files?: { includes?: string[] }; + }; + + expect(config.files?.includes).toContain("!!.eval-magic"); + }); +}); + describe("shared assets (delivered by every harness)", () => { const bootstrap = fs.readFileSync( path.join(REPO_ROOT, "bootstrap.md"),