diff --git a/docs/guides/guard.md b/docs/guides/guard.md index d24df9d..85aa571 100644 --- a/docs/guides/guard.md +++ b/docs/guides/guard.md @@ -139,6 +139,12 @@ Explicit commands and expanded profile commands are deduplicated in the effectiv ## Audit the effective policy +The run preflight resolves harness support, staging, `--guard`, and `--no-guard` into one effective +state. `run` records that state as `guard_armed` in `conditions.json`, mirrors it as `guard` in +`dispatch.json`, and carries it into `benchmark.json`. The `BASELINE.md` file written by +`promote-baseline` renders the value as `true` or `false`; a historical conditions file without the +field renders as `unknown`, not `false`. + Each task in `dispatch.json` records its fully expanded `guard_policy`. The armed marker records the same policy as `guardPolicy`, so the live hook and the campaign plan cannot resolve defaults differently. `detect-stray-writes` reads the frozen task policy from `dispatch.json` and applies the diff --git a/schema/benchmark.schema.json b/schema/benchmark.schema.json index 18230da..7fbba62 100644 --- a/schema/benchmark.schema.json +++ b/schema/benchmark.schema.json @@ -62,6 +62,10 @@ "$ref": "#/definitions/skillSource", "description": "The skill under test, echoed from conditions.json. Absent for a benchmark written before skills were sourced." }, + "guard_armed": { + "type": "boolean", + "description": "Whether the compared runs used the eval write guard, echoed from conditions.json. Absent when historical conditions do not record the effective state." + }, "diff_scope": { "type": "object", "description": "Raw final-environment diff metrics per condition, ordered by eval id and then run index. Omitted for iterations created before diff-scope capture.", diff --git a/src/cli/args.rs b/src/cli/args.rs index ebb5782..94c0c5e 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -9,6 +9,7 @@ use std::path::PathBuf; use clap::{Args, Parser, Subcommand}; pub(crate) use super::init_args::InitArgs; +pub(crate) use super::run_args::RunArgs; /// Run skill evals — measure whether an agent skill actually shifts behavior. /// @@ -335,218 +336,6 @@ pub struct PromoteBaselineArgs { pub responder_model: Option, } -/// `run` adds the build-time flags (mode/baseline selection, staging toggles, -/// guard, plan-mode, bootstrap) on top of the common set. -#[derive(Debug, Args)] -pub struct RunArgs { - #[command(flatten)] - pub common: CommonArgs, - /// Baseline snapshot label (defaults to `baseline` in `--mode revision`). - /// - /// The snapshot label to use as the `old_skill` arm in revision mode (see - /// `snapshot`). - #[arg(long)] - pub baseline: Option, - /// SessionStart-equivalent bootstrap file inlined into each dispatch. - /// - /// A Markdown file prepended verbatim to every dispatch prompt inside a - /// `` block — product-specific framing a SessionStart - /// hook would inject. It does NOT enumerate skills (the auto-built - /// available-skills block is the single source of the skill list). Omit it and - /// dispatches carry only that inventory. - #[arg(long)] - pub bootstrap: Option, - /// Build the workspace but skip guard install and stop before next steps. - #[arg(long)] - pub dry_run: bool, - /// Inline each condition's SKILL.md into the dispatch prompt instead of - /// staging it under the harness skills dir. - /// - /// For harnesses without project-local skill discovery. Forces the LLM-judge - /// meta-check tier and inlines only SKILL.md (not sibling skills or sibling - /// asset files); use the staged (default) path when the measured behavior - /// depends on sibling files. The isolated env (`env/`) is still built either - /// way — `--no-stage` only skips populating the harness skills dir. Also - /// disables the write guard (auto-armed or explicit) — it requires staging — - /// so no-stage runs are unguarded and rely on `detect-stray-writes` after - /// the fact. - #[arg(long)] - pub no_stage: bool, - /// Arm the write guard explicitly (it auto-arms when the harness supports it). - /// - /// The write guard arms automatically on guard-capable built-in harnesses - /// whenever staging is active, so this flag only makes the request explicit: - /// where auto-arm quietly stays off (no declared guard, or `--no-stage`), an - /// explicit `--guard` warns, and a harness defined only by user-supplied - /// descriptors rejects it in preflight (rerun without it; - /// `detect-stray-writes` audits after the fact). Opt out with `--no-guard`. - /// - /// The guard is a harness-native `PreToolUse` hook that *blocks* subagent - /// writes/installs outside the isolated run env (the agent-under-test's cwd) - /// while dispatches run. The task env is its sole allowed write root; host temp - /// directories are out of bounds. Dispatch prompts name `/tmp` as - /// the task-local scratch directory (create it when needed); eval-magic does - /// not rewrite `TMPDIR`, `TMP`, or `TEMP`. Because the framework designates - /// that directory, what an agent puts there is excluded from diff scope and - /// from the project's own ignore files — so a `diff_scope` budget covers the - /// change, not the scratch work, and judges never read scratch notes as the - /// deliverable. - /// Because the harness already cwd-bounds the agent's direct file tools to the - /// env, the guard's main remaining value is blocking Bash-subprocess escapes the - /// cwd boundary doesn't cover and acting as a backstop when the isolated session - /// runs with relaxed permissions. Recognized development mutations require an - /// allowance from the eval's `guard` configuration. `allow_commands` grants - /// literal shell-token prefixes; `allow_tools` grants every invocation of an - /// executable basename. A per-eval block replaces the config-level default. With - /// no explicit block, eval-magic composes packaged profiles detected from the - /// staged task tree. See `eval-magic docs guard` for configuration, matching, - /// packaged profiles, and examples. - /// - /// Command allowances never bypass containment checks. Known destination options - /// with dynamic, missing, or outside values are blocked, as are global/user - /// install modes that do not have a supported in-env destination. Recognized - /// destinations include npm `--prefix`, pnpm `-C`/`--dir`, Yarn/Bun `--cwd`, - /// pip `--target`/`--prefix`/`--root`/`--src`, and Cargo `-C`/`--target-dir` - /// plus its target-dir environment variables. Generic shell commands are not a - /// complete parser: for example, a bare `touch /outside` remains an - /// after-the-fact `detect-stray-writes` concern. - /// - /// Local Git operations such as status, diff, add, commit, and branching are - /// allowed inside the task repository. Repository-routing escapes and remote Git - /// operations are blocked; `--no-guard` opts out of those blocks, though task - /// repositories still begin with no remotes. Literal relative redirect and `tee` - /// targets resolve from the tool invocation cwd; dynamic, malformed, or outside - /// targets are blocked. Every denial appends privacy-safe metadata - /// (never the full command or patch) to the task's - /// `.eval-magic-outputs/guard-denials.jsonl`; `ingest` joins those logs into - /// `guard-denials.json`, and `aggregate` emits one validity warning per - /// affected task. Re-arming truncates stale raw records; disarming the guard - /// preserves the current log. The marker auto-expires after 6h and is torn - /// down at the next run; while armed the - /// hook fires on your own tool calls too. If it remains armed after `finalize`, - /// `finalize` reminds you to run `teardown` before editing source (which disarms - /// the cwd guard and every per-`(group, condition)` Cli env's guard). Requires - /// staging — with `--no-stage` the guard stays off and the run is unguarded. - /// Codex eval-agent dispatches must include - /// `--dangerously-bypass-hook-trust` so the vetted project-local eval hook - /// runs; judge dispatches omit it because judges run outside guarded task envs. - /// Unguarded, stray writes are only *detected* after the fact by - /// `detect-stray-writes`, never blocked. - /// Under Claude Code the `PreToolUse` hook is staged in each env's - /// `.claude/settings.local.json`, and each `claude -p` dispatch loads it from - /// that cwd (`cd `), enforcing the eval boundary (the recipe never - /// passes `--bare`). - /// When invoking this from inside Codex, staging writes `.agents/skills` and - /// guarded runs also write `.codex/hooks.json`; Codex protects those paths in - /// its default workspace-write sandbox, so approval/escalation may be needed. - #[arg(long)] - pub guard: bool, - /// Opt out of the write guard for this run. - /// - /// The guard arms automatically on harnesses that declare one (see - /// `--guard`). Pass this to run unguarded — e.g. when the skill under test - /// legitimately writes outside the isolated run env — and to silence the - /// unguarded-harness preflight warning. Unguarded, out-of-bounds writes are - /// only *detected* after the fact by `detect-stray-writes` (folded into - /// `ingest`), never blocked. - /// - /// Dispatches deliberately run with relaxed harness permissions so the - /// agent-under-test can actually execute commands, which makes the guard - /// the only enforcement boundary. Opting out therefore leaves the dispatch - /// with no boundary at all, not merely a weaker one. - #[arg(long, conflicts_with = "guard")] - pub no_guard: bool, - /// Stage the skill-under-test under this verbatim name instead of the - /// conspicuous `slow-powers-eval-…` slug. - /// - /// For name-confound experiments. A scalar `skill_name` and one staging - /// condition are required; a multi-skill treatment is rejected because one - /// override cannot name every member. Refuses to clobber an existing dir and - /// registers the staged name for next-run cleanup. - #[arg(long)] - pub stage_name: Option, - /// Inject the shared plan-mode profile as an operating-context layer. - /// - /// Injects the shared, harness-agnostic plan-mode procedure - /// (`profiles/shared/plan-mode.md`) as a `` in every - /// dispatch, identical across arms and harnesses. Opt-in, for - /// plan-mode-relevant skills. It is text the subagent reads, not a real - /// injected mode. - #[arg(long)] - pub plan_mode: bool, - /// Runs per condition cell, for variance reduction (default: 1). - /// - /// Dispatches every eval N times per condition, so an iteration needs - /// `evals × 2 conditions × N` dispatches. Each run gets its own - /// `run-/` directory under the condition (own `inputs/`, `outputs/`, - /// `run.json`, `timing.json`, `grading.json`) and a unique - /// `agent_description` carrying an `r` segment. With N=1 the layout is - /// unchanged (artifacts sit directly in the condition directory). The - /// benchmark's per-condition `mean`/`stddev`/`n` then reflect all runs. A - /// per-eval `runs` field in evals.json overrides this flag for that eval. - /// Before staging, a fully binary run summary prints the minimum attainable - /// two-sided Fisher exact p-value for each effective run count, assuming - /// perfect separation between the two conditions. A run with sampled LLM - /// assertions instead identifies vote proportion and pass^k as non-binary - /// endpoints. eval-magic does not calculate observed p-values or apply a - /// significance threshold. - #[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u32).range(1..))] - pub runs: u32, - /// Agent-under-test model for CLI dispatches; otherwise recorded as - /// provenance. - /// - /// The commands `dispatch` spawns include the harness-native model flag when - /// the adapter supports one (e.g. Codex's `-m`, Claude Code's `--model`); - /// otherwise the value is persisted to `conditions.json` for - /// `promote-baseline`. - #[arg(long)] - pub agent_model: Option, - /// Environment override for eval-agent dispatches (`KEY=VALUE`, repeatable). - /// - /// Descriptor defaults from `[dispatch.env]` apply first; repeated CLI - /// entries override them by key, with the last occurrence winning. Values - /// may be empty and may contain `=`. The resolved map is recorded in - /// `conditions.json` and `dispatch.json`, so do not use this flag for - /// secrets. Runner-owned `command_check` assertions are unaffected. Unset - /// keys keep inheriting the operator's environment. - #[arg(long, value_name = "KEY=VALUE")] - pub agent_env: Vec, - /// Default judge model for emitted judge tasks. - /// - /// `grade` writes this into `judge-tasks.json` for judge tasks that do not - /// have an assertion-level `model` override, and `dispatch --judges` passes - /// it through using the harness-native model flag. Also persists to - /// `conditions.json` for `promote-baseline`. - #[arg(long)] - pub judge_model: Option, - /// Default verdict count for authored LLM-judge assertions (default: 1). - /// - /// An assertion-level `samples` value overrides this option. Counts above one - /// dispatch independent judges over the same bounded evidence bundle and are - /// reported as vote proportion p plus pass^k = p^N. The framework-injected - /// skill-invocation meta-check remains single-shot. See - /// `eval-magic docs judging`. - #[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u32).range(1..))] - pub judge_samples: u32, - /// Model that answers the agent for evals declaring a `responder`. - /// - /// `dispatch` consults it once after every round, through the same harness - /// as the agent under test, using the harness-native model flag. It is - /// run-level on purpose: answering one eval with a different model than its - /// neighbours puts a second uncontrolled variable inside the comparison. - /// Omit it to answer on the harness's default model. Also persists to - /// `conditions.json` for `promote-baseline`. See - /// `eval-magic docs conversations`. - #[arg(long)] - pub responder_model: Option, - /// Provenance label for this run, persisted into `conditions.json`. - /// - /// Surfaced in `BASELINE.md` by `promote-baseline` (its own `--label` flag - /// still overrides). - #[arg(long)] - pub label: Option, -} - #[derive(Debug, Args)] pub struct DispatchArgs { #[command(flatten)] diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 289f69d..4b1a5d9 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -36,6 +36,7 @@ mod compare_args; mod help; mod init_args; mod run; +mod run_args; use args::{Cli, Commands, CommonArgs, RunArgs}; use commands::*; diff --git a/src/cli/run/orchestrate/build.rs b/src/cli/run/orchestrate/build.rs index bf8d87f..800bae8 100644 --- a/src/cli/run/orchestrate/build.rs +++ b/src/cli/run/orchestrate/build.rs @@ -90,8 +90,12 @@ pub(super) fn write_dispatch( // was prepared with; the path makes the remedy copy-pasteable. harness_file: ctx.harness_file.as_deref().map(artifact_path), harness_descriptor_digest: Some(crate::adapters::registry::descriptor_digest(ctx.harness)), + guard_armed: Some(opts.guard_armed()), }; write_json(&r.iteration_dir.join("conditions.json"), &conditions)?; + let guard_armed = conditions + .guard_armed + .expect("run-created conditions record guard state"); // One record, cloned into every task: grading reads `run.json` alone, so a // result can only be tied to a skill revision if each record names one. let skill_source_record = r.skill.record(); @@ -305,7 +309,7 @@ pub(super) fn write_dispatch( &tasks, ManifestContext { harness: ctx.harness, - guard: opts.guard_armed(), + guard: guard_armed, agent_model: opts.agent_model, agent_env: &opts.agent_env, }, @@ -359,7 +363,7 @@ pub(super) fn write_dispatch( let envelope = dispatch_json .as_object_mut() .expect("dispatch envelope is an object"); - envelope.insert("guard".to_string(), Value::Bool(opts.guard_armed())); + envelope.insert("guard".to_string(), Value::Bool(guard_armed)); envelope.insert("harness_descriptor".to_string(), descriptor.clone()); } // The isolation-batch plan the executing session/human follows: which evals diff --git a/src/cli/run_args.rs b/src/cli/run_args.rs new file mode 100644 index 0000000..6280c78 --- /dev/null +++ b/src/cli/run_args.rs @@ -0,0 +1,220 @@ +//! Arguments and help text for preparing an eval run. + +use clap::Args; + +use super::args::CommonArgs; + +/// `run` adds the build-time flags (mode/baseline selection, staging toggles, +/// guard, plan-mode, bootstrap) on top of the common set. +#[derive(Debug, Args)] +pub(crate) struct RunArgs { + #[command(flatten)] + pub common: CommonArgs, + /// Baseline snapshot label (defaults to `baseline` in `--mode revision`). + /// + /// The snapshot label to use as the `old_skill` arm in revision mode (see + /// `snapshot`). + #[arg(long)] + pub baseline: Option, + /// SessionStart-equivalent bootstrap file inlined into each dispatch. + /// + /// A Markdown file prepended verbatim to every dispatch prompt inside a + /// `` block — product-specific framing a SessionStart + /// hook would inject. It does NOT enumerate skills (the auto-built + /// available-skills block is the single source of the skill list). Omit it and + /// dispatches carry only that inventory. + #[arg(long)] + pub bootstrap: Option, + /// Build the workspace but skip guard install and stop before next steps. + #[arg(long)] + pub dry_run: bool, + /// Inline each condition's SKILL.md into the dispatch prompt instead of + /// staging it under the harness skills dir. + /// + /// For harnesses without project-local skill discovery. Forces the LLM-judge + /// meta-check tier and inlines only SKILL.md (not sibling skills or sibling + /// asset files); use the staged (default) path when the measured behavior + /// depends on sibling files. The isolated env (`env/`) is still built either + /// way — `--no-stage` only skips populating the harness skills dir. Also + /// disables the write guard (auto-armed or explicit) — it requires staging — + /// so no-stage runs are unguarded and rely on `detect-stray-writes` after + /// the fact. + #[arg(long)] + pub no_stage: bool, + /// Arm the write guard explicitly (it auto-arms when the harness supports it). + /// + /// The write guard arms automatically on guard-capable built-in harnesses + /// whenever staging is active, so this flag only makes the request explicit: + /// where auto-arm quietly stays off (no declared guard, or `--no-stage`), an + /// explicit `--guard` warns, and a harness defined only by user-supplied + /// descriptors rejects it in preflight (rerun without it; + /// `detect-stray-writes` audits after the fact). Opt out with `--no-guard`. + /// The resolved state is written as `guard_armed` in `conditions.json`, + /// mirrored as `guard` in `dispatch.json` and `guard_armed` in + /// `benchmark.json`, and included in promoted baseline provenance. + /// + /// The guard is a harness-native `PreToolUse` hook that *blocks* subagent + /// writes/installs outside the isolated run env (the agent-under-test's cwd) + /// while dispatches run. The task env is its sole allowed write root; host temp + /// directories are out of bounds. Dispatch prompts name `/tmp` as + /// the task-local scratch directory (create it when needed); eval-magic does + /// not rewrite `TMPDIR`, `TMP`, or `TEMP`. Because the framework designates + /// that directory, what an agent puts there is excluded from diff scope and + /// from the project's own ignore files — so a `diff_scope` budget covers the + /// change, not the scratch work, and judges never read scratch notes as the + /// deliverable. + /// Because the harness already cwd-bounds the agent's direct file tools to the + /// env, the guard's main remaining value is blocking Bash-subprocess escapes the + /// cwd boundary doesn't cover and acting as a backstop when the isolated session + /// runs with relaxed permissions. Recognized development mutations require an + /// allowance from the eval's `guard` configuration. `allow_commands` grants + /// literal shell-token prefixes; `allow_tools` grants every invocation of an + /// executable basename. A per-eval block replaces the config-level default. With + /// no explicit block, eval-magic composes packaged profiles detected from the + /// staged task tree. See `eval-magic docs guard` for configuration, matching, + /// packaged profiles, and examples. + /// + /// Command allowances never bypass containment checks. Known destination options + /// with dynamic, missing, or outside values are blocked, as are global/user + /// install modes that do not have a supported in-env destination. Recognized + /// destinations include npm `--prefix`, pnpm `-C`/`--dir`, Yarn/Bun `--cwd`, + /// pip `--target`/`--prefix`/`--root`/`--src`, and Cargo `-C`/`--target-dir` + /// plus its target-dir environment variables. Generic shell commands are not a + /// complete parser: for example, a bare `touch /outside` remains an + /// after-the-fact `detect-stray-writes` concern. + /// + /// Local Git operations such as status, diff, add, commit, and branching are + /// allowed inside the task repository. Repository-routing escapes and remote Git + /// operations are blocked; `--no-guard` opts out of those blocks, though task + /// repositories still begin with no remotes. Literal relative redirect and `tee` + /// targets resolve from the tool invocation cwd; dynamic, malformed, or outside + /// targets are blocked. Every denial appends privacy-safe metadata + /// (never the full command or patch) to the task's + /// `.eval-magic-outputs/guard-denials.jsonl`; `ingest` joins those logs into + /// `guard-denials.json`, and `aggregate` emits one validity warning per + /// affected task. Re-arming truncates stale raw records; disarming the guard + /// preserves the current log. The marker auto-expires after 6h and is torn + /// down at the next run; while armed the + /// hook fires on your own tool calls too. If it remains armed after `finalize`, + /// `finalize` reminds you to run `teardown` before editing source (which disarms + /// the cwd guard and every per-`(group, condition)` Cli env's guard). Requires + /// staging — with `--no-stage` the guard stays off and the run is unguarded. + /// Codex eval-agent dispatches must include + /// `--dangerously-bypass-hook-trust` so the vetted project-local eval hook + /// runs; judge dispatches omit it because judges run outside guarded task envs. + /// Unguarded, stray writes are only *detected* after the fact by + /// `detect-stray-writes`, never blocked. + /// Under Claude Code the `PreToolUse` hook is staged in each env's + /// `.claude/settings.local.json`, and each `claude -p` dispatch loads it from + /// that cwd (`cd `), enforcing the eval boundary (the recipe never + /// passes `--bare`). + /// When invoking this from inside Codex, staging writes `.agents/skills` and + /// guarded runs also write `.codex/hooks.json`; Codex protects those paths in + /// its default workspace-write sandbox, so approval/escalation may be needed. + #[arg(long)] + pub guard: bool, + /// Opt out of the write guard for this run. + /// + /// The guard arms automatically on harnesses that declare one (see + /// `--guard`). Pass this to run unguarded — e.g. when the skill under test + /// legitimately writes outside the isolated run env — and to silence the + /// unguarded-harness preflight warning. Unguarded, out-of-bounds writes are + /// only *detected* after the fact by `detect-stray-writes` (folded into + /// `ingest`), never blocked. + /// + /// Dispatches deliberately run with relaxed harness permissions so the + /// agent-under-test can actually execute commands, which makes the guard + /// the only enforcement boundary. Opting out therefore leaves the dispatch + /// with no boundary at all, not merely a weaker one. + #[arg(long, conflicts_with = "guard")] + pub no_guard: bool, + /// Stage the skill-under-test under this verbatim name instead of the + /// conspicuous `slow-powers-eval-…` slug. + /// + /// For name-confound experiments. A scalar `skill_name` and one staging + /// condition are required; a multi-skill treatment is rejected because one + /// override cannot name every member. Refuses to clobber an existing dir and + /// registers the staged name for next-run cleanup. + #[arg(long)] + pub stage_name: Option, + /// Inject the shared plan-mode profile as an operating-context layer. + /// + /// Injects the shared, harness-agnostic plan-mode procedure + /// (`profiles/shared/plan-mode.md`) as a `` in every + /// dispatch, identical across arms and harnesses. Opt-in, for + /// plan-mode-relevant skills. It is text the subagent reads, not a real + /// injected mode. + #[arg(long)] + pub plan_mode: bool, + /// Runs per condition cell, for variance reduction (default: 1). + /// + /// Dispatches every eval N times per condition, so an iteration needs + /// `evals × 2 conditions × N` dispatches. Each run gets its own + /// `run-/` directory under the condition (own `inputs/`, `outputs/`, + /// `run.json`, `timing.json`, `grading.json`) and a unique + /// `agent_description` carrying an `r` segment. With N=1 the layout is + /// unchanged (artifacts sit directly in the condition directory). The + /// benchmark's per-condition `mean`/`stddev`/`n` then reflect all runs. A + /// per-eval `runs` field in evals.json overrides this flag for that eval. + /// Before staging, a fully binary run summary prints the minimum attainable + /// two-sided Fisher exact p-value for each effective run count, assuming + /// perfect separation between the two conditions. A run with sampled LLM + /// assertions instead identifies vote proportion and pass^k as non-binary + /// endpoints. eval-magic does not calculate observed p-values or apply a + /// significance threshold. + #[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u32).range(1..))] + pub runs: u32, + /// Agent-under-test model for CLI dispatches; otherwise recorded as + /// provenance. + /// + /// The commands `dispatch` spawns include the harness-native model flag when + /// the adapter supports one (e.g. Codex's `-m`, Claude Code's `--model`); + /// otherwise the value is persisted to `conditions.json` for + /// `promote-baseline`. + #[arg(long)] + pub agent_model: Option, + /// Environment override for eval-agent dispatches (`KEY=VALUE`, repeatable). + /// + /// Descriptor defaults from `[dispatch.env]` apply first; repeated CLI + /// entries override them by key, with the last occurrence winning. Values + /// may be empty and may contain `=`. The resolved map is recorded in + /// `conditions.json` and `dispatch.json`, so do not use this flag for + /// secrets. Runner-owned `command_check` assertions are unaffected. Unset + /// keys keep inheriting the operator's environment. + #[arg(long, value_name = "KEY=VALUE")] + pub agent_env: Vec, + /// Default judge model for emitted judge tasks. + /// + /// `grade` writes this into `judge-tasks.json` for judge tasks that do not + /// have an assertion-level `model` override, and `dispatch --judges` passes + /// it through using the harness-native model flag. Also persists to + /// `conditions.json` for `promote-baseline`. + #[arg(long)] + pub judge_model: Option, + /// Default verdict count for authored LLM-judge assertions (default: 1). + /// + /// An assertion-level `samples` value overrides this option. Counts above one + /// dispatch independent judges over the same bounded evidence bundle and are + /// reported as vote proportion p plus pass^k = p^N. The framework-injected + /// skill-invocation meta-check remains single-shot. See + /// `eval-magic docs judging`. + #[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u32).range(1..))] + pub judge_samples: u32, + /// Model that answers the agent for evals declaring a `responder`. + /// + /// `dispatch` consults it once after every round, through the same harness + /// as the agent under test, using the harness-native model flag. It is + /// run-level on purpose: answering one eval with a different model than its + /// neighbours puts a second uncontrolled variable inside the comparison. + /// Omit it to answer on the harness's default model. Also persists to + /// `conditions.json` for `promote-baseline`. See + /// `eval-magic docs conversations`. + #[arg(long)] + pub responder_model: Option, + /// Provenance label for this run, persisted into `conditions.json`. + /// + /// Surfaced in `BASELINE.md` by `promote-baseline` (its own `--label` flag + /// still overrides). + #[arg(long)] + pub label: Option, +} diff --git a/src/core/types.rs b/src/core/types.rs index 1995e4d..361146b 100644 --- a/src/core/types.rs +++ b/src/core/types.rs @@ -512,6 +512,10 @@ pub struct ConditionsRecord { /// existed, which read as "nothing to compare against". #[serde(default, skip_serializing_if = "Option::is_none")] pub harness_descriptor_digest: Option, + /// Whether the run preflight resolved the write guard to armed. Absent in + /// historical records, where the effective state is unknown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub guard_armed: Option, } /// Comparison mode for a run. @@ -921,6 +925,7 @@ mod tests { skill_source: None, harness_file: None, harness_descriptor_digest: None, + guard_armed: None, }; let out = serde_json::to_value(&rec).unwrap(); assert_eq!(out.get("mode"), Some(&Value::String("new-skill".into()))); diff --git a/src/pipeline/aggregate.rs b/src/pipeline/aggregate.rs index f5dce92..6c1dc85 100644 --- a/src/pipeline/aggregate.rs +++ b/src/pipeline/aggregate.rs @@ -147,6 +147,10 @@ pub struct Benchmark { /// both sides without a reader holding two artifacts side by side. #[serde(skip_serializing_if = "Option::is_none")] pub skill_source: Option, + /// Effective write-guard state echoed from `conditions.json`. Absence + /// means the iteration predates guard provenance. + #[serde(skip_serializing_if = "Option::is_none")] + pub guard_armed: Option, #[serde(skip_serializing_if = "Option::is_none")] pub diff_scope: Option, delta: Delta, @@ -528,6 +532,7 @@ pub fn aggregate( baseline: conditions.baseline.clone(), codebases: conditions.codebases.clone(), skill_source: conditions.skill_source.clone(), + guard_armed: conditions.guard_armed, conditions_compared: vec![a.clone(), b.clone()], missing_gradings, validity_warnings, diff --git a/src/workspace/promote.rs b/src/workspace/promote.rs index e8d91a7..93f437a 100644 --- a/src/workspace/promote.rs +++ b/src/workspace/promote.rs @@ -420,6 +420,9 @@ fn provenance(opts: &PromoteOptions, conditions: Option<&ConditionsRecord>, head condition_names.join(", ") }; let harness = label(&opts.harness); + let guard_armed = conditions + .and_then(|c| c.guard_armed) + .map_or_else(|| "unknown".to_string(), |armed| armed.to_string()); // Provenance precedence: explicit promote-baseline flag → value recorded in // the iteration's conditions.json (set via `run`) → placeholder. @@ -460,6 +463,7 @@ fn provenance(opts: &PromoteOptions, conditions: Option<&ConditionsRecord>, head format!("| Mode | {mode} |"), format!("| Iteration | iteration-{} |", opts.iteration), format!("| Harness | {harness} |"), + format!("| Guard armed | {guard_armed} |"), format!("| Agent model | {agent_model} |"), format!("| Judge model | {judge_model} |"), format!("| Responder model | {responder_model} |"), diff --git a/src/workspace/promote/tests.rs b/src/workspace/promote/tests.rs index 7b368c7..20fadb0 100644 --- a/src/workspace/promote/tests.rs +++ b/src/workspace/promote/tests.rs @@ -2,6 +2,8 @@ use super::*; use serde_json::Value; use tempfile::TempDir; +mod guard_provenance; + /// Write `body` to `path`, creating parent dirs. fn write(path: &Path, body: &str) { fs::create_dir_all(path.parent().unwrap()).unwrap(); diff --git a/src/workspace/promote/tests/guard_provenance.rs b/src/workspace/promote/tests/guard_provenance.rs new file mode 100644 index 0000000..6e1ca2d --- /dev/null +++ b/src/workspace/promote/tests/guard_provenance.rs @@ -0,0 +1,33 @@ +use super::*; + +#[test] +fn distinguishes_guard_state_from_legacy_unknown() { + for (guard_armed, expected) in [ + (Some(true), "true"), + (Some(false), "false"), + (None, "unknown"), + ] { + let f = fixture(1); + let mut conditions: Value = serde_json::from_str(CONDITIONS_WITH_PROVENANCE).unwrap(); + if let Some(guard_armed) = guard_armed { + conditions["guard_armed"] = Value::Bool(guard_armed); + } + write( + &f.iteration_dir.join("conditions.json"), + &serde_json::to_string(&conditions).unwrap(), + ); + write( + &f.iteration_dir.join("benchmark.json"), + r#"{"delta":{"pass_rate":0}}"#, + ); + + promote_baseline(&opts(&f, 1)).unwrap(); + + let provenance = + fs::read_to_string(f.skill_subdir.join("evals/baseline/BASELINE.md")).unwrap(); + assert!( + provenance.contains(&format!("| Guard armed | {expected} |")), + "guard_armed={guard_armed:?}: {provenance}" + ); + } +} diff --git a/tests/cli/aggregate.rs b/tests/cli/aggregate.rs index 47fa906..8aef295 100644 --- a/tests/cli/aggregate.rs +++ b/tests/cli/aggregate.rs @@ -6,6 +6,7 @@ use std::fs; use std::process::Command as StdCommand; mod assertions; +mod guard_provenance; mod shadow; mod shadow_verification; diff --git a/tests/cli/aggregate/guard_provenance.rs b/tests/cli/aggregate/guard_provenance.rs new file mode 100644 index 0000000..5ccb382 --- /dev/null +++ b/tests/cli/aggregate/guard_provenance.rs @@ -0,0 +1,51 @@ +use super::*; + +fn aggregate_with_guard_state(guard_armed: Option) -> serde_json::Value { + let (_tmp, root) = canonical_root(); + let (skill_dir, skill_md, iteration_dir, cwd) = setup_agg(&root); + new_skill_conditions(&iteration_dir, &skill_md); + if let Some(guard_armed) = guard_armed { + let conditions_path = iteration_dir.join("conditions.json"); + let mut conditions = serde_json::from_str::( + &fs::read_to_string(&conditions_path).unwrap(), + ) + .unwrap(); + conditions["guard_armed"] = serde_json::json!(guard_armed); + fs::write( + &conditions_path, + serde_json::to_string(&conditions).unwrap(), + ) + .unwrap(); + } + for condition in ["with_skill", "without_skill"] { + write_grading(&iteration_dir, condition, 1.0); + write_timing( + &iteration_dir, + condition, + serde_json::json!({"total_tokens": 100, "duration_ms": 1}), + ); + } + + agg_cmd(&cwd, &skill_dir).assert().success(); + + read_benchmark(&iteration_dir) +} + +#[test] +fn echoes_effective_guard_provenance() { + for guard_armed in [true, false] { + assert_eq!( + aggregate_with_guard_state(Some(guard_armed))["guard_armed"], + guard_armed + ); + } +} + +#[test] +fn leaves_historical_guard_state_unknown() { + let benchmark = aggregate_with_guard_state(None); + assert!( + benchmark.get("guard_armed").is_none(), + "legacy conditions must not be rewritten as unguarded: {benchmark}" + ); +} diff --git a/tests/cli/docs.rs b/tests/cli/docs.rs index 53c32b4..a84b84b 100644 --- a/tests/cli/docs.rs +++ b/tests/cli/docs.rs @@ -180,13 +180,16 @@ fn docs_guard_keeps_configuration_defaults_and_boundary_contracts() { .stdout(contains("framework/nextjs")) .stdout(contains("replaces")) .stdout(contains("dispatch.json")) + .stdout(contains("guard_armed")) + .stdout(contains("unknown")) .stdout(contains("cannot override")); skill_eval() .args(["run", "--help"]) .assert() .success() - .stdout(contains("eval-magic docs guard")); + .stdout(contains("eval-magic docs guard")) + .stdout(contains("guard_armed")); } /// Judge evidence is the primary grading input, so the shipped reference must diff --git a/tests/fixtures/conditions/claude-code.json b/tests/fixtures/conditions/claude-code.json index 199a907..d3cf63a 100644 --- a/tests/fixtures/conditions/claude-code.json +++ b/tests/fixtures/conditions/claude-code.json @@ -15,5 +15,6 @@ "timestamp": "2026-07-14T04:29:09.009Z", "harness": "claude-code", "run_nonce": "mrk5j5zj-6d2180", - "runs": 1 + "runs": 1, + "guard_armed": true } diff --git a/tests/fixtures/conditions/cline.json b/tests/fixtures/conditions/cline.json index d82b2a6..af2aa13 100644 --- a/tests/fixtures/conditions/cline.json +++ b/tests/fixtures/conditions/cline.json @@ -15,5 +15,6 @@ "timestamp": "2026-07-14T04:29:09.075Z", "harness": "cline", "run_nonce": "mrk5j619-276678", - "runs": 1 + "runs": 1, + "guard_armed": true } diff --git a/tests/fixtures/conditions/codex.json b/tests/fixtures/conditions/codex.json index 95e25bf..acb83ff 100644 --- a/tests/fixtures/conditions/codex.json +++ b/tests/fixtures/conditions/codex.json @@ -15,5 +15,6 @@ "timestamp": "2026-07-14T04:29:09.046Z", "harness": "codex", "run_nonce": "mrk5j60f-535258", - "runs": 1 + "runs": 1, + "guard_armed": true } diff --git a/tests/fixtures/conditions/opencode.json b/tests/fixtures/conditions/opencode.json index 521c90d..90eb4b9 100644 --- a/tests/fixtures/conditions/opencode.json +++ b/tests/fixtures/conditions/opencode.json @@ -15,5 +15,6 @@ "timestamp": "2026-07-14T04:29:09.075Z", "harness": "opencode", "run_nonce": "mrk5j619-276678", - "runs": 1 + "runs": 1, + "guard_armed": true } diff --git a/tests/run/byoh.rs b/tests/run/byoh.rs index 0595efd..17825d3 100644 --- a/tests/run/byoh.rs +++ b/tests/run/byoh.rs @@ -12,6 +12,7 @@ use std::fs; use std::path::{Path, PathBuf}; mod extract; +mod guard_provenance; /// A runner-ready BYOH descriptor using the named Codex transcript capability. const COOL_DESCRIPTOR: &str = r#"label = "cool-custom-harness" @@ -247,37 +248,6 @@ fn guard_with_a_user_only_harness_is_rejected_in_preflight() { ); } -/// Auto-arm never turns the user-only-descriptor restriction into an error: -/// without an explicit `--guard`, the run proceeds unguarded with a warning -/// naming the fallback. -#[test] -fn auto_guard_stays_off_without_error_on_user_only_harness() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - write_project_descriptor(&cwd, COOL_DESCRIPTOR); - - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args([ - "--skill", - "mr-review", - "--mode", - "new-skill", - "--harness", - "cool-custom-harness", - ]) - .assert() - .success() - .stderr(contains("declares no write guard").and(contains("detect-stray-writes"))); - - assert!( - iteration_dir(&cwd).join("dispatch.json").exists(), - "the run builds; only explicit --guard is rejected" - ); -} - /// A transcript parser is a runner-readiness requirement regardless of which /// assertion types the eval declares. #[test] diff --git a/tests/run/byoh/guard_provenance.rs b/tests/run/byoh/guard_provenance.rs new file mode 100644 index 0000000..bb9d5e5 --- /dev/null +++ b/tests/run/byoh/guard_provenance.rs @@ -0,0 +1,36 @@ +use super::*; + +/// Auto-arm never turns the user-only-descriptor restriction into an error: +/// without an explicit `--guard`, the run proceeds unguarded with a warning +/// naming the fallback. +#[test] +fn auto_guard_stays_off_without_error_on_user_only_harness() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); + write_project_descriptor( + &cwd, + &format!("skills_dir = \".cool/skills\"\nconfig_dirs = [\".cool\"]\n{COOL_DESCRIPTOR}"), + ); + + skill_eval() + .current_dir(&cwd) + .args(["run", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--mode", + "new-skill", + "--harness", + "cool-custom-harness", + ]) + .assert() + .success() + .stderr(contains("declares no write guard").and(contains("detect-stray-writes"))); + + let iteration = iteration_dir(&cwd); + let dispatch = read_json(&iteration.join("dispatch.json")); + let conditions = read_json(&iteration.join("conditions.json")); + assert_eq!(conditions["guard_armed"], false); + assert_eq!(dispatch["guard"], conditions["guard_armed"]); +} diff --git a/tests/run/codex.rs b/tests/run/codex.rs index f6764df..aa7cda5 100644 --- a/tests/run/codex.rs +++ b/tests/run/codex.rs @@ -31,6 +31,8 @@ fn codex_no_stage_keeps_inline_fallback() { let conditions = read_json(&iteration_dir(&cwd).join("conditions.json")); assert_eq!(dispatch["harness"], "codex"); assert_eq!(conditions["harness"], "codex"); + assert_eq!(conditions["guard_armed"], false); + assert_eq!(dispatch["guard"], conditions["guard_armed"]); assert!(!cwd.join(".claude/skills").exists()); assert!(!cwd.join(".agents/skills").exists()); } diff --git a/tests/run/lifecycle.rs b/tests/run/lifecycle.rs index 11d808e..af7b779 100644 --- a/tests/run/lifecycle.rs +++ b/tests/run/lifecycle.rs @@ -7,6 +7,8 @@ use predicates::str::contains; use std::fs; use std::path::Path; +mod guard_provenance; + #[test] fn missing_git_fails_before_creating_an_iteration() { let tmp = tempfile::TempDir::new().unwrap(); @@ -233,57 +235,6 @@ fn dry_run_skips_guard_install() { ); } -#[test] -fn default_run_auto_arms_guard_in_each_env() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - // No guard flag at all: the harness declares guard support, so the run - // arms it automatically (#126 — enhancements are provided, not opted into). - let assert = skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args(["--skill", "mr-review", "--mode", "new-skill"]) - .assert() - .success(); - let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); - assert!( - stdout.contains("guard: armed"), - "the run plan reports the armed guard: {stdout}" - ); - for condition in ["with_skill", "without_skill"] { - let env = cli_env_dir(&cwd, "g1", condition); - assert!( - env.join(".claude/settings.local.json").exists(), - "guard hook staged in env-g1-{condition}" - ); - assert!( - env.join(".claude/skills/.slow-powers-eval-guard.json") - .exists(), - "guard marker armed in env-g1-{condition}" - ); - } -} - -#[test] -fn no_guard_run_installs_no_guard() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args(["--skill", "mr-review", "--mode", "new-skill", "--no-guard"]) - .assert() - .success(); - let env = cli_env_dir(&cwd, "g1", "with_skill"); - assert!(!env.join(".claude/settings.local.json").exists()); - assert!( - !env.join(".claude/skills/.slow-powers-eval-guard.json") - .exists() - ); -} - #[test] fn namespaces_agent_description_and_records_run_nonce() { let tmp = tempfile::TempDir::new().unwrap(); diff --git a/tests/run/lifecycle/guard_provenance.rs b/tests/run/lifecycle/guard_provenance.rs new file mode 100644 index 0000000..35a591e --- /dev/null +++ b/tests/run/lifecycle/guard_provenance.rs @@ -0,0 +1,62 @@ +use super::*; + +#[test] +fn default_run_auto_arms_guard_in_each_env() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); + // No guard flag at all: the harness declares guard support, so the run + // arms it automatically (#126 — enhancements are provided, not opted into). + let assert = skill_eval() + .current_dir(&cwd) + .args(["run", "--skill-dir"]) + .arg(&skill_dir) + .args(["--skill", "mr-review", "--mode", "new-skill"]) + .assert() + .success(); + let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); + assert!( + stdout.contains("guard: armed"), + "the run plan reports the armed guard: {stdout}" + ); + let iteration = iteration_dir(&cwd); + let conditions = read_json(&iteration.join("conditions.json")); + let dispatch = read_json(&iteration.join("dispatch.json")); + assert_eq!(conditions["guard_armed"], true); + assert_eq!(dispatch["guard"], conditions["guard_armed"]); + for condition in ["with_skill", "without_skill"] { + let env = cli_env_dir(&cwd, "g1", condition); + assert!( + env.join(".claude/settings.local.json").exists(), + "guard hook staged in env-g1-{condition}" + ); + assert!( + env.join(".claude/skills/.slow-powers-eval-guard.json") + .exists(), + "guard marker armed in env-g1-{condition}" + ); + } +} + +#[test] +fn no_guard_run_installs_no_guard() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); + skill_eval() + .current_dir(&cwd) + .args(["run", "--skill-dir"]) + .arg(&skill_dir) + .args(["--skill", "mr-review", "--mode", "new-skill", "--no-guard"]) + .assert() + .success(); + let env = cli_env_dir(&cwd, "g1", "with_skill"); + assert!(!env.join(".claude/settings.local.json").exists()); + assert!( + !env.join(".claude/skills/.slow-powers-eval-guard.json") + .exists() + ); + let iteration = iteration_dir(&cwd); + let conditions = read_json(&iteration.join("conditions.json")); + let dispatch = read_json(&iteration.join("dispatch.json")); + assert_eq!(conditions["guard_armed"], false); + assert_eq!(dispatch["guard"], conditions["guard_armed"]); +}