Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions docs/guides/guard.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ Without an explicit `guard` field, eval-magic detects packaged profiles from the

## Understand the two policy layers

Containment checks run before command allowances. A command policy cannot override these checks:
Containment checks run before command allowances. A command policy cannot override these checks.
A command that violates both layers is denied once, with a verdict that names each blocking reason,
so the denial never points at a fix that cannot unblock the command:

- Direct write and patch tools must target the task environment.
- Shell redirects and `tee` targets must resolve inside the task environment.
Expand Down Expand Up @@ -113,8 +115,8 @@ The packaged profiles are:
- `language/rust` is detected from `Cargo.toml`. It allows `cargo build`, `check`, `test`, `run`,
`fmt`, and `clippy`.
- `language/javascript` is detected from `package.json`. It allows npm install, CI, test, build,
lint, and typecheck commands, plus corresponding pnpm, Yarn, and Bun install, add, test, build,
lint, and typecheck commands.
dev, start, lint, and typecheck commands, plus corresponding pnpm, Yarn, and Bun install, add,
test, build, dev, start, lint, and typecheck commands.
- `framework/nextjs` is detected when `package.json` declares a `next` dependency. It allows npm,
pnpm, Yarn, and Bun dev, build, and start scripts, plus direct `next` invocations through `npx`,
`pnpm exec`, `yarn`, and `bunx`.
Expand Down
8 changes: 8 additions & 0 deletions guard-profiles/language-javascript.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,32 @@ allow_commands = [
"npm ci",
"npm test",
"npm run build",
"npm run dev",
"npm run lint",
"npm run start",
"npm run typecheck",
"pnpm install",
"pnpm add",
"pnpm test",
"pnpm run build",
"pnpm run dev",
"pnpm run lint",
"pnpm run start",
"pnpm run typecheck",
"yarn install",
"yarn add",
"yarn test",
"yarn run build",
"yarn run dev",
"yarn run lint",
"yarn run start",
"yarn run typecheck",
"bun install",
"bun add",
"bun test",
"bun run build",
"bun run dev",
"bun run lint",
"bun run start",
"bun run typecheck",
]
2 changes: 1 addition & 1 deletion src/adapters/guard.rs
Original file line number Diff line number Diff line change
Expand Up @@ -650,7 +650,7 @@ mod tests {
assert_eq!(
verdict("codex", payload, Some(marker())).expect("should block"),
"{\"decision\":\"block\",\"reason\":\"eval guard: blocked Bash \
(package install/add) — runs outside the eval sandbox\"}"
(package install/add — runs outside the eval sandbox; command not allowed by eval guard policy)\"}"
);
}

Expand Down
110 changes: 95 additions & 15 deletions src/sandbox/decide.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use crate::core::fs::artifact_path;

use super::command_policy::COMMAND_POLICY_REASON;
use super::policy::{
OUTPUT_REDIRECTION_REASON, apply_patch_paths, classify_bash_with_policy, is_patch_tool,
OUTPUT_REDIRECTION_REASON, apply_patch_paths, classify_bash_denials, is_patch_tool,
is_shell_tool, is_under_any, is_write_tool, path_arg, resolve_path,
};

Expand Down Expand Up @@ -220,25 +220,49 @@ pub(crate) fn decide_with_cwd(
.get("command")
.and_then(Value::as_str)
.unwrap_or("");
if let Some(classification) =
classify_bash_with_policy(command, &roots, invocation_cwd, guard_policy)
{
let hint = if classification.reason == OUTPUT_REDIRECTION_REASON {
scratch_hint(&roots)
let denials = classify_bash_denials(command, &roots, invocation_cwd, guard_policy);
if !denials.is_empty() {
// A containment denial stays a self-contained clause when the
// command policy also denies, so one verdict can name every
// blocking layer instead of sending the agent to fix a problem
// that cannot unblock the command.
let clause = |reason: &str| {
if reason == COMMAND_POLICY_REASON {
reason.to_string()
} else {
format!("{reason} — runs outside the eval sandbox")
}
};
let verdict = if let [only] = denials.as_slice() {
let boundary = if only.reason == COMMAND_POLICY_REASON {
""
} else {
" — runs outside the eval sandbox"
};
format!("({}){boundary}", only.reason)
} else {
String::new()
let clauses = denials
.iter()
.map(|denial| clause(denial.reason))
.collect::<Vec<_>>()
.join("; ");
format!("({clauses})")
};
let boundary = if classification.reason == COMMAND_POLICY_REASON {
""
let hint = if denials
.iter()
.any(|denial| denial.reason == OUTPUT_REDIRECTION_REASON)
{
scratch_hint(&roots)
} else {
" — runs outside the eval sandbox"
String::new()
};
let resolved_targets = denials
.into_iter()
.flat_map(|denial| denial.resolved_targets)
.collect();
return GuardEvaluation::deny(
format!(
"{GUARD_REASON_PREFIX}blocked {tool_name} ({}){boundary}{hint}",
classification.reason,
),
classification.resolved_targets,
format!("{GUARD_REASON_PREFIX}blocked {tool_name} {verdict}{hint}"),
resolved_targets,
);
}
}
Expand Down Expand Up @@ -407,6 +431,62 @@ mod tests {
);
}

/// Issue #297: a command both layers would deny must get one verdict that
/// names both reasons. Naming only the redirect (with its actionable
/// scratch hint) sends the agent to fix a problem that cannot unblock the
/// command, because the command policy was already denying it.
#[test]
fn a_bash_denial_names_every_blocking_layer_in_one_verdict() {
let marker: GuardMarker = serde_json::from_value(json!({
"active": true,
"allowedRoots": ["/work/.eval-magic/task"],
"guardPolicy": { "allow_tools": ["cargo"] }
}))
.unwrap();

let denied = decide_with_cwd(
"Bash",
&json!({ "command": "npm run dev > /tmp/dev-server.log 2>&1 &" }),
Some(&marker),
now_ms(),
Path::new("/work/.eval-magic/task"),
);

assert!(!denied.decision.allow);
let reason = denied.decision.reason.unwrap();
assert!(reason.contains("output redirection to a file"), "{reason}");
assert!(
reason.contains("command not allowed by eval guard policy"),
"{reason}"
);
assert!(
reason.ends_with("For temporary or scratch files, use /work/.eval-magic/task/tmp."),
"{reason}"
);
assert_eq!(
denied.resolved_targets,
vec!["/tmp/dev-server.log".to_string()]
);
}

#[test]
fn a_bash_denial_from_containment_alone_keeps_the_single_reason_verdict() {
let d = decide_now(
"Bash",
json!({ "command": "echo hi > /tmp/out.log" }),
Some(&marker()),
);

assert_eq!(
d.reason.as_deref(),
Some(
"eval guard: blocked Bash (output redirection to a file) \
— runs outside the eval sandbox. For temporary or scratch files, use \
/work/.eval-magic/tmp."
)
);
}

#[test]
fn allows_bash_with_an_in_bounds_redirect() {
let d = decide_now(
Expand Down
30 changes: 30 additions & 0 deletions src/sandbox/guard_profiles.rs
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,36 @@ mod tests {
assert!(!expanded.allow_commands.contains(&"npm test".to_string()));
}

/// Issue #297: a plain package.json project (the pinned Weeknight fixture
/// is Vite + React) must be able to start its own dev server. `dev` and
/// `start` are generic lifecycle script names, not Next.js-specific, so
/// they belong to the language profile.
#[test]
fn language_javascript_allows_dev_and_start_scripts() {
let policy = GuardPolicyConfig {
profiles: vec!["language/javascript".to_string()],
..GuardPolicyConfig::default()
};

let expanded = expand_policy(&policy).unwrap();

for command in [
"npm run dev",
"npm run start",
"pnpm run dev",
"pnpm run start",
"yarn run dev",
"yarn run start",
"bun run dev",
"bun run start",
] {
assert!(
expanded.allow_commands.contains(&command.to_string()),
"{command}"
);
}
}

#[test]
fn detection_joins_language_and_framework_profiles_recursively() {
let root = tempdir().unwrap();
Expand Down
33 changes: 29 additions & 4 deletions src/sandbox/policy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -198,18 +198,43 @@ pub(crate) fn classify_bash_with_cwd(
)
}

/// Classify one shell tool call under its resolved eval command policy.
/// Classify one shell tool call under its resolved eval command policy,
/// returning the first applicable denial.
pub(crate) fn classify_bash_with_policy(
command: &str,
allowed_roots: &[String],
invocation_cwd: &Path,
policy: &crate::core::GuardPolicyConfig,
) -> Option<BashClassification> {
classify_bash_denials(command, allowed_roots, invocation_cwd, policy)
.into_iter()
.next()
}

/// Every layer that denies one shell tool call, containment checks first and
/// the eval command policy second (at most one denial each).
///
/// The stray-write audit wants only the first denial, but the live guard owes
/// the agent all of them in one verdict: a verdict that names just the
/// redirect it found sends the agent to fix a problem that cannot unblock the
/// command when the command policy was already denying it.
pub(crate) fn classify_bash_denials(
command: &str,
allowed_roots: &[String],
invocation_cwd: &Path,
policy: &crate::core::GuardPolicyConfig,
) -> Vec<BashClassification> {
if command.is_empty() {
return None;
return Vec::new();
}
let mut denials = Vec::new();
if let Some(denial) = classify_fixed_containment(command, allowed_roots, invocation_cwd) {
denials.push(denial);
}
if let Some(denial) = super::command_policy::classify_command_policy(command, policy) {
denials.push(denial);
}
classify_fixed_containment(command, allowed_roots, invocation_cwd)
.or_else(|| super::command_policy::classify_command_policy(command, policy))
denials
}

fn classify_fixed_containment(
Expand Down
75 changes: 75 additions & 0 deletions src/sandbox/policy/tests/command_policy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -154,3 +154,78 @@ fn shell_wrapper_allowance_cannot_bypass_fixed_containment() {
Some("cargo build/test output")
);
}

#[test]
fn denials_reports_every_denying_layer_containment_first() {
let roots = vec!["/work/env".to_string()];
let policy = crate::core::GuardPolicyConfig {
allow_commands: vec!["cargo build".to_string()],
..crate::core::GuardPolicyConfig::default()
};

let denials = classify_bash_denials(
"npm run dev > /tmp/dev-server.log",
&roots,
Path::new("/work/env"),
&policy,
);

let reasons: Vec<&str> = denials.iter().map(|denial| denial.reason).collect();
assert_eq!(
reasons,
[
"output redirection to a file",
"command not allowed by eval guard policy"
]
);
assert_eq!(
denials[0].resolved_targets,
vec!["/tmp/dev-server.log".to_string()]
);
}

#[test]
fn denials_reports_a_single_layer_when_only_one_applies() {
let roots = vec!["/work/env".to_string()];

// Containment only: echo is unrecognized by the command policy.
let denials = classify_bash_denials(
"echo hi > /tmp/out.log",
&roots,
Path::new("/work/env"),
&crate::core::GuardPolicyConfig::default(),
);
assert_eq!(denials.len(), 1);
assert_eq!(denials[0].reason, "output redirection to a file");

// Policy only: the redirect target is in bounds, but npm is claimed and
// not allowed.
let policy = crate::core::GuardPolicyConfig {
allow_commands: vec!["cargo build".to_string()],
..crate::core::GuardPolicyConfig::default()
};
let denials = classify_bash_denials(
"npm run dev > /work/env/dev-server.log",
&roots,
Path::new("/work/env"),
&policy,
);
assert_eq!(denials.len(), 1);
assert_eq!(
denials[0].reason,
"command not allowed by eval guard policy"
);
}

#[test]
fn denials_is_empty_for_an_allowed_command() {
let roots = vec!["/work/env".to_string()];
let policy = crate::core::GuardPolicyConfig {
allow_commands: vec!["cargo build".to_string()],
..crate::core::GuardPolicyConfig::default()
};

assert!(
classify_bash_denials("cargo build", &roots, Path::new("/work/env"), &policy).is_empty()
);
}
2 changes: 1 addition & 1 deletion tests/cli/guard.rs
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ fn guard_codex_block_verdict_bytes_are_stable() {
.success()
.stdout(
"{\"decision\":\"block\",\"reason\":\"eval guard: blocked Bash \
(package install/add) — runs outside the eval sandbox\"}",
(package install/add — runs outside the eval sandbox; command not allowed by eval guard policy)\"}",
);
}

Expand Down
Loading
Loading