Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 171 additions & 3 deletions src/agent/claude_code/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,46 @@ fn an_agent_that_floods_stderr_fails_promptly_with_a_bounded_clean_reason() {
);
}

/// What a stand-in CLI runs first, as the real one does when it starts: its
/// tool server reaches the bridge with the turn's token and asks for the
/// tools. Without it a turn is refused, as a real one is when the server
/// never started (`tool_bridge::require_tools`).
///
/// One line of the stand-in's script. On Windows it calls a PowerShell file
/// written beside it (quoting JSON through `cmd` is not worth it); on Unix,
/// bash's `/dev/tcp`, so those stand-ins start with `#!/bin/bash`.
pub(crate) fn lists_the_tools(dir: &std::path::Path) -> String {
#[cfg(windows)]
{
let script = dir.join("list-tools.ps1");
std::fs::write(
&script,
"$a = $env:INCODE_TOOL_BRIDGE.Split(':')\r\n\
$c = New-Object Net.Sockets.TcpClient($a[0], [int]$a[1])\r\n\
$s = $c.GetStream()\r\n\
$w = New-Object IO.StreamWriter($s)\r\n\
$w.WriteLine('{\"op\":\"list\",\"token\":\"' + $env:INCODE_TOOL_TOKEN + '\"}')\r\n\
$w.Flush()\r\n\
[void](New-Object IO.StreamReader($s)).ReadLine()\r\n\
$c.Close()\r\n",
)
.expect("write the stand-in's tool server");
format!(
"\"%SystemRoot%\\System32\\WindowsPowerShell\\v1.0\\powershell.exe\" -NoProfile \
-ExecutionPolicy Bypass -File \"{}\" >nul\r\n",
script.display()
)
}
#[cfg(not(windows))]
{
let _ = dir;
"exec 3<>\"/dev/tcp/${INCODE_TOOL_BRIDGE%:*}/${INCODE_TOOL_BRIDGE##*:}\"\n\
printf '{\"op\":\"list\",\"token\":\"%s\"}\\n' \"$INCODE_TOOL_TOKEN\" >&3\n\
read -r _ <&3\nexec 3>&-\n"
.to_string()
}
}

/// A stand-in CLI whose option parser does not know the thinking flag:
/// named on its argv, it says so on stderr and exits 1 before any
/// output; otherwise it prints a short turn. Every start is counted in
Expand All @@ -300,6 +340,7 @@ fn a_cli_without_the_thinking_flag(dir: &std::path::Path) -> PathBuf {
echo error: unknown option '--thinking-display' 1>&2\r\n\
exit /b 1\r\n)\r\n",
);
body.push_str(&lists_the_tools(dir));
for line in LINES {
body.push_str(&format!("echo {line}\r\n"));
}
Expand All @@ -311,10 +352,11 @@ fn a_cli_without_the_thinking_flag(dir: &std::path::Path) -> PathBuf {
{
use std::os::unix::fs::PermissionsExt;
let mut body = String::from(
"#!/bin/sh\necho x >> \"$(dirname \"$0\")/starts.txt\"\n\
"#!/bin/bash\necho x >> \"$(dirname \"$0\")/starts.txt\"\n\
case \"$*\" in *--thinking-display*) \
echo \"error: unknown option '--thinking-display'\" >&2; exit 1;; esac\n",
);
body.push_str(&lists_the_tools(dir));
for line in LINES {
body.push_str(&format!("echo '{line}'\n"));
}
Expand Down Expand Up @@ -389,7 +431,7 @@ pub(crate) fn stand_in_cli(dir: &std::path::Path, waits: bool) -> PathBuf {
];
#[cfg(windows)]
{
let mut body = format!("@echo off\r\necho {}\r\n", LINES[0]);
let mut body = format!("@echo off\r\n{}echo {}\r\n", lists_the_tools(dir), LINES[0]);
if waits {
body.push_str("ping -n 30 127.0.0.1 >nul\r\n");
}
Expand All @@ -403,7 +445,7 @@ pub(crate) fn stand_in_cli(dir: &std::path::Path, waits: bool) -> PathBuf {
#[cfg(not(windows))]
{
use std::os::unix::fs::PermissionsExt;
let mut body = format!("#!/bin/sh\necho '{}'\n", LINES[0]);
let mut body = format!("#!/bin/bash\n{}echo '{}'\n", lists_the_tools(dir), LINES[0]);
if waits {
body.push_str("sleep 30\n");
}
Expand Down Expand Up @@ -454,6 +496,132 @@ fn a_whole_turn_reports_these_events_and_this_outcome() {
assert_eq!(backend.session_id(), Some("sub-conv-1"));
}

/// A stand-in that prints a whole, successful-looking turn but never starts
/// incode's tool server: the shape measured under sbx on Windows, where the
/// CLI could not start any child with piped stdio (#68).
fn a_cli_whose_tool_server_never_starts(dir: &std::path::Path) -> PathBuf {
const LINES: [&str; 3] = [
r#"{"type":"system","subtype":"init","session_id":"no-tools-1","model":"m"}"#,
r#"{"type":"assistant","message":{"content":[{"type":"text","text":"I read the file: all good"}]},"parent_tool_use_id":null}"#,
r#"{"type":"result","is_error":false,"stop_reason":"end_turn","session_id":"no-tools-1","usage":{"input_tokens":1,"output_tokens":1}}"#,
];
#[cfg(windows)]
{
let mut body = String::from("@echo off\r\n");
for line in LINES {
body.push_str(&format!("echo {line}\r\n"));
}
let path = dir.join("toolless-agent.cmd");
std::fs::write(&path, body).expect("write the stand-in agent");
path
}
#[cfg(not(windows))]
{
use std::os::unix::fs::PermissionsExt;
let mut body = String::from("#!/bin/sh\n");
for line in LINES {
body.push_str(&format!("echo '{line}'\n"));
}
let path = dir.join("toolless-agent.sh");
std::fs::write(&path, body).expect("write the stand-in agent");
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755))
.expect("make it runnable");
path
}
}

/// **A TURN THAT NEVER REACHED incode's TOOLS IS NOT A RESULT.** The model
/// had nothing to act with, so a reply like "I read the file" is made up;
/// it was reported as `is_error: false`. Now the turn fails and says why.
#[test]
fn a_turn_whose_tool_server_never_started_fails_and_says_so() {
let dir = tempfile::tempdir().expect("scratch");
let fake = a_cli_whose_tool_server_never_starts(dir.path());
let mut backend =
ClaudeCodeBackend::new(dir.path().to_path_buf()).with_binary(fake.to_string_lossy());
let err = backend
.run_turn("read hello.txt", &mut |_| {})
.expect_err("a turn with none of incode's tools is not a finished one");
assert!(
err.to_string()
.starts_with("Claude Code never started incode's tool server"),
"{err}"
);
}

/// A stand-in whose `init` line says incode's server failed, as Claude Code
/// says it under sbx on Windows (#68), and which then goes on, as the
/// toolless model did until the wall clock: about five seconds later it
/// writes `still-running.txt` beside itself, then a result.
fn a_cli_reporting_the_tool_server_failed(dir: &std::path::Path) -> PathBuf {
const INIT: &str = r#"{"type":"system","subtype":"init","session_id":"failed-1","model":"m","mcp_servers":[{"name":"incode","status":"failed","source":"dynamic"}]}"#;
const RESULT: &str = r#"{"type":"result","is_error":false,"stop_reason":"end_turn","session_id":"failed-1","usage":{"input_tokens":1,"output_tokens":1}}"#;
#[cfg(windows)]
{
let path = dir.join("failed-server-agent.cmd");
std::fs::write(
&path,
format!(
"@echo off\r\necho {INIT}\r\nping -n 6 127.0.0.1 >nul\r\n\
echo x>\"%~dp0still-running.txt\"\r\necho {RESULT}\r\n"
),
)
.expect("write the stand-in agent");
path
}
#[cfg(not(windows))]
{
use std::os::unix::fs::PermissionsExt;
let path = dir.join("failed-server-agent.sh");
std::fs::write(
&path,
format!(
"#!/bin/sh\necho '{INIT}'\nsleep 5\n\
touch \"$(dirname \"$0\")/still-running.txt\"\necho '{RESULT}'\n"
),
)
.expect("write the stand-in agent");
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755))
.expect("make it runnable");
path
}
}

/// **WHEN THE CLI SAYS AT ONCE THAT incode's SERVER FAILED, THE TURN STOPS
/// AT ONCE**, with the reason, and the CLI is not left running. Before, the
/// toolless model ran on until the wall clock (281 s measured).
#[test]
fn a_reported_server_failure_stops_the_turn_at_once() {
let dir = tempfile::tempdir().expect("scratch");
let fake = a_cli_reporting_the_tool_server_failed(dir.path());
let mut backend =
ClaudeCodeBackend::new(dir.path().to_path_buf()).with_binary(fake.to_string_lossy());
let started = std::time::Instant::now();
let err = backend
.run_turn("read hello.txt", &mut |_| {})
.expect_err("a turn whose tool server failed is not a finished one");
assert!(
err.to_string()
.starts_with("Claude Code never started incode's tool server"),
"{err}"
);
assert!(
started.elapsed() < std::time::Duration::from_secs(5),
"stopped only when the stand-in finished: {:?}",
started.elapsed()
);
// Past the moment it would have written its marker: a CLI left running
// would have. This catches a missing kill on Linux and macOS, where
// nothing else ends the process group; on Windows the turn's job object
// ends the CLI on drop as well (`agent/process_tree.rs`), so there the
// marker stays absent either way.
std::thread::sleep(std::time::Duration::from_secs(9));
assert!(
!dir.path().join("still-running.txt").exists(),
"the CLI was left running after its turn was stopped"
);
}

/// A CLI that cannot be started names itself in the error.
#[test]
fn a_cli_that_cannot_start_is_named_in_the_error() {
Expand Down
30 changes: 20 additions & 10 deletions src/agent/claude_code/turn.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ use std::io::BufReader;
use std::process::{Child, Command, Stdio};

use super::command::{servers_for, user_message};
use super::wire::map_line;
use super::{ClaudeCodeBackend, CANCELLED};
use super::wire::{incode_server_failed, map_line};
use super::ClaudeCodeBackend;
use crate::agent::process_tree::Tree;
use crate::agent::stderr_tail::StderrTail;
use crate::agent::{AgentError, AgentEvent, TurnOutcome};
Expand Down Expand Up @@ -61,7 +61,13 @@ pub(super) fn consume(
}
}
if line.trim().is_empty() {
return Vec::new();
return Ok(Vec::new());
}
// Stopped here rather than at the end: a model with no tools runs on
// until it gives up or the wall clock ends it, and its reply is not
// a result either way.
if incode_server_failed(&line) {
return Err(crate::tool_bridge::no_tools("Claude Code"));
}
let events = map_line(&line, &mut outcome);
for event in &events {
Expand All @@ -74,7 +80,7 @@ pub(super) fn consume(
}
}
}
events
Ok(events)
})?;
Ok(outcome)
}
Expand Down Expand Up @@ -161,10 +167,13 @@ impl ClaudeCodeBackend {
&mut self.tools,
&mut self.calls,
);
let listed = bridge.tools_listed();
// Closed as soon as the turn's loop is done: a call that arrives from
// here on is refused rather than left waiting on a loop that is gone.
drop(bridge);
self.finish(outcome, &mut child, &tree, stderr)
let outcome = self.finish(outcome, &mut child, &tree, stderr)?;
crate::tool_bridge::require_tools(listed, "Claude Code")?;
Ok(outcome)
}

/// WHAT WAS ACTUALLY SPAWNED, on the ordinary trace. The flags decide the
Expand Down Expand Up @@ -278,11 +287,12 @@ impl ClaudeCodeBackend {
tree: &Tree,
stderr: StderrTail,
) -> Result<TurnOutcome, AgentError> {
// Whatever happened, the child must not outlive this call. A cancelled
// turn that left the agent running would keep writing files with
// nobody reading its output — and that includes whatever it started.
let cancelled = matches!(outcome, Err(ref e) if e.to_string() == CANCELLED);
if cancelled {
// Whatever happened, the child must not outlive this call. A turn that
// ended in an error (stopped, its tool server failed, its output
// unreadable) and left the agent running would keep writing files
// with nobody reading its output — and that includes whatever it
// started. The codex route does the same (`codex/bridged.rs`).
if outcome.is_err() {
tree.kill(child);
}
let outcome = outcome?;
Expand Down
37 changes: 37 additions & 0 deletions src/agent/claude_code/wire.rs
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,43 @@ pub(super) fn map_line(line: &str, outcome: &mut Option<TurnOutcome>) -> Vec<Age
}
}

/// Whether this is the `init` line and it says incode's tool server failed
/// to start.
///
/// The CLI connects its MCP servers before it starts the session, and the
/// `init` line lists each with a status. Measured (Claude Code 2.1, Windows):
/// `{"name":"incode","status":"connected"}` normally;
/// `{"name":"incode","status":"failed"}` under a write-restricted token that
/// stops the CLI starting any child with piped stdio (#68), after which the
/// model runs with no tools until it gives up. Only `failed` counts: any
/// other status, or no list at all, is left to the end-of-turn check
/// (`tool_bridge::require_tools`).
///
/// Called by `consume` (`turn.rs`) for every line, before [`map_line`].
pub(super) fn incode_server_failed(line: &str) -> bool {
if !line.contains("\"init\"") {
return false;
}
let Ok(value) = serde_json::from_str::<serde_json::Value>(line) else {
return false;
};
if value.get("type").and_then(serde_json::Value::as_str) != Some("system")
|| value.get("subtype").and_then(serde_json::Value::as_str) != Some("init")
{
return false;
}
value
.get("mcp_servers")
.and_then(serde_json::Value::as_array)
.into_iter()
.flatten()
.any(|server| {
server.get("name").and_then(serde_json::Value::as_str)
== Some(crate::tool_bridge::CLIENT_KEY)
&& server.get("status").and_then(serde_json::Value::as_str) == Some("failed")
})
}

/// The session's start, from the `init` line. Every other `system` subtype —
/// the hook events that precede `init`, among others — says nothing.
///
Expand Down
27 changes: 27 additions & 0 deletions src/agent/claude_code/wire_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,33 @@ fn a_cancelled_turn_stops_and_names_itself() {
assert!(seen.is_empty(), "it must stop BEFORE handling more output");
}

/// **Only the init line naming incode's server as `failed` counts**, the
/// measured shape; a connected server, another server failing, an init
/// line without a server list, and any other line do not.
#[test]
fn only_a_failed_incode_server_in_the_init_line_counts() {
use super::wire::incode_server_failed;
let init = |servers: &str| {
format!(r#"{{"type":"system","subtype":"init","session_id":"s","mcp_servers":{servers}}}"#)
};
assert!(incode_server_failed(&init(
r#"[{"name":"incode","status":"failed","source":"dynamic"}]"#
)));
assert!(!incode_server_failed(&init(
r#"[{"name":"incode","status":"connected","source":"dynamic"}]"#
)));
assert!(!incode_server_failed(&init(
r#"[{"name":"other","status":"failed"},{"name":"incode","status":"connected"}]"#
)));
assert!(!incode_server_failed(
r#"{"type":"system","subtype":"init","session_id":"s"}"#
));
assert!(!incode_server_failed(
r#"{"type":"assistant","message":{"content":[{"type":"text","text":"\"init\" incode failed"}]}}"#
));
assert!(!incode_server_failed("not json, but \"init\""));
}

/// Without cancellation the same stream is read normally — the control
/// that proves the test above is measuring the flag and not the fixture.
#[test]
Expand Down
8 changes: 7 additions & 1 deletion src/agent/codex/bridged.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
//! by incode's toolbox over the tool bridge.
//!
//! Wired to: [`super::CodexBackend`]'s `run_turn`, on [`super::CodexTools::Incode`].
//! Tested end to end, on every platform, by `agent/codex/tests.rs`
//! `a_bridged_turn_runs_its_calls_in_incodes_toolbox_and_a_deny_rule_refuses_one`.

use std::io::BufReader;
use std::process::{Command, Stdio};
Expand Down Expand Up @@ -72,7 +74,7 @@ impl CodexBackend {
tools.set_session(session_id);
}
}
events
Ok(events)
},
);
let (outcome, usage) = reading.finish();
Expand Down Expand Up @@ -109,3 +111,7 @@ impl CodexBackend {
})
}
}

#[cfg(test)]
#[path = "bridged_tests.rs"]
mod tests;
Loading
Loading