Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 41 additions & 8 deletions crates/gaviero-core/src/swarm/pipeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1242,7 +1242,6 @@ pub async fn execute(
unit
};

let sem = semaphore.clone();
let root = config.workspace_root.clone();
let mem = memory.clone();
let ns: Vec<String> = unit
Expand Down Expand Up @@ -1277,8 +1276,10 @@ pub async fn execute(
root.clone()
};

let permit =
claim_slot(&semaphore, observer, unit_id, &unit.description).await?;
handles.push(AbortOnDrop(tokio::spawn(async move {
let _permit = sem.acquire().await.unwrap();
let _permit = permit;

invalidate_stale_sources(&mem, &unit, &root).await;

Expand Down Expand Up @@ -2067,7 +2068,6 @@ pub async fn execute(
AbortOnDrop<anyhow::Result<AgentManifest>>,
)> = Vec::new();
for (agent_id, unit, agent_root, branch_override) in prepared {
let sem = semaphore.clone();
let mem = memory.clone();
let ns: Vec<String> = unit
.read_namespaces
Expand All @@ -2091,11 +2091,13 @@ pub async fn execute(
let in_worktree = worktree_mgr.is_some();
let override_branch_name = branch_override.as_ref().map(|ov| ov.branch.clone());

let permit =
claim_slot(&semaphore, observer, &agent_id, &unit.description).await?;
handles.push((
agent_id.clone(),
unit.clone(),
AbortOnDrop(tokio::spawn(async move {
let _permit = sem.acquire().await.unwrap();
let _permit = permit;
let write_gate = Arc::new(Mutex::new(WriteGatePipeline::new(
WriteMode::AutoAccept,
Box::new(NoopWriteGateObserver),
Expand Down Expand Up @@ -4328,6 +4330,25 @@ fn warn_unreported_budget_costs(plan: &CompiledPlan, observer: &dyn SwarmObserve
observer.on_phase_changed(&msg);
}

/// Take a `max_parallel` slot for a unit about to be spawned, then report it
/// running. The dispatcher does this, not the task: the observer cannot move
/// into the task, and "queued" must mean "waiting for a slot". The permit
/// goes into the task, so the dispatcher never holds one while it waits.
async fn claim_slot(
semaphore: &Arc<Semaphore>,
observer: &dyn SwarmObserver,
unit_id: &str,
detail: &str,
) -> Result<tokio::sync::OwnedSemaphorePermit> {
let permit = semaphore
.clone()
.acquire_owned()
.await
.context("swarm dispatch semaphore closed")?;
observer.on_agent_state_changed(unit_id, &AgentStatus::Running, detail);
Ok(permit)
}

/// Everything a dynamically materialised unit needs to run, borrowed from
/// `execute`. Dynamic units — runtime `for` clones and `pattern map_reduce`
/// workers — are not plan nodes, so they run outside the tier loop, in the
Expand Down Expand Up @@ -4407,6 +4428,14 @@ where
}
}

// Every unit is listed as queued before the first one takes a slot.
for unit in &units {
if !(resume_skip && exec_state.status(&unit.id) == NodeStatus::Completed) {
ctx.observer
.on_agent_state_changed(&unit.id, &AgentStatus::Pending, "queued");
}
}

let mut handles: Vec<(String, WorkUnit, AbortOnDrop<AgentManifest>)> = Vec::new();
for unit in units {
if let Some(reason) = ctx.limits.exhausted(spent_usd(all_manifests)) {
Expand All @@ -4425,11 +4454,9 @@ where
unit.id
);
}
let permit = claim_slot(&ctx.semaphore, ctx.observer, &unit.id, &unit.description).await?;
exec_state.set_status(&unit.id, NodeStatus::Running);
ctx.observer
.on_agent_state_changed(&unit.id, &AgentStatus::Pending, "queued");

let sem = ctx.semaphore.clone();
let root = ctx.config.workspace_root.clone();
let mem = ctx.memory.clone();
let ns: Vec<String> = unit
Expand All @@ -4452,7 +4479,7 @@ where
let skill_catalog = ctx.config.skill_catalog.clone();
let task_unit = unit.clone();
let handle = AbortOnDrop(tokio::spawn(async move {
let _permit = sem.acquire().await.unwrap();
let _permit = permit;
invalidate_stale_sources(&mem, &task_unit, &root).await;
let write_gate = Arc::new(Mutex::new(WriteGatePipeline::new(
WriteMode::AutoAccept,
Expand Down Expand Up @@ -4615,6 +4642,12 @@ where
.collect(),
None => clones.clone(),
};
// The template's row is the barrier: it runs while its clones do.
ctx.observer.on_agent_state_changed(
&op.unit_id,
&AgentStatus::Running,
&format!("{} item(s)", clones.len()),
);
run_dynamic_units(
dispatched,
pass.is_none(),
Expand Down
131 changes: 131 additions & 0 deletions crates/gaviero-core/tests/swarm_parallel_dispatch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,44 @@ impl SwarmObserver for QuietObserver {
fn on_completed(&self, _result: &gaviero_core::swarm::models::SwarmResult) {}
}

/// Records every agent status change as `(id, "pending" | "running" | …)`.
#[derive(Default)]
struct RecordingObserver {
states: std::sync::Mutex<Vec<(String, &'static str)>>,
}

impl RecordingObserver {
/// The statuses `id` went through, in order.
fn of(&self, id: &str) -> Vec<&'static str> {
self.states
.lock()
.unwrap()
.iter()
.filter(|(agent, _)| agent == id)
.map(|(_, status)| *status)
.collect()
}
}

impl SwarmObserver for RecordingObserver {
fn on_phase_changed(&self, _phase: &str) {}
fn on_agent_state_changed(&self, work_unit_id: &str, status: &AgentStatus, _detail: &str) {
let label = match status {
AgentStatus::Pending => "pending",
AgentStatus::Running => "running",
AgentStatus::Completed => "completed",
AgentStatus::Failed(_) => "failed",
};
self.states
.lock()
.unwrap()
.push((work_unit_id.to_string(), label));
}
fn on_tier_started(&self, _current: usize, _total: usize) {}
fn on_merge_conflict(&self, _branch: &str, _files: &[String]) {}
fn on_completed(&self, _result: &gaviero_core::swarm::models::SwarmResult) {}
}

struct NoopAcpObserver;
impl AcpObserver for NoopAcpObserver {
fn on_stream_chunk(&self, _text: &str) {}
Expand Down Expand Up @@ -365,3 +403,96 @@ async fn a_loop_whose_first_pass_ran_in_parallel_iterates() {
count_containing(&prompts, "MARK-YYY")
);
}

/// Each run must be reported: a unit that finished without a `running` just
/// before it shows as queued for its whole run in the SWARM panel.
fn assert_each_run_reported(statuses: &[&str], id: &str, runs: usize) {
let running = statuses.iter().filter(|s| **s == "running").count();
assert_eq!(running, runs, "{id} reported running {running} times: {statuses:?}");
for (i, status) in statuses.iter().enumerate() {
if *status == "completed" {
assert!(
i > 0 && statuses[i - 1] == "running",
"{id} completed without running first: {statuses:?}"
);
}
}
}

#[tokio::test]
async fn parallel_tier_agents_are_reported_running_even_when_waiting_for_a_slot() {
let tmp = tempfile::tempdir().expect("tempdir");
let workspace = tmp.path();
std::fs::create_dir_all(workspace.join("out")).unwrap();
let server = MockServer::start().await;
answer(&server, "MARK-AAA", "out/a.md").await;
answer(&server, "MARK-BBB", "out/b.md").await;
answer(&server, "MARK-DDD", "out/d.md").await;
// Three units in one tier, two slots: one of them must wait.
let mut plan = CompiledPlan::from_work_units(
vec![
unit("a", &["out/a.md"], "Write. MARK-AAA", &[]),
unit("b", &["out/b.md"], "Write. MARK-BBB", &[]),
unit("d", &["out/d.md"], "Write. MARK-DDD", &[]),
],
Some(2),
);
plan.execution_mode = ExecutionMode::Document;
let observer = RecordingObserver::default();

execute(
&plan,
&config_for(workspace, &server.uri(), 2),
None,
None,
&observer,
make_obs,
)
.await
.expect("run succeeds");

for id in ["a", "b", "d"] {
assert_eq!(
observer.of(id),
vec!["pending", "running", "completed"],
"{id}: queued, then running, then completed"
);
}
}

#[tokio::test]
async fn parallel_loop_pass_agents_are_reported_running_every_pass() {
let tmp = tempfile::tempdir().expect("tempdir");
let workspace = tmp.path();
std::fs::create_dir_all(workspace.join("out")).unwrap();
let server = MockServer::start().await;
answer(&server, "MARK-XXX", "out/x.md").await;
answer(&server, "MARK-YYY", "out/y.md").await;
let mut plan = CompiledPlan::from_work_units(
vec![
unit("x", &["out/x.md"], "Iteration {{ITER}}. MARK-XXX", &[]),
unit("y", &["out/y.md"], "Iteration {{ITER}}. MARK-YYY", &[]),
],
Some(2),
);
plan.execution_mode = ExecutionMode::Document;
plan.loop_configs = vec![command_loop(&["x", "y"], 2)];
let observer = RecordingObserver::default();

let _ = execute(
&plan,
&config_for(workspace, &server.uri(), 2),
None,
None,
&observer,
make_obs,
)
.await;

let prompts = prompts_seen(&server).await;
for (id, marker) in [("x", "MARK-XXX"), ("y", "MARK-YYY")] {
let runs = count_containing(&prompts, marker);
assert!(runs >= 2, "{id} ran {runs} times");
assert_each_run_reported(&observer.of(id), id, runs);
}
}
71 changes: 71 additions & 0 deletions crates/gaviero-core/tests/swarm_runtime_for.rs
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,44 @@ impl SwarmObserver for QuietObserver {
fn on_completed(&self, _result: &SwarmResult) {}
}

/// Records every agent status change as `(id, "pending" | "running" | …)`.
#[derive(Default)]
struct RecordingObserver {
states: std::sync::Mutex<Vec<(String, &'static str)>>,
}

impl RecordingObserver {
/// The statuses `id` went through, in order.
fn of(&self, id: &str) -> Vec<&'static str> {
self.states
.lock()
.unwrap()
.iter()
.filter(|(agent, _)| agent == id)
.map(|(_, status)| *status)
.collect()
}
}

impl SwarmObserver for RecordingObserver {
fn on_phase_changed(&self, _phase: &str) {}
fn on_agent_state_changed(&self, work_unit_id: &str, status: &AgentStatus, _detail: &str) {
let label = match status {
AgentStatus::Pending => "pending",
AgentStatus::Running => "running",
AgentStatus::Completed => "completed",
AgentStatus::Failed(_) => "failed",
};
self.states
.lock()
.unwrap()
.push((work_unit_id.to_string(), label));
}
fn on_tier_started(&self, _current: usize, _total: usize) {}
fn on_merge_conflict(&self, _branch: &str, _files: &[String]) {}
fn on_completed(&self, _result: &SwarmResult) {}
}

struct NoopAcpObserver;
impl AcpObserver for NoopAcpObserver {
fn on_stream_chunk(&self, _text: &str) {}
Expand Down Expand Up @@ -310,6 +348,39 @@ async fn top_level_runtime_for_runs_every_item_then_the_dependent() {
}
}

#[tokio::test]
async fn runtime_for_clones_and_their_template_are_reported_running() {
let tmp = workspace_with(&[("out/items.json", THREE_ITEMS)]);
let server = standard_server().await;
let plan = top_level_plan(16);
let observer = RecordingObserver::default();

// Three clones, two slots: one clone waits for a slot.
execute(
&plan,
&config_for(tmp.path(), &server.uri(), 2),
None,
None,
&observer,
make_obs,
)
.await
.expect("run succeeds");

for id in ["worker@a", "worker@b", "worker@c"] {
assert_eq!(
observer.of(id),
vec!["pending", "running", "completed"],
"{id}: queued, then running, then completed"
);
}
assert_eq!(
observer.of("worker"),
vec!["pending", "running", "completed"],
"the template runs while its clones do"
);
}

#[tokio::test]
async fn resume_skips_completed_items() {
let tmp = workspace_with(&[("out/items.json", THREE_ITEMS)]);
Expand Down
Loading