diff --git a/crates/gaviero-core/src/swarm/pipeline.rs b/crates/gaviero-core/src/swarm/pipeline.rs index 27441e80..6f6ac61e 100644 --- a/crates/gaviero-core/src/swarm/pipeline.rs +++ b/crates/gaviero-core/src/swarm/pipeline.rs @@ -1242,7 +1242,6 @@ pub async fn execute( unit }; - let sem = semaphore.clone(); let root = config.workspace_root.clone(); let mem = memory.clone(); let ns: Vec = unit @@ -1277,8 +1276,10 @@ pub async fn execute( root.clone() }; + let permit = + claim_slot(&semaphore, observer, unit_id, &unit.description).await?; handles.push(AbortOnDrop(tokio::spawn(async move { - let _permit = sem.acquire().await.unwrap(); + let _permit = permit; invalidate_stale_sources(&mem, &unit, &root).await; @@ -2067,7 +2068,6 @@ pub async fn execute( AbortOnDrop>, )> = Vec::new(); for (agent_id, unit, agent_root, branch_override) in prepared { - let sem = semaphore.clone(); let mem = memory.clone(); let ns: Vec = unit .read_namespaces @@ -2091,11 +2091,13 @@ pub async fn execute( let in_worktree = worktree_mgr.is_some(); let override_branch_name = branch_override.as_ref().map(|ov| ov.branch.clone()); + let permit = + claim_slot(&semaphore, observer, &agent_id, &unit.description).await?; handles.push(( agent_id.clone(), unit.clone(), AbortOnDrop(tokio::spawn(async move { - let _permit = sem.acquire().await.unwrap(); + let _permit = permit; let write_gate = Arc::new(Mutex::new(WriteGatePipeline::new( WriteMode::AutoAccept, Box::new(NoopWriteGateObserver), @@ -4328,6 +4330,25 @@ fn warn_unreported_budget_costs(plan: &CompiledPlan, observer: &dyn SwarmObserve observer.on_phase_changed(&msg); } +/// Take a `max_parallel` slot for a unit about to be spawned, then report it +/// running. The dispatcher does this, not the task: the observer cannot move +/// into the task, and "queued" must mean "waiting for a slot". The permit +/// goes into the task, so the dispatcher never holds one while it waits. +async fn claim_slot( + semaphore: &Arc, + observer: &dyn SwarmObserver, + unit_id: &str, + detail: &str, +) -> Result { + let permit = semaphore + .clone() + .acquire_owned() + .await + .context("swarm dispatch semaphore closed")?; + observer.on_agent_state_changed(unit_id, &AgentStatus::Running, detail); + Ok(permit) +} + /// Everything a dynamically materialised unit needs to run, borrowed from /// `execute`. Dynamic units — runtime `for` clones and `pattern map_reduce` /// workers — are not plan nodes, so they run outside the tier loop, in the @@ -4407,6 +4428,14 @@ where } } + // Every unit is listed as queued before the first one takes a slot. + for unit in &units { + if !(resume_skip && exec_state.status(&unit.id) == NodeStatus::Completed) { + ctx.observer + .on_agent_state_changed(&unit.id, &AgentStatus::Pending, "queued"); + } + } + let mut handles: Vec<(String, WorkUnit, AbortOnDrop)> = Vec::new(); for unit in units { if let Some(reason) = ctx.limits.exhausted(spent_usd(all_manifests)) { @@ -4425,11 +4454,9 @@ where unit.id ); } + let permit = claim_slot(&ctx.semaphore, ctx.observer, &unit.id, &unit.description).await?; exec_state.set_status(&unit.id, NodeStatus::Running); - ctx.observer - .on_agent_state_changed(&unit.id, &AgentStatus::Pending, "queued"); - let sem = ctx.semaphore.clone(); let root = ctx.config.workspace_root.clone(); let mem = ctx.memory.clone(); let ns: Vec = unit @@ -4452,7 +4479,7 @@ where let skill_catalog = ctx.config.skill_catalog.clone(); let task_unit = unit.clone(); let handle = AbortOnDrop(tokio::spawn(async move { - let _permit = sem.acquire().await.unwrap(); + let _permit = permit; invalidate_stale_sources(&mem, &task_unit, &root).await; let write_gate = Arc::new(Mutex::new(WriteGatePipeline::new( WriteMode::AutoAccept, @@ -4615,6 +4642,12 @@ where .collect(), None => clones.clone(), }; + // The template's row is the barrier: it runs while its clones do. + ctx.observer.on_agent_state_changed( + &op.unit_id, + &AgentStatus::Running, + &format!("{} item(s)", clones.len()), + ); run_dynamic_units( dispatched, pass.is_none(), diff --git a/crates/gaviero-core/tests/swarm_parallel_dispatch.rs b/crates/gaviero-core/tests/swarm_parallel_dispatch.rs index dbd4e24b..9dcc24ff 100644 --- a/crates/gaviero-core/tests/swarm_parallel_dispatch.rs +++ b/crates/gaviero-core/tests/swarm_parallel_dispatch.rs @@ -88,6 +88,44 @@ impl SwarmObserver for QuietObserver { fn on_completed(&self, _result: &gaviero_core::swarm::models::SwarmResult) {} } +/// Records every agent status change as `(id, "pending" | "running" | …)`. +#[derive(Default)] +struct RecordingObserver { + states: std::sync::Mutex>, +} + +impl RecordingObserver { + /// The statuses `id` went through, in order. + fn of(&self, id: &str) -> Vec<&'static str> { + self.states + .lock() + .unwrap() + .iter() + .filter(|(agent, _)| agent == id) + .map(|(_, status)| *status) + .collect() + } +} + +impl SwarmObserver for RecordingObserver { + fn on_phase_changed(&self, _phase: &str) {} + fn on_agent_state_changed(&self, work_unit_id: &str, status: &AgentStatus, _detail: &str) { + let label = match status { + AgentStatus::Pending => "pending", + AgentStatus::Running => "running", + AgentStatus::Completed => "completed", + AgentStatus::Failed(_) => "failed", + }; + self.states + .lock() + .unwrap() + .push((work_unit_id.to_string(), label)); + } + fn on_tier_started(&self, _current: usize, _total: usize) {} + fn on_merge_conflict(&self, _branch: &str, _files: &[String]) {} + fn on_completed(&self, _result: &gaviero_core::swarm::models::SwarmResult) {} +} + struct NoopAcpObserver; impl AcpObserver for NoopAcpObserver { fn on_stream_chunk(&self, _text: &str) {} @@ -365,3 +403,96 @@ async fn a_loop_whose_first_pass_ran_in_parallel_iterates() { count_containing(&prompts, "MARK-YYY") ); } + +/// Each run must be reported: a unit that finished without a `running` just +/// before it shows as queued for its whole run in the SWARM panel. +fn assert_each_run_reported(statuses: &[&str], id: &str, runs: usize) { + let running = statuses.iter().filter(|s| **s == "running").count(); + assert_eq!(running, runs, "{id} reported running {running} times: {statuses:?}"); + for (i, status) in statuses.iter().enumerate() { + if *status == "completed" { + assert!( + i > 0 && statuses[i - 1] == "running", + "{id} completed without running first: {statuses:?}" + ); + } + } +} + +#[tokio::test] +async fn parallel_tier_agents_are_reported_running_even_when_waiting_for_a_slot() { + let tmp = tempfile::tempdir().expect("tempdir"); + let workspace = tmp.path(); + std::fs::create_dir_all(workspace.join("out")).unwrap(); + let server = MockServer::start().await; + answer(&server, "MARK-AAA", "out/a.md").await; + answer(&server, "MARK-BBB", "out/b.md").await; + answer(&server, "MARK-DDD", "out/d.md").await; + // Three units in one tier, two slots: one of them must wait. + let mut plan = CompiledPlan::from_work_units( + vec![ + unit("a", &["out/a.md"], "Write. MARK-AAA", &[]), + unit("b", &["out/b.md"], "Write. MARK-BBB", &[]), + unit("d", &["out/d.md"], "Write. MARK-DDD", &[]), + ], + Some(2), + ); + plan.execution_mode = ExecutionMode::Document; + let observer = RecordingObserver::default(); + + execute( + &plan, + &config_for(workspace, &server.uri(), 2), + None, + None, + &observer, + make_obs, + ) + .await + .expect("run succeeds"); + + for id in ["a", "b", "d"] { + assert_eq!( + observer.of(id), + vec!["pending", "running", "completed"], + "{id}: queued, then running, then completed" + ); + } +} + +#[tokio::test] +async fn parallel_loop_pass_agents_are_reported_running_every_pass() { + let tmp = tempfile::tempdir().expect("tempdir"); + let workspace = tmp.path(); + std::fs::create_dir_all(workspace.join("out")).unwrap(); + let server = MockServer::start().await; + answer(&server, "MARK-XXX", "out/x.md").await; + answer(&server, "MARK-YYY", "out/y.md").await; + let mut plan = CompiledPlan::from_work_units( + vec![ + unit("x", &["out/x.md"], "Iteration {{ITER}}. MARK-XXX", &[]), + unit("y", &["out/y.md"], "Iteration {{ITER}}. MARK-YYY", &[]), + ], + Some(2), + ); + plan.execution_mode = ExecutionMode::Document; + plan.loop_configs = vec![command_loop(&["x", "y"], 2)]; + let observer = RecordingObserver::default(); + + let _ = execute( + &plan, + &config_for(workspace, &server.uri(), 2), + None, + None, + &observer, + make_obs, + ) + .await; + + let prompts = prompts_seen(&server).await; + for (id, marker) in [("x", "MARK-XXX"), ("y", "MARK-YYY")] { + let runs = count_containing(&prompts, marker); + assert!(runs >= 2, "{id} ran {runs} times"); + assert_each_run_reported(&observer.of(id), id, runs); + } +} diff --git a/crates/gaviero-core/tests/swarm_runtime_for.rs b/crates/gaviero-core/tests/swarm_runtime_for.rs index 4554dbe3..e9fa2756 100644 --- a/crates/gaviero-core/tests/swarm_runtime_for.rs +++ b/crates/gaviero-core/tests/swarm_runtime_for.rs @@ -113,6 +113,44 @@ impl SwarmObserver for QuietObserver { fn on_completed(&self, _result: &SwarmResult) {} } +/// Records every agent status change as `(id, "pending" | "running" | …)`. +#[derive(Default)] +struct RecordingObserver { + states: std::sync::Mutex>, +} + +impl RecordingObserver { + /// The statuses `id` went through, in order. + fn of(&self, id: &str) -> Vec<&'static str> { + self.states + .lock() + .unwrap() + .iter() + .filter(|(agent, _)| agent == id) + .map(|(_, status)| *status) + .collect() + } +} + +impl SwarmObserver for RecordingObserver { + fn on_phase_changed(&self, _phase: &str) {} + fn on_agent_state_changed(&self, work_unit_id: &str, status: &AgentStatus, _detail: &str) { + let label = match status { + AgentStatus::Pending => "pending", + AgentStatus::Running => "running", + AgentStatus::Completed => "completed", + AgentStatus::Failed(_) => "failed", + }; + self.states + .lock() + .unwrap() + .push((work_unit_id.to_string(), label)); + } + fn on_tier_started(&self, _current: usize, _total: usize) {} + fn on_merge_conflict(&self, _branch: &str, _files: &[String]) {} + fn on_completed(&self, _result: &SwarmResult) {} +} + struct NoopAcpObserver; impl AcpObserver for NoopAcpObserver { fn on_stream_chunk(&self, _text: &str) {} @@ -310,6 +348,39 @@ async fn top_level_runtime_for_runs_every_item_then_the_dependent() { } } +#[tokio::test] +async fn runtime_for_clones_and_their_template_are_reported_running() { + let tmp = workspace_with(&[("out/items.json", THREE_ITEMS)]); + let server = standard_server().await; + let plan = top_level_plan(16); + let observer = RecordingObserver::default(); + + // Three clones, two slots: one clone waits for a slot. + execute( + &plan, + &config_for(tmp.path(), &server.uri(), 2), + None, + None, + &observer, + make_obs, + ) + .await + .expect("run succeeds"); + + for id in ["worker@a", "worker@b", "worker@c"] { + assert_eq!( + observer.of(id), + vec!["pending", "running", "completed"], + "{id}: queued, then running, then completed" + ); + } + assert_eq!( + observer.of("worker"), + vec!["pending", "running", "completed"], + "the template runs while its clones do" + ); +} + #[tokio::test] async fn resume_skips_completed_items() { let tmp = workspace_with(&[("out/items.json", THREE_ITEMS)]);