From f2f4f4765b058cb87be8e95fbe21a095b0482e1d Mon Sep 17 00:00:00 2001 From: rdfitted Date: Tue, 1 Sep 2026 20:57:02 -0400 Subject: [PATCH 1/2] feat(#260,#262,#263,#264,#265,#266): make work-graph completion a declared fact (v0.47.0) Closes six issues across two subsystems: five that form one causal chain behind "finished sessions render every node pending", and one independent PTY inject defect. Work-graph completion truth (child of epic #209) - #266 `?source=auto` now consults session state. `state/work-graph.json` is written once at PlanReady and never removed, so the old `else` branch was unreachable and a finished session always got the all-pending live skeleton. `SessionState::is_terminal()` (Completed|Closed|Failed) makes a terminal session with an archive serve the archive; `Closing` stays live. A terminal session with no archive serves live plus a SourceUnreadable omission, never a 404. - #265 splits the queue projection instead of changing it. The `Pending` reset in `project_queue_statuses` is load-bearing: it feeds `promote_initial_ready_nodes`, and `workers.rs:218` uses the same projection for claimability and conflict advice. The operational wrapper is unchanged; a new view projection preserves an unbacked terminal status, records a typed omission, skips readiness promotion, and returns provenance. Archive derivation now reports the anchor node as `observed` and same-binding fan-out as `inferred`. - #264 records the principal binding at spawn and resolves through it. `HierarchyNode` gains `#[serde(default)] principal`; the write lives in the two `update_hierarchy` sinks and the value is derived at the one spawn site that knows a plan task. The suffix match is deleted; an agent with no recorded binding yields `resolution_incomplete` and no guess. - #263 adds the first-class fact. New `WorkNodeCompleted` event and an append-only ledger at `state/work-graph-completions.jsonl` with provenance `queue_finalize | heartbeat | evaluator_verdict`, an optional `completed_nodes` heartbeat field (unknown ids are 400, all-or-nothing), and ledger reads on both the live projection and archive derivation. - #262 makes PlanReady omission accumulation idempotent across all four post-reconcile append sites, not just the dangling-dependency one. PTY inject - #260 stops confirming a submit against an already-streaming receiver. A bounded pre-write baseline is sampled in the handler inside the existing transaction lock; a would-be positive is downgraded to `null` with basis `busy-receiver-indeterminate`. The retry key stays strictly `Some(false)`, so a busy receiver never triggers a retry. `pty/session.rs` and `session_stub.rs` are untouched. Verification - cargo test 885 passed, 0 failed; clippy 0 errors - codegraph reachability gate: 147 modules, 0 findings, 0 certain - npm run check 0 errors/0 warnings; npm test 34 files, 192 tests - Mutation-tested: each new branch was broken and the corresponding test observed failing, then restored. #265's projection split was mutated in both directions independently. Co-Authored-By: Claude Opus 5 --- docs/pty-submit-sweep.md | 7 +- package.json | 2 +- src-tauri/Cargo.lock | 2 +- src-tauri/Cargo.toml | 2 +- src-tauri/src/actions/coordination.rs | 138 +++- src-tauri/src/coordination/queue_manager.rs | 321 +++++--- src-tauri/src/coordination/state.rs | 111 ++- src-tauri/src/domain/event.rs | 1 + src-tauri/src/events/bus.rs | 24 +- src-tauri/src/http/handlers/evaluator.rs | 194 ++++- src-tauri/src/http/handlers/heartbeats.rs | 122 ++- src-tauri/src/http/handlers/inject.rs | 280 ++++++- src-tauri/src/http/handlers/work_graph.rs | 257 ++++++- src-tauri/src/http/tests_wg_api.rs | 201 ++++- src-tauri/src/http/tests_wg_queue.rs | 708 +++++++++++++----- src-tauri/src/http/tests_wg_retro.rs | 1 + src-tauri/src/http/tests_wg_roles.rs | 1 + src-tauri/src/http/tests_wg_runtime.rs | 538 ++++++++++--- src-tauri/src/http/tests_wg_verifier.rs | 1 + .../src/orchestrator/work_graph/archive.rs | 220 +++--- .../work_graph/completion_ledger.rs | 175 +++++ src-tauri/src/orchestrator/work_graph/mod.rs | 6 +- .../src/orchestrator/work_graph/plan_parse.rs | 29 +- .../src/orchestrator/work_graph/runtime.rs | 374 +++++---- src-tauri/src/session/controller.rs | 679 ++++++++++++----- src-tauri/src/templates/mod.rs | 75 +- src-tauri/tauri.conf.json | 2 +- .../components/workgraph/NodeInspector.svelte | 103 ++- .../workgraph/NodeInspector.svelte.test.ts | 29 + .../workgraph/ProgressHeader.svelte | 19 + .../workgraph/ProgressHeader.svelte.test.ts | 8 + .../components/workgraph/WorkGraphView.svelte | 103 ++- .../workgraph/WorkGraphView.svelte.test.ts | 117 ++- src/lib/workgraph/graphUtils.test.ts | 31 + src/lib/workgraph/graphUtils.ts | 31 +- src/lib/workgraph/types.ts | 11 + 36 files changed, 3875 insertions(+), 1048 deletions(-) create mode 100644 src-tauri/src/orchestrator/work_graph/completion_ledger.rs diff --git a/docs/pty-submit-sweep.md b/docs/pty-submit-sweep.md index a066ee86..bd305c37 100644 --- a/docs/pty-submit-sweep.md +++ b/docs/pty-submit-sweep.md @@ -83,9 +83,10 @@ The inject response reports measured facts, not request echoes: after an Enter write, over a bounded 1,500 ms window per attempt: `true` means sustained ring activity consistent with the composer accepting Enter and starting a turn, `false` means the Enter produced no observable ring reaction at all, and `null` means unknown, - unobservable, or `"submit": false`. An agent that was already streaming output can - produce a false positive; the field never upgrades an ambiguous buffer observation to a - sweep PASS. + unobservable, or `"submit": false`. The handler also samples a bounded pre-write activity + baseline. When that baseline shows the receiver was already streaming, a would-be positive is + downgraded to `null` with basis `busy-receiver-indeterminate`; the field never upgrades an + ambiguous buffer observation to a sweep PASS. - The retry is keyed strictly on the first `false` verdict and is capped at one extra CR. Its observation baseline is captured after that CR so local echo cannot become false receiver evidence. The response reports the retry observation as the final verdict; two diff --git a/package.json b/package.json index 4cb601d4..cc389d84 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hive-manager", - "version": "0.46.2", + "version": "0.47.0", "description": "Multi-agent orchestration and monitoring for Claude Code workflows", "type": "module", "scripts": { diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 4f9201e7..d23879c1 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -1683,7 +1683,7 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hive-manager" -version = "0.46.2" +version = "0.47.0" dependencies = [ "async-trait", "axum", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index e16cfebd..8fdb05b6 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hive-manager" -version = "0.46.2" +version = "0.47.0" description = "Multi-agent orchestration and monitoring for Claude Code workflows" authors = ["RDuff"] edition = "2021" diff --git a/src-tauri/src/actions/coordination.rs b/src-tauri/src/actions/coordination.rs index c84cbd97..ab652aca 100644 --- a/src-tauri/src/actions/coordination.rs +++ b/src-tauri/src/actions/coordination.rs @@ -59,6 +59,8 @@ pub struct PlanTask { pub description: String, pub status: String, pub assignee: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub assignee_label: Option, pub priority: Option, #[serde(default)] pub depends_on: Vec, @@ -74,6 +76,14 @@ pub struct PlanTask { pub(crate) explicit_id: bool, #[serde(skip)] pub(crate) checkbox_source: bool, + /// Whether the assignee was normalized from a supported principal token. + /// Unknown values remain schedulable and are surfaced as PlanReady warnings. + #[serde(skip, default = "default_binding_recognized")] + pub(crate) assignee_recognized: bool, +} + +fn default_binding_recognized() -> bool { + true } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] @@ -870,7 +880,7 @@ fn parse_task_line_with_diagnostics( let (title, priority) = extract_priority(&rest); let (title, explicit_id) = extract_explicit_task_id(&title); let has_explicit_id = explicit_id.is_some(); - let (title, assignee) = extract_assignee(&title); + let (title, assignee, assignee_label, assignee_recognized) = extract_assignee(&title); ( Some(PlanTask { @@ -879,6 +889,7 @@ fn parse_task_line_with_diagnostics( description: String::new(), status: status.to_string(), assignee, + assignee_label, priority, depends_on: metadata.depends_on, inputs: metadata.inputs, @@ -886,6 +897,7 @@ fn parse_task_line_with_diagnostics( acceptance: metadata.acceptance, explicit_id: has_explicit_id, checkbox_source, + assignee_recognized, }), metadata_error, ) @@ -986,14 +998,62 @@ fn extract_priority(text: &str) -> (String, Option) { (text.to_string(), None) } -fn extract_assignee(text: &str) -> (String, Option) { +fn extract_assignee( + text: &str, +) -> (String, Option, Option, bool) { for separator in ["->", "\u{2192}"] { if let Some((title, assignee)) = text.split_once(separator) { - return (title.to_string(), Some(assignee.trim().to_string())); + let assignee = assignee.trim(); + if assignee.is_empty() { + return (title.to_string(), None, None, false); + } + let first_token = assignee.split_whitespace().next().unwrap_or(assignee); + let normalized = normalize_principal_token(first_token); + if let Some(principal) = normalized { + let label = assignee[first_token.len()..].trim(); + return ( + title.to_string(), + Some(principal), + (!label.is_empty()).then(|| label.to_string()), + true, + ); + } + return ( + title.to_string(), + Some(assignee.to_string()), + None, + false, + ); } } - (text.to_string(), None) + (text.to_string(), None, None, true) +} + +fn normalize_principal_token(token: &str) -> Option { + let mut characters = token.chars(); + if matches!(characters.next(), Some('P' | 'p')) { + let digits = characters.as_str(); + if !digits.is_empty() && digits.chars().all(|character| character.is_ascii_digit()) { + return Some(format!("P{digits}")); + } + } + if token.eq_ignore_ascii_case("queen") { + return Some("Queen".to_string()); + } + if token.eq_ignore_ascii_case("operator") { + return Some("Operator".to_string()); + } + if let Some(index) = token + .to_ascii_lowercase() + .strip_prefix("worker-") + .filter(|index| { + !index.is_empty() && index.chars().all(|character| character.is_ascii_digit()) + }) + { + return Some(format!("worker-{index}")); + } + None } #[cfg(test)] @@ -1077,11 +1137,69 @@ mod tests { fn extract_assignee_supports_ascii_and_unicode_arrows() { assert_eq!( extract_assignee("Fix launch -> worker-8"), - ("Fix launch ".to_string(), Some("worker-8".to_string())) + ( + "Fix launch ".to_string(), + Some("worker-8".to_string()), + None, + true, + ) ); assert_eq!( extract_assignee("Fix launch \u{2192} worker-9"), - ("Fix launch ".to_string(), Some("worker-9".to_string())) + ( + "Fix launch ".to_string(), + Some("worker-9".to_string()), + None, + true, + ) + ); + } + + #[test] + fn assignee_normalizes_principal_and_preserves_display_label() { + let mut counter = 0; + let task = parse_task_line( + "- [ ] [P1] T1: Implement completion truth -> P1 WS-A #126", + &mut counter, + ) + .expect("principal-bound task"); + + assert_eq!(task.assignee.as_deref(), Some("P1")); + assert_eq!(task.assignee_label.as_deref(), Some("WS-A #126")); + assert!(task.assignee_recognized); + + let plan = super::SessionPlan { + title: "Principal plan".to_string(), + summary: String::new(), + tasks: vec![task], + generated_at: String::new(), + raw_content: String::new(), + }; + let graph = crate::orchestrator::work_graph::plan_parse::task_graph_from_plan(&plan); + assert_eq!( + graph.nodes[0].binding, + crate::orchestrator::work_graph::BindingRef::Role("P1".to_string()) + ); + } + + #[test] + fn unrecognized_assignee_is_preserved_as_plan_ready_warning() { + let plan = super::parse_plan_markdown_checked( + "# Plan\n\n## Tasks\n- [ ] T1: Custom lane -> Planner 1\n", + ) + .expect("unrecognized bindings are warnings, not parse errors"); + assert_eq!(plan.tasks[0].assignee.as_deref(), Some("Planner 1")); + assert!(!plan.tasks[0].assignee_recognized); + + let graph = crate::orchestrator::work_graph::plan_parse::task_graph_from_plan(&plan); + assert_eq!( + graph.nodes[0].binding, + crate::orchestrator::work_graph::BindingRef::Role("Planner 1".to_string()) + ); + assert_eq!(graph.omissions.len(), 1); + assert_eq!( + graph.omissions[0].reason, + crate::orchestrator::work_graph::WorkGraphOmissionReason::ResolutionIncomplete ); } @@ -1095,6 +1213,7 @@ mod tests { assert_eq!(task.title, "Fix launch regression"); assert_eq!(task.priority.as_deref(), Some("high")); assert_eq!(task.assignee.as_deref(), Some("worker-8")); + assert_eq!(task.assignee_label, None); } #[test] @@ -1109,7 +1228,12 @@ mod tests { ); assert_eq!( extract_assignee("Title -> worker-1 -> trailing"), - ("Title ".to_string(), Some("worker-1 -> trailing".to_string())) + ( + "Title ".to_string(), + Some("worker-1".to_string()), + Some("-> trailing".to_string()), + true, + ) ); } diff --git a/src-tauri/src/coordination/queue_manager.rs b/src-tauri/src/coordination/queue_manager.rs index 55055b7a..376a14de 100644 --- a/src-tauri/src/coordination/queue_manager.rs +++ b/src-tauri/src/coordination/queue_manager.rs @@ -12,11 +12,17 @@ use std::sync::Arc; use chrono::Utc; use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; use crate::domain::event::{Event, EventType, Severity}; use crate::events::EventBus; +use crate::orchestrator::work_graph::completion_ledger::{ + append_node_completion_facts, NodeCompletionFact, NodeCompletionProvenance, +}; use crate::orchestrator::work_graph::plan_parse::promote_initial_ready_nodes; -use crate::orchestrator::work_graph::{NodeStatus, TaskGraph}; +use crate::orchestrator::work_graph::{ + NodeStatus, TaskGraph, TaskId, WorkGraphOmission, WorkGraphOmissionReason, +}; use crate::storage::queue::{ QueueConflictCoverage, QueueConflictRow, QueueConflictWait, QueueRepo, QueueResolutionUpdate, QueueRow, QueueSnapshot, QueueStatus, SpawnFailureRelease, @@ -29,6 +35,19 @@ use crate::storage::StorageError; /// genuinely-working-but-quiet worker that keeps heartbeating is never reclaimed. pub const STUCK_CUTOFF_MS: i64 = 90_000; +/// Evidence source for a terminal status in a read-only work-graph projection. +/// Later evidence layers deliberately use this precedence: +/// declared > queue > inferred > plan. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CompletionProvenance { + Declared, + Queue, + Observed, + Inferred, + Plan, +} + /// Grace allowed between a successful spawn claim and the worker's first heartbeat. /// /// This is deliberately independent of [`STUCK_CUTOFF_MS`]. Some CLIs can spend 8-12 @@ -133,9 +152,8 @@ pub const NO_PROGRESS_WINDOW_SECS: u64 = 1_800; /// `record_heartbeat` matches no rows and `reclaim_stuck` can never recover it. /// /// Ceiling division so the realised budget is never shorter than the window. -pub const MAX_NO_PROGRESS_CONTINUATIONS: i64 = NO_PROGRESS_WINDOW_SECS - .div_ceil(HEARTBEAT_MIN_INTERVAL_SECS) - as i64; +pub const MAX_NO_PROGRESS_CONTINUATIONS: i64 = + NO_PROGRESS_WINDOW_SECS.div_ceil(HEARTBEAT_MIN_INTERVAL_SECS) as i64; /// How many times a single queue row may be claimed for a spawn that then fails /// before the row is retired as terminally `failed` (#175d). Bounds the retry @@ -157,11 +175,17 @@ pub enum ReleaseAfterFailure { /// Outcome of an operator/agent claim release. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ReleaseOutcome { - Released { previous: QueueStatus }, + Released { + previous: QueueStatus, + }, /// The exact claim is still constructing its worktree/PTY in this process. - SpawnInFlight { epoch: i64 }, + SpawnInFlight { + epoch: i64, + }, AlreadyQueued, - Terminal { status: QueueStatus }, + Terminal { + status: QueueStatus, + }, NoRow, } @@ -174,7 +198,10 @@ pub enum ReleaseOutcome { pub enum GuardedRelease { SpawnInFlight(SpawnInFlightReservation), PreparationFailed(E), - Complete { prepared: T, outcome: ReleaseOutcome }, + Complete { + prepared: T, + outcome: ReleaseOutcome, + }, } /// An in-process reservation protecting the interval between an atomic queue claim and the @@ -217,9 +244,12 @@ impl Drop for ReservationReturnGuard<'_> { if self.armed { let reservation = &self.reservation; let mut reservations = self.reservations.lock(); - if reservations.get(&reservation.queue_id).is_some_and(|current| { - current.epoch == reservation.epoch && current.worker_id == reservation.worker_id - }) { + if reservations + .get(&reservation.queue_id) + .is_some_and(|current| { + current.epoch == reservation.epoch && current.worker_id == reservation.worker_id + }) + { reservations.remove(&reservation.queue_id); } } @@ -229,13 +259,22 @@ impl Drop for ReservationReturnGuard<'_> { /// Result of the atomic claim UPDATE. Dependency diagnostics are gathered only after a loss. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ClaimOutcome { - Claimed { epoch: i64 }, - ResolutionIncomplete { task_id: String, reason: String }, + Claimed { + epoch: i64, + }, + ResolutionIncomplete { + task_id: String, + reason: String, + }, /// Retryable serialization wait. This post-loss snapshot is advisory by construction. - ConflictsPending { conflicts: Vec }, + ConflictsPending { + conflicts: Vec, + }, /// Retryable queue wait. The advisory prerequisite snapshot may be stale or empty when /// readiness changed after the loss, or when another task briefly reserved the worker slot. - DependenciesPending { task_ids: Vec }, + DependenciesPending { + task_ids: Vec, + }, AlreadyClaimed, } @@ -365,7 +404,7 @@ impl QueueManager { EventType::WorkerQueued, Severity::Info, ) - .await; + .await; Ok(()) } @@ -421,7 +460,7 @@ impl QueueManager { EventType::WorkerQueued, Severity::Info, ) - .await; + .await; Ok(()) } @@ -541,7 +580,7 @@ impl QueueManager { EventType::WorkerClaimed, Severity::Info, ) - .await; + .await; return_guard.disarm(); } else { self.emit( @@ -551,16 +590,14 @@ impl QueueManager { EventType::WorkerClaimed, Severity::Info, ) - .await; + .await; } Ok(ClaimOutcome::Claimed { epoch }) } else { // Losing the atomic UPDATE is authoritative. Reads below are diagnostic only and // cannot authorize or retry a claim. let row_after_claim = self.repo.get_row(id)?; - let task_id = row_after_claim - .as_ref() - .and_then(|row| row.task_id.clone()); + let task_id = row_after_claim.as_ref().and_then(|row| row.task_id.clone()); let queued_after_loss = row_after_claim .as_ref() .is_some_and(|row| row.status == QueueStatus::Queued); @@ -573,14 +610,12 @@ impl QueueManager { } else { Vec::new() }; - let task_ids = if queued_after_loss - && resolution_issue.is_none() - && conflicts.is_empty() - { - self.repo.pending_dependencies(id)? - } else { - Vec::new() - }; + let task_ids = + if queued_after_loss && resolution_issue.is_none() && conflicts.is_empty() { + self.repo.pending_dependencies(id)? + } else { + Vec::new() + }; self.emit( session_id, worker_id, @@ -646,10 +681,7 @@ impl QueueManager { /// Find a protected spawn by its reserved worker slot. This is a deny-only lookup used by /// the manual release handler before it touches roster or PTY state. - pub fn spawn_in_flight_for_worker( - &self, - worker_id: &str, - ) -> Option { + pub fn spawn_in_flight_for_worker(&self, worker_id: &str) -> Option { self.spawn_in_flight .lock() .values() @@ -712,7 +744,7 @@ impl QueueManager { match result { ReleaseAfterFailure::Exhausted { attempts } => { self.emit_for_row(id, EventType::WorkerFinalized, Severity::Warning) - .await; + .await; Ok(ReleaseAfterFailure::Exhausted { attempts }) } ReleaseAfterFailure::Released => { @@ -769,10 +801,7 @@ impl QueueManager { QueueStatus::Running | QueueStatus::Failed => { let previous = row.status; if self.repo.release_claim_manual(&row.id, Self::now_ms())? { - ( - ReleaseOutcome::Released { previous }, - Some(row.id.clone()), - ) + (ReleaseOutcome::Released { previous }, Some(row.id.clone())) } else { (ReleaseOutcome::NoRow, None) } @@ -781,10 +810,7 @@ impl QueueManager { other => (ReleaseOutcome::Terminal { status: other }, None), }, }; - ( - GuardedRelease::Complete { prepared, outcome }, - reclaimed_id, - ) + (GuardedRelease::Complete { prepared, outcome }, reclaimed_id) }; if let Some(id) = reclaimed_id { @@ -833,10 +859,7 @@ impl QueueManager { QueueStatus::Running | QueueStatus::Failed => { let previous = row.status; if self.repo.release_claim_manual(&row.id, Self::now_ms())? { - ( - ReleaseOutcome::Released { previous }, - Some(row.id.clone()), - ) + (ReleaseOutcome::Released { previous }, Some(row.id.clone())) } else { (ReleaseOutcome::NoRow, None) } @@ -876,19 +899,46 @@ impl QueueManager { status: &str, ) -> Result { let now = Self::now_ms(); - let updated_row_id = self - .repo - .record_heartbeat_for_assignment( - session_id, - worker_id, - assignment_id, - status, - now, - )?; + let updated_row_id = self.repo.record_heartbeat_for_assignment( + session_id, + worker_id, + assignment_id, + status, + now, + )?; if status == "completed" { if let Some(row_id) = updated_row_id.as_deref() { + let completion = self.repo.get_row(row_id)?.and_then(|row| { + row.task_id.map(|task_id| { + ( + row.session_id, + NodeCompletionFact::new( + task_id, + row.worker_id, + NodeCompletionProvenance::QueueFinalize, + ), + ) + }) + }); + if let Some((fact_session_id, fact)) = completion.as_ref() { + let session_dir = self + .event_bus + .data_dir() + .join("sessions") + .join(fact_session_id); + append_node_completion_facts(&session_dir, std::slice::from_ref(fact))?; + } self.emit_for_row(row_id, EventType::WorkerFinalized, Severity::Info) .await; + if let Some((fact_session_id, fact)) = completion { + if let Err(error) = self.event_bus.publish(fact.event(&fact_session_id)).await { + tracing::warn!( + session_id = %fact_session_id, + task_id = %fact.task_id, + "Failed to publish work-node completion event: {error}" + ); + } + } } } Ok(updated_row_id.is_some()) @@ -954,9 +1004,11 @@ impl QueueManager { /// `WorkerFinalized` per finalized row. pub async fn finalize_no_progress(&self) -> Result, StorageError> { let now = Self::now_ms(); - let ids = self - .repo - .finalize_no_progress(MAX_CONTINUATIONS, MAX_NO_PROGRESS_CONTINUATIONS, now)?; + let ids = self.repo.finalize_no_progress( + MAX_CONTINUATIONS, + MAX_NO_PROGRESS_CONTINUATIONS, + now, + )?; for id in &ids { self.emit_for_row(id, EventType::WorkerFinalized, Severity::Info) .await; @@ -1019,22 +1071,90 @@ impl QueueManager { session_id: &str, graph: &TaskGraph, ) -> Result { - let mut projected = graph.clone(); - let mut latest: BTreeMap<&str, &QueueRow> = BTreeMap::new(); - let rows = self.repo.rows_for_session(session_id)?; - for row in &rows { - let Some(task_id) = row.task_id.as_deref() else { + let latest = self.latest_queue_rows_by_task(session_id)?; + let mut projected = Self::overlay_queue_rows(graph, &latest); + + for node in &mut projected.nodes { + if !latest.contains_key(&node.id) + && matches!( + node.status, + NodeStatus::Running + | NodeStatus::Completed + | NodeStatus::Failed + | NodeStatus::Blocked + | NodeStatus::Cancelled + ) + { + // A persisted runtime-looking status without a queue row cannot satisfy the SQL + // prerequisite lookup. Reset it conservatively before readiness promotion. + node.status = NodeStatus::Pending; + } + } + promote_initial_ready_nodes(&mut projected); + Ok(projected) + } + + /// Project queue evidence for the read API without changing readiness or erasing + /// persisted terminal statuses that have no queue row. + pub fn project_queue_statuses_for_view( + &self, + session_id: &str, + graph: &TaskGraph, + ) -> Result<(TaskGraph, BTreeMap), StorageError> { + let latest = self.latest_queue_rows_by_task(session_id)?; + let mut projected = Self::overlay_queue_rows(graph, &latest); + let mut provenance = BTreeMap::new(); + let mut unbacked_terminal = Vec::new(); + + for node in &projected.nodes { + if matches!(node.status, NodeStatus::Completed | NodeStatus::Failed) { + if latest.contains_key(&node.id) { + provenance.insert(node.id.clone(), CompletionProvenance::Queue); + } else { + provenance.insert(node.id.clone(), CompletionProvenance::Plan); + unbacked_terminal.push(format!("queue:{}", node.id)); + } + } + } + if !unbacked_terminal.is_empty() { + let mut omission = WorkGraphOmission::new( + WorkGraphOmissionReason::ResolutionIncomplete, + unbacked_terminal.len(), + unbacked_terminal, + ); + omission.detail = "persisted terminal node status had no backing queue row; the plan value was preserved for view projection".to_string(); + projected.omissions.push(omission); + } + + Ok((projected, provenance)) + } + + fn latest_queue_rows_by_task( + &self, + session_id: &str, + ) -> Result, StorageError> { + let mut latest: BTreeMap = BTreeMap::new(); + for row in self.repo.rows_for_session(session_id)? { + let Some(task_id) = row.task_id.clone() else { continue; }; - let replace = latest.get(task_id).is_none_or(|existing| { + let replace = latest.get(&task_id).is_none_or(|existing| { (row.updated_at, row.created_at, row.id.as_str()) - > (existing.updated_at, existing.created_at, existing.id.as_str()) + > ( + existing.updated_at, + existing.created_at, + existing.id.as_str(), + ) }); if replace { latest.insert(task_id, row); } } + Ok(latest) + } + fn overlay_queue_rows(graph: &TaskGraph, latest: &BTreeMap) -> TaskGraph { + let mut projected = graph.clone(); for node in &mut projected.nodes { if let Some(row) = latest.get(node.id.as_str()) { node.status = match row.status { @@ -1044,21 +1164,9 @@ impl QueueManager { QueueStatus::Failed => NodeStatus::Failed, QueueStatus::Blocked => NodeStatus::Blocked, }; - } else if matches!( - node.status, - NodeStatus::Running - | NodeStatus::Completed - | NodeStatus::Failed - | NodeStatus::Blocked - | NodeStatus::Cancelled - ) { - // A persisted runtime-looking status without a queue row cannot satisfy the SQL - // prerequisite lookup. Reset it conservatively before readiness promotion. - node.status = NodeStatus::Pending; } } - promote_initial_ready_nodes(&mut projected); - Ok(projected) + projected } /// Cancel a task and block all downstream queue rows with a visible reason. @@ -1113,7 +1221,7 @@ impl QueueManager { event_type, severity, ) - .await + .await } Ok(None) => {} Err(e) => tracing::warn!("Failed to load queue row {id} for event: {e}"), @@ -1156,8 +1264,16 @@ mod tests { assert_eq!(snap.queued, 1); // First claim wins, second loses (already running, fresh). - assert!(mgr.claim_and_spawn("s1-worker-1", "s1", "s1-worker-1").await.unwrap().is_some()); - assert!(mgr.claim_and_spawn("s1-worker-1", "s1", "s1-worker-1").await.unwrap().is_none()); + assert!(mgr + .claim_and_spawn("s1-worker-1", "s1", "s1-worker-1") + .await + .unwrap() + .is_some()); + assert!(mgr + .claim_and_spawn("s1-worker-1", "s1", "s1-worker-1") + .await + .unwrap() + .is_none()); let snap = mgr.queue_snapshot("s1").unwrap(); assert_eq!(snap.running, 1); @@ -1170,10 +1286,20 @@ mod tests { // Subscribe BEFORE the operations so we capture every event. let mut rx = mgr.event_bus.subscribe(); - mgr.enqueue_worker("r1", "s1", "s1-worker-1", "backend", "codex", json!({}), None) + mgr.enqueue_worker( + "r1", + "s1", + "s1-worker-1", + "backend", + "codex", + json!({}), + None, + ) + .await + .unwrap(); + mgr.claim_and_spawn("r1", "s1", "s1-worker-1") .await .unwrap(); - mgr.claim_and_spawn("r1", "s1", "s1-worker-1").await.unwrap(); let e1 = rx.recv().await.unwrap(); assert_eq!(e1.event_type, EventType::WorkerQueued); @@ -1183,7 +1309,9 @@ mod tests { assert_eq!(e2.event_type, EventType::WorkerClaimed); // A lost claim emits WorkerClaimFailed. - mgr.claim_and_spawn("r1", "s1", "s1-worker-1").await.unwrap(); + mgr.claim_and_spawn("r1", "s1", "s1-worker-1") + .await + .unwrap(); let e3 = rx.recv().await.unwrap(); assert_eq!(e3.event_type, EventType::WorkerClaimFailed); } @@ -1266,18 +1394,33 @@ mod tests { #[tokio::test] async fn test_reconcile_repairs_orphaned_running() { let (_dir, mgr) = manager(); - mgr.enqueue_worker("r1", "s1", "s1-worker-1", "backend", "codex", json!({}), None) + mgr.enqueue_worker( + "r1", + "s1", + "s1-worker-1", + "backend", + "codex", + json!({}), + None, + ) + .await + .unwrap(); + mgr.claim_and_spawn("r1", "s1", "s1-worker-1") .await .unwrap(); - mgr.claim_and_spawn("r1", "s1", "s1-worker-1").await.unwrap(); // After a crash there is no live PTY for s1-worker-1 → reconcile requeues it. let reclaimed = mgr.reconcile("s1", &[]).await.unwrap(); assert_eq!(reclaimed, vec!["r1".to_string()]); assert_eq!(mgr.queue_snapshot("s1").unwrap().queued, 1); // If the worker is still live, reconcile leaves it running. - mgr.claim_and_spawn("r1", "s1", "s1-worker-1").await.unwrap(); - let reclaimed = mgr.reconcile("s1", &["s1-worker-1".to_string()]).await.unwrap(); + mgr.claim_and_spawn("r1", "s1", "s1-worker-1") + .await + .unwrap(); + let reclaimed = mgr + .reconcile("s1", &["s1-worker-1".to_string()]) + .await + .unwrap(); assert!(reclaimed.is_empty()); assert_eq!(mgr.queue_snapshot("s1").unwrap().running, 1); } diff --git a/src-tauri/src/coordination/state.rs b/src-tauri/src/coordination/state.rs index e99d3116..a77b8b3f 100644 --- a/src-tauri/src/coordination/state.rs +++ b/src-tauri/src/coordination/state.rs @@ -12,6 +12,9 @@ use crate::orchestrator::org_graph::ownership::{ derive_path_ownership, LivePrincipal, OrchestratorWriteAttempt, OrchestratorWriteOutcome, OwnershipSessionState, }; +use crate::orchestrator::work_graph::completion_ledger::{ + append_node_completion_facts, read_node_completion_facts, NodeCompletionFact, +}; use crate::orchestrator::work_graph::divergence::DivergenceSummary; use crate::orchestrator::work_graph::review::ReviewExpansionSidecar; use crate::orchestrator::work_graph::runtime::GraphCompositionState; @@ -54,6 +57,8 @@ pub struct WorkerStateInfo { pub struct HierarchyNode { pub id: String, pub role: String, + #[serde(default)] + pub principal: Option, pub parent_id: Option, pub children: Vec, } @@ -108,6 +113,17 @@ impl StateManager { Self { session_path } } + pub fn append_node_completion_facts( + &self, + facts: &[NodeCompletionFact], + ) -> Result<(), StateError> { + append_node_completion_facts(&self.session_path, facts).map_err(StateError::Io) + } + + pub fn read_node_completion_facts(&self) -> Result, StateError> { + read_node_completion_facts(&self.session_path).map_err(StateError::Io) + } + /// Get path to state directory fn state_dir(&self) -> PathBuf { self.session_path.join("state") @@ -138,9 +154,9 @@ impl StateManager { } fn write_atomic_text(&self, target: PathBuf, content: &str) -> Result<(), StateError> { - let parent = target - .parent() - .ok_or_else(|| StateError::Io(std::io::Error::other("target has no parent directory")))?; + let parent = target.parent().ok_or_else(|| { + StateError::Io(std::io::Error::other("target has no parent directory")) + })?; fs::create_dir_all(parent)?; let mut temp = NamedTempFile::new_in(parent)?; @@ -191,7 +207,10 @@ impl StateManager { for worker in workers { content.push_str(&format!("### {} ({})\n", worker.id, worker.role.label)); content.push_str(&format!("- CLI: {}\n", worker.cli)); - content.push_str(&format!("- Specialization: {}\n", self.get_role_description(&worker.role))); + content.push_str(&format!( + "- Specialization: {}\n", + self.get_role_description(&worker.role) + )); content.push_str("\n"); } @@ -229,8 +248,12 @@ impl StateManager { // For now, we read from hierarchy.json instead since that's more reliable // workers.md is mainly for the Queen to read self.read_hierarchy().map(|nodes| { - nodes.into_iter().filter(|n| n.role != "Queen" && n.role != "Evaluator" && !n.role.starts_with("QaWorker-")).map(|n| { - WorkerStateInfo { + nodes + .into_iter() + .filter(|n| { + n.role != "Queen" && n.role != "Evaluator" && !n.role.starts_with("QaWorker-") + }) + .map(|n| WorkerStateInfo { id: n.id, role: WorkerRole { role_type: n.role.clone(), @@ -244,8 +267,8 @@ impl StateManager { current_task: None, last_update: Utc::now(), last_heartbeat: None, - } - }).collect() + }) + .collect() }) } @@ -323,13 +346,19 @@ impl StateManager { let commit_sha = commit_sha.map(str::to_string); tokio::task::spawn_blocking(move || { - StateManager::new(session_path) - .write_qa_verdict(&from, &to, &content, commit_sha.as_deref()) + StateManager::new(session_path).write_qa_verdict( + &from, + &to, + &content, + commit_sha.as_deref(), + ) }) .await - .map_err(|err| StateError::Io(std::io::Error::other(format!( - "QA verdict write task failed: {err}" - ))))? + .map_err(|err| { + StateError::Io(std::io::Error::other(format!( + "QA verdict write task failed: {err}" + ))) + })? } /// Write the Prince's remediation verdict (peer/prince-verdict.json). The Queen @@ -368,8 +397,12 @@ impl StateManager { let commit_sha = commit_sha.map(str::to_string); tokio::task::spawn_blocking(move || { - StateManager::new(session_path) - .write_prince_verdict(&from, &to, &content, commit_sha.as_deref()) + StateManager::new(session_path).write_prince_verdict( + &from, + &to, + &content, + commit_sha.as_deref(), + ) }) .await .map_err(|err| { @@ -443,11 +476,7 @@ impl StateManager { let Some(graph) = self.read_work_graph()? else { return Ok(None); }; - let artifact = self.write_portable_work_graph_artifact( - lifecycle_stage, - &graph, - None, - )?; + let artifact = self.write_portable_work_graph_artifact(lifecycle_stage, &graph, None)?; let snapshot = WorkGraphLifecycleSnapshot { lifecycle_stage: lifecycle_stage.to_string(), emitted_at: Utc::now(), @@ -457,10 +486,7 @@ impl StateManager { }; self.ensure_state_dir()?; let json = serde_json::to_string_pretty(&snapshot)?; - self.write_atomic_text( - self.state_dir().join("work-graph-lifecycle.json"), - &json, - )?; + self.write_atomic_text(self.state_dir().join("work-graph-lifecycle.json"), &json)?; Ok(Some(artifact)) } @@ -638,13 +664,16 @@ impl StateManager { HashMap::new() }; - assignments.insert(worker_id.to_string(), TaskAssignment { - worker_id: worker_id.to_string(), - task: task.to_string(), - assigned_at: Utc::now(), - status: AssignmentStatus::Pending, - plan_task_id, - }); + assignments.insert( + worker_id.to_string(), + TaskAssignment { + worker_id: worker_id.to_string(), + task: task.to_string(), + assigned_at: Utc::now(), + status: AssignmentStatus::Pending, + plan_task_id, + }, + ); let json = serde_json::to_string_pretty(&assignments)?; fs::write(assignments_path, json)?; @@ -695,7 +724,10 @@ impl StateManager { /// Get assignment for a specific worker #[allow(dead_code)] - pub fn get_worker_assignment(&self, worker_id: &str) -> Result, StateError> { + pub fn get_worker_assignment( + &self, + worker_id: &str, + ) -> Result, StateError> { let assignments = self.get_assignments()?; Ok(assignments.get(worker_id).cloned()) } @@ -737,7 +769,6 @@ impl StateManager { .map_err(|err| StateError::ContractParse(err.to_string()))?; Ok(Some(contract)) } - } const PORTABLE_WORK_GRAPH_HTML: &str = r##" @@ -851,12 +882,17 @@ mod tests { .unwrap(); let path = temp.path().join("peer").join("milestone-ready.json"); - let record: PeerMessageRecord = serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap(); + let record: PeerMessageRecord = + serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap(); assert_eq!(record.kind, "milestone-ready"); assert_eq!(record.content, "Milestone B is ready"); assert!(temp.path().join("peer").read_dir().unwrap().all(|entry| { - !entry.unwrap().file_name().to_string_lossy().ends_with(".tmp") + !entry + .unwrap() + .file_name() + .to_string_lossy() + .ends_with(".tmp") })); } @@ -881,7 +917,10 @@ mod tests { let read_back = manager.read_contract(2).unwrap().unwrap(); assert_eq!(written, read_back); - assert_eq!(read_back.criterion(1).unwrap().description, "Dashboard loads with current account data"); + assert_eq!( + read_back.criterion(1).unwrap().description, + "Dashboard loads with current account data" + ); } #[test] diff --git a/src-tauri/src/domain/event.rs b/src-tauri/src/domain/event.rs index 134f44c1..cb2a3d6a 100644 --- a/src-tauri/src/domain/event.rs +++ b/src-tauri/src/domain/event.rs @@ -34,6 +34,7 @@ pub enum EventType { WorkerClaimFailed, WorkerReclaimed, WorkerFinalized, + WorkNodeCompleted, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] diff --git a/src-tauri/src/events/bus.rs b/src-tauri/src/events/bus.rs index c22ecf68..a34c9b71 100644 --- a/src-tauri/src/events/bus.rs +++ b/src-tauri/src/events/bus.rs @@ -1,3 +1,4 @@ +use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -42,6 +43,10 @@ impl EventBus { self.sender.subscribe() } + pub fn data_dir(&self) -> &Path { + &self.data_dir + } + /// Subscribe and filter events by session_id. pub fn subscribe_session(&self, session_id: String) -> FilteredReceiver { FilteredReceiver { @@ -72,7 +77,10 @@ impl EventBus { line.push('\n'); let mut writers = self.writers.lock().await; - if !writers.iter().any(|(session_id, _)| session_id == &event.session_id) { + if !writers + .iter() + .any(|(session_id, _)| session_id == &event.session_id) + { let session_dir = self.data_dir.join(&event.session_id); tokio::fs::create_dir_all(&session_dir) .await @@ -198,10 +206,9 @@ mod tests { let event = make_event("sess-persist", EventType::SessionCreated); bus.publish(event.clone()).await.unwrap(); - let contents = - tokio::fs::read_to_string(tmp.path().join("sess-persist/events.jsonl")) - .await - .unwrap(); + let contents = tokio::fs::read_to_string(tmp.path().join("sess-persist/events.jsonl")) + .await + .unwrap(); let deserialized: Event = serde_json::from_str(contents.trim()).unwrap(); assert_eq!(deserialized.id, event.id); assert_eq!(deserialized.event_type, EventType::SessionCreated); @@ -219,10 +226,9 @@ mod tests { .await .unwrap(); - let contents = - tokio::fs::read_to_string(tmp.path().join("sess-append/events.jsonl")) - .await - .unwrap(); + let contents = tokio::fs::read_to_string(tmp.path().join("sess-append/events.jsonl")) + .await + .unwrap(); let lines: Vec<&str> = contents.trim().lines().collect(); assert_eq!(lines.len(), 2); diff --git a/src-tauri/src/http/handlers/evaluator.rs b/src-tauri/src/http/handlers/evaluator.rs index fd966afb..a0aeb444 100644 --- a/src-tauri/src/http/handlers/evaluator.rs +++ b/src-tauri/src/http/handlers/evaluator.rs @@ -10,14 +10,15 @@ use std::{fmt, sync::Arc}; use crate::coordination::{CoordinationMessage, StateManager}; use crate::http::error::ApiError; use crate::http::state::AppState; +use crate::orchestrator::work_graph::completion_ledger::{ + NodeCompletionFact, NodeCompletionProvenance, +}; use crate::orchestrator::work_graph::review::ReviewExpansionSidecar; use crate::orchestrator::work_graph::runtime::{ record_review_verdict_and_record, route_failed_verdict_and_record, GraphCompositionState, ReviewVerdict, }; -use crate::orchestrator::work_graph::{ - WorkGraph, WorkGraphOmission, WorkGraphOmissionReason, -}; +use crate::orchestrator::work_graph::{WorkGraph, WorkGraphOmission, WorkGraphOmissionReason}; use crate::pty::{AgentConfig, AgentRole}; use crate::session::{AuthStrategy, SessionController, SessionState}; @@ -296,9 +297,9 @@ pub async fn add_qa_worker( status: "Running".to_string(), task_file: { let controller = state.session_controller.read(); - let session = controller - .get_session(&session_id) - .ok_or_else(|| ApiError::not_found(format!("Session {} not found", session_id)))?; + let session = controller.get_session(&session_id).ok_or_else(|| { + ApiError::not_found(format!("Session {} not found", session_id)) + })?; SessionController::absolute_task_file_path_for_qa_worker( &session.project_path, &session_id, @@ -365,19 +366,16 @@ pub async fn dev_login( .ok_or_else(|| ApiError::not_found(format!("Session {} not found", session_id)))?; match &session.auth_strategy { - AuthStrategy::DevBypass { token } if *token == query.token => { - Ok(Json(json!({ - "session_id": session_id, - "auth": "dev-bypass", - "granted": true - }))) - } - AuthStrategy::DevBypass { .. } => { - Err(ApiError::new(StatusCode::UNAUTHORIZED, "Invalid dev-bypass token")) - } - AuthStrategy::None => { - Err(ApiError::not_found("Auth not configured for this session")) - } + AuthStrategy::DevBypass { token } if *token == query.token => Ok(Json(json!({ + "session_id": session_id, + "auth": "dev-bypass", + "granted": true + }))), + AuthStrategy::DevBypass { .. } => Err(ApiError::new( + StatusCode::UNAUTHORIZED, + "Invalid dev-bypass token", + )), + AuthStrategy::None => Err(ApiError::not_found("Auth not configured for this session")), } } @@ -545,10 +543,7 @@ pub(crate) enum WorkGraphVerdictError { MissingGraph, MissingSidecar, UnknownVerdict(String), - StaleVerdict { - requested: String, - current: String, - }, + StaleVerdict { requested: String, current: String }, Mutation(String), } @@ -618,7 +613,15 @@ fn persist_work_graph_verdict( graph: &WorkGraph, sidecar: &ReviewExpansionSidecar, mut composition: Option, + completion_facts: &[NodeCompletionFact], ) -> Result<(), WorkGraphVerdictError> { + state_manager + .append_node_completion_facts(completion_facts) + .map_err(|error| { + WorkGraphVerdictError::State(format!( + "Failed to persist declared completion after QA verdict: {error}" + )) + })?; state_manager.write_work_graph(graph).map_err(|error| { WorkGraphVerdictError::State(format!( "Failed to persist work graph after QA verdict: {error}" @@ -677,6 +680,23 @@ pub(crate) fn apply_work_graph_verdict( work_graph_verdict_id: Option<&str>, verdict: &str, ) -> Result { + apply_work_graph_verdict_for_agent( + state_manager, + session_id, + work_graph_verdict_id, + verdict, + None, + ) + .map(|(routing, _)| routing) +} + +fn apply_work_graph_verdict_for_agent( + state_manager: &StateManager, + session_id: &str, + work_graph_verdict_id: Option<&str>, + verdict: &str, + evaluator_agent_id: Option<&str>, +) -> Result<(WorkGraphVerdictRouting, Vec), WorkGraphVerdictError> { let verdict_id = work_graph_verdict_id .map(str::trim) .filter(|value| !value.is_empty()); @@ -688,9 +708,12 @@ pub(crate) fn apply_work_graph_verdict( omission = MISSING_WORK_GRAPH_VERDICT_ID, "QA verdict has no explicit work-graph verdict id; no graph node was guessed" ); - return Ok(WorkGraphVerdictRouting::OmittedMissingVerdictId { - omission_persisted: false, - }); + return Ok(( + WorkGraphVerdictRouting::OmittedMissingVerdictId { + omission_persisted: false, + }, + Vec::new(), + )); }; let omission = WorkGraphOmission::new( WorkGraphOmissionReason::ResolutionIncomplete, @@ -709,9 +732,12 @@ pub(crate) fn apply_work_graph_verdict( "Could not persist missing work-graph verdict-id omission" ); } - return Ok(WorkGraphVerdictRouting::OmittedMissingVerdictId { - omission_persisted: persisted, - }); + return Ok(( + WorkGraphVerdictRouting::OmittedMissingVerdictId { + omission_persisted: persisted, + }, + Vec::new(), + )); } let verdict_id = verdict_id.expect("checked above").to_string(); let (mut graph, composition) = load_authoritative_work_graph(state_manager)?; @@ -789,8 +815,24 @@ pub(crate) fn apply_work_graph_verdict( ))) } }; - persist_work_graph_verdict(state_manager, &graph, &sidecar, composition)?; - Ok(routing) + let completion_facts = match (&routing, evaluator_agent_id) { + (WorkGraphVerdictRouting::Passed { verdict_id, .. }, Some(agent_id)) => { + vec![NodeCompletionFact::new( + verdict_id.clone(), + agent_id.to_string(), + NodeCompletionProvenance::EvaluatorVerdict, + )] + } + _ => Vec::new(), + }; + persist_work_graph_verdict( + state_manager, + &graph, + &sidecar, + composition, + &completion_facts, + )?; + Ok((routing, completion_facts)) } pub(crate) fn apply_verdict( @@ -942,7 +984,8 @@ pub async fn post_verdict( )) })?; - let verdict_message = CoordinationMessage::qa_verdict(&evaluator_id, &queen_id, &verdict_content); + let verdict_message = + CoordinationMessage::qa_verdict(&evaluator_id, &queen_id, &verdict_content); if let Err(err) = state .storage .append_coordination_log(&session_id, &verdict_message) @@ -958,13 +1001,23 @@ pub async fn post_verdict( // explicitly supplied join id may mutate the graph; the legacy request // shape records a ResolutionIncomplete omission without selecting a node. let graph_state_manager = StateManager::new(state.storage.session_dir(&session_id)); - let work_graph_routing = apply_work_graph_verdict( + let (work_graph_routing, completion_facts) = apply_work_graph_verdict_for_agent( &graph_state_manager, &session_id, req.work_graph_verdict_id.as_deref(), verdict, + Some(&evaluator_id), ) .map_err(map_work_graph_verdict_error)?; + for fact in completion_facts { + if let Err(error) = state.event_bus.publish(fact.event(&session_id)).await { + tracing::warn!( + session_id = %session_id, + task_id = %fact.task_id, + "Failed to publish durable evaluator completion event: {error}" + ); + } + } let new_state = { let controller = state.session_controller.read(); @@ -993,19 +1046,24 @@ fn blocked_reason_message( blocked_detail: Option<&str>, rationale: Option<&str>, ) -> String { - let category = match blocked_reason.map(str::trim).filter(|value| !value.is_empty()) { + let category = match blocked_reason + .map(str::trim) + .filter(|value| !value.is_empty()) + { Some("ui-unavailable") | Some("ui_unavailable") => { "A pass-criterion requires a UI/host that isn't running, so it can't be exercised." } Some("http-failure") | Some("http_failure") => { "One or more QA-worker verdicts could not be delivered over HTTP." } - Some(other) => return match blocked_detail.or(rationale) { - Some(detail) if !detail.trim().is_empty() => { - format!("QA blocked ({}): {}", other, detail.trim()) + Some(other) => { + return match blocked_detail.or(rationale) { + Some(detail) if !detail.trim().is_empty() => { + format!("QA blocked ({}): {}", other, detail.trim()) + } + _ => format!("QA blocked: {}", other), } - _ => format!("QA blocked: {}", other), - }, + } None => "QA could not reach a PASS/FAIL verdict.", }; match blocked_detail.or(rationale) { @@ -1201,8 +1259,18 @@ pub async fn force_fail( #[cfg(test)] mod tests { - use super::map_add_qa_worker_error; + use super::{apply_work_graph_verdict_for_agent, map_add_qa_worker_error}; use axum::http::StatusCode; + use tempfile::TempDir; + + use crate::coordination::StateManager; + use crate::orchestrator::work_graph::completion_ledger::NodeCompletionProvenance; + use crate::orchestrator::work_graph::review::{ + instantiate_review_templates, ReviewExpansionSidecar, ReviewTemplate, + }; + use crate::orchestrator::work_graph::{ + BindingRef, NodeContract, NodeKind, NodeStatus, TaskGraph, WorkNode, + }; #[test] fn maps_missing_session_to_not_found() { @@ -1223,4 +1291,50 @@ mod tests { let error = map_add_qa_worker_error("Failed to spawn QA worker 1: boom".to_string()); assert_eq!(error.status, StatusCode::INTERNAL_SERVER_ERROR); } + + #[test] + fn passing_evaluator_verdict_appends_an_exact_completion_fact() { + let temp = TempDir::new().unwrap(); + let manager = StateManager::new(temp.path().to_path_buf()); + let session_id = format!("evaluator-completion-{}", uuid::Uuid::new_v4()); + let mut graph = TaskGraph::new( + vec![WorkNode::new( + "implementation", + NodeKind::Task, + "Implementation", + NodeContract { + inputs: Vec::new(), + outputs: vec!["code".to_string()], + acceptance: vec!["accepted".to_string()], + }, + BindingRef::Role("worker".to_string()), + NodeStatus::Pending, + )], + Vec::new(), + ); + let template = ReviewTemplate::code_tasks("qa"); + let expansions = instantiate_review_templates(&mut graph, &[template.clone()]).unwrap(); + let verdict_id = expansions[0].rounds[0].verdict_id.clone(); + let sidecar = ReviewExpansionSidecar::from_expansions(&[template], expansions).unwrap(); + manager.write_work_graph(&graph).unwrap(); + manager.write_review_expansion_sidecar(&sidecar).unwrap(); + + let (_, facts) = apply_work_graph_verdict_for_agent( + &manager, + &session_id, + Some(&verdict_id), + "PASS", + Some("session-evaluator"), + ) + .unwrap(); + + assert_eq!(facts.len(), 1); + assert_eq!(facts[0].task_id, verdict_id); + assert_eq!(facts[0].agent_id, "session-evaluator"); + assert_eq!( + facts[0].provenance, + NodeCompletionProvenance::EvaluatorVerdict + ); + assert_eq!(manager.read_node_completion_facts().unwrap(), facts); + } } diff --git a/src-tauri/src/http/handlers/heartbeats.rs b/src-tauri/src/http/handlers/heartbeats.rs index 4ce728d0..5b3f928c 100644 --- a/src-tauri/src/http/handlers/heartbeats.rs +++ b/src-tauri/src/http/handlers/heartbeats.rs @@ -4,12 +4,17 @@ use axum::{ Json, }; use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; use std::sync::Arc; use super::validate_agent_id; use super::validate_session_id; +use crate::coordination::StateManager; use crate::http::error::ApiError; use crate::http::state::AppState; +use crate::orchestrator::work_graph::completion_ledger::{ + NodeCompletionFact, NodeCompletionProvenance, +}; /// POST /api/sessions/{id}/heartbeat - Body #[derive(Debug, Deserialize)] @@ -22,6 +27,9 @@ pub struct PostHeartbeatRequest { /// deterministic fallback; a supplied identity is always treated as an exact fence. #[serde(default)] pub assignment_id: Option, + /// Exact work-graph node ids completed by this heartbeat's resolved agent. + #[serde(default)] + pub completed_nodes: Vec, } /// Response for POST heartbeat @@ -71,7 +79,10 @@ pub async fn post_heartbeat( "Status must be one of: working, idle, completed", )); } - if req.assignment_id.is_some_and(|assignment_id| assignment_id <= 0) { + if req + .assignment_id + .is_some_and(|assignment_id| assignment_id <= 0) + { return Err(ApiError::bad_request( "assignment_id must be a positive server-issued identity", )); @@ -100,13 +111,12 @@ pub async fn post_heartbeat( let (raw_known, qualified_known) = { let controller = state.session_controller.read(); - let session = controller.get_session(&session_id).ok_or_else(|| { - ApiError::not_found(format!("Session {} not found", session_id)) - })?; + let session = controller + .get_session(&session_id) + .ok_or_else(|| ApiError::not_found(format!("Session {} not found", session_id)))?; let pty_manager = state.pty_manager.read(); - let known = |id: &str| { - session.agents.iter().any(|a| a.id == id) || pty_manager.is_alive(id) - }; + let known = + |id: &str| session.agents.iter().any(|a| a.id == id) || pty_manager.is_alive(id); (known(&req.agent_id), known(&qualified)) }; @@ -139,6 +149,59 @@ pub async fn post_heartbeat( } }; + // Validate the whole declared set before the queue or controller is mutated. Exact ids are + // deliberate: aliases and labels are not node identity, and a mixed valid/invalid request + // must apply none of its declarations. + let completed_nodes = if req.completed_nodes.is_empty() { + Vec::new() + } else { + if req.status != "completed" { + return Err(ApiError::bad_request( + "completed_nodes may only be supplied with status completed", + )); + } + let state_manager = StateManager::new(state.storage.session_dir(&session_id)); + let composition = state_manager + .read_graph_composition_state() + .map_err(|error| { + ApiError::internal(format!( + "Failed to read graph composition for completed_nodes validation: {error}" + )) + })?; + let graph = if let Some(composition) = composition { + Some(composition.graph) + } else { + state_manager.read_work_graph().map_err(|error| { + ApiError::internal(format!( + "Failed to read work graph for completed_nodes validation: {error}" + )) + })? + }; + let known_ids: BTreeSet<&str> = graph + .as_ref() + .into_iter() + .flat_map(|graph| graph.nodes.iter().map(|node| node.id.as_str())) + .collect(); + let unknown: BTreeSet<&str> = req + .completed_nodes + .iter() + .map(String::as_str) + .filter(|task_id| !known_ids.contains(task_id)) + .collect(); + if !unknown.is_empty() { + return Err(ApiError::bad_request(format!( + "Unknown completed_nodes: {}", + unknown.into_iter().collect::>().join(", ") + ))); + } + req.completed_nodes + .iter() + .cloned() + .collect::>() + .into_iter() + .collect::>() + }; + // A supplied assignment identity is an exact durable fence. Check it before // mutating controller liveness so a stale worker cannot report a completion // that the queue rejected. Assignment-free callers remain fail-open below: @@ -183,6 +246,33 @@ pub async fn post_heartbeat( .map_err(|e| ApiError::internal(e.to_string()))?; } + if !completed_nodes.is_empty() { + let facts = completed_nodes + .into_iter() + .map(|task_id| { + NodeCompletionFact::new( + task_id, + agent_id.clone(), + NodeCompletionProvenance::Heartbeat, + ) + }) + .collect::>(); + StateManager::new(state.storage.session_dir(&session_id)) + .append_node_completion_facts(&facts) + .map_err(|error| { + ApiError::internal(format!("Failed to persist completed_nodes: {error}")) + })?; + for fact in facts { + if let Err(error) = state.event_bus.publish(fact.event(&session_id)).await { + tracing::warn!( + session_id = %session_id, + task_id = %fact.task_id, + "Failed to publish durable heartbeat completion event: {error}" + ); + } + } + } + Ok(( StatusCode::OK, Json(PostHeartbeatResponse { @@ -285,8 +375,7 @@ mod tests { fn fixture() -> HeartbeatFixture { let temp = TempDir::new().expect("heartbeat fixture directory"); let storage = Arc::new( - SessionStorage::new_with_base(temp.path().to_path_buf()) - .expect("session storage"), + SessionStorage::new_with_base(temp.path().to_path_buf()).expect("session storage"), ); storage .create_session_dir(SESSION_ID) @@ -295,19 +384,17 @@ mod tests { storage.load_config().expect("test config"), )); let pty_manager = Arc::new(RwLock::new(PtyManager::new())); - let session_controller = Arc::new(RwLock::new(SessionController::new( - Arc::clone(&pty_manager), - ))); + let session_controller = Arc::new(RwLock::new(SessionController::new(Arc::clone( + &pty_manager, + )))); session_controller.write().set_storage(Arc::clone(&storage)); let injection_manager = Arc::new(RwLock::new(InjectionManager::new( Arc::clone(&pty_manager), - SessionStorage::new_with_base(temp.path().to_path_buf()) - .expect("injection storage"), + SessionStorage::new_with_base(temp.path().to_path_buf()).expect("injection storage"), ))); let event_bus = EventBus::new(storage.base_dir().clone()); - let app_state_db = Arc::new( - ApplicationStateDb::open_in_memory().expect("application state database"), - ); + let app_state_db = + Arc::new(ApplicationStateDb::open_in_memory().expect("application state database")); let queue_repo = Arc::new(QueueRepo::new(Arc::clone(&app_state_db))); queue_repo.ensure_schema().expect("queue schema"); let queue_manager = Arc::new(QueueManager::new(queue_repo, Arc::clone(&event_bus))); @@ -399,6 +486,7 @@ mod tests { status: status.to_string(), summary: Some(format!("{status} from handler test")), assignment_id, + completed_nodes: Vec::new(), }), ) .await diff --git a/src-tauri/src/http/handlers/inject.rs b/src-tauri/src/http/handlers/inject.rs index a3f30c6d..94e746ec 100644 --- a/src-tauri/src/http/handlers/inject.rs +++ b/src-tauri/src/http/handlers/inject.rs @@ -17,7 +17,7 @@ fn default_submit() -> bool { true } -const INJECTION_EVIDENCE_SCOPE: &str = "These facts prove only that bytes were written and the PTY output ring was observed for a bounded window; they do not prove that the agent took a turn. submit_confirmed is heuristic: true means sustained post-submit ring activity consistent with the composer accepting Enter, false means Enter produced no observable reaction within the confirmation window, null means unknown or not applicable. An agent that was already streaming output can produce a false positive."; +const INJECTION_EVIDENCE_SCOPE: &str = "These facts prove only that bytes were written and the PTY output ring was observed for bounded windows; they do not prove that the agent took a turn. submit_confirmed is heuristic: true means sustained post-submit ring activity consistent with the composer accepting Enter from a receiver that was quiet before the payload write, false means Enter produced no observable reaction within the confirmation window, null means unknown or not applicable. Sustained pre-write receiver activity makes an otherwise positive observation indeterminate."; const INJECTION_ACTIVITY_OBSERVATION_WINDOW: Duration = Duration::from_millis(250); const INJECTION_ACTIVITY_POLL_INTERVAL: Duration = Duration::from_millis(10); const SUBMIT_KEYSTROKE: &[u8] = b"\r"; @@ -27,6 +27,19 @@ const SUBMIT_CONFIRMATION_WINDOW: Duration = Duration::from_millis(1500); /// count as sustained rather than a single composer repaint. const SUBMIT_CONFIRMATION_MIN_SPAN: Duration = Duration::from_millis(250); +struct PreWriteActivityBaseline { + pty_observation_available: bool, + change_offsets: Vec, + observation_elapsed_ms: u64, +} + +impl PreWriteActivityBaseline { + fn receiver_busy(&self) -> Option { + self.pty_observation_available + .then_some(pre_write_activity_is_sustained(&self.change_offsets)) + } +} + struct InjectionObservation { pty_observation_available: bool, pty_output_bytes_after: Option, @@ -40,6 +53,7 @@ struct InjectionObservation { } struct InjectionMeasurement { + pre_write_baseline: PreWriteActivityBaseline, observation: InjectionObservation, submit_attempts: usize, submit_bytes_written: usize, @@ -123,21 +137,80 @@ impl InjectionObservation { } } -/// Classify post-Enter ring behaviour into the tri-state delivery signal (#256). +/// Classify post-Enter ring behaviour into the tri-state delivery signal (#256, #260). /// -/// `change_offsets` holds the elapsed time of every poll at which the ring content +/// `post_submit_change_offsets` holds the elapsed time of every poll at which the ring content /// differed from the previous poll. Changes spread over at least /// [`SUBMIT_CONFIRMATION_MIN_SPAN`] are the signature of a composer that accepted Enter /// and started a turn. A short isolated burst is ambiguous — a swallowed Enter also /// repaints the composer once. Zero changes mean the Enter provably produced no visible -/// reaction, which a live TUI never does for an accepted submit. -fn classify_submit_confirmation(change_offsets: &[Duration]) -> (Option, &'static str) { - match change_offsets { +/// reaction, which a live TUI never does for an accepted submit. Two or more changes in +/// the fixed pre-write baseline identify an already-streaming receiver; only a would-be +/// positive is downgraded because that output cannot be attributed to Enter. +fn classify_submit_confirmation( + post_submit_change_offsets: &[Duration], + pre_write_change_offsets: &[Duration], +) -> (Option, &'static str) { + let classification = match post_submit_change_offsets { [] => (Some(false), "no-post-submit-activity"), [first, .., last] if last.saturating_sub(*first) >= SUBMIT_CONFIRMATION_MIN_SPAN => { (Some(true), "sustained-post-submit-activity") } _ => (None, "ambiguous-post-submit-activity"), + }; + + if classification.0 == Some(true) && pre_write_activity_is_sustained(pre_write_change_offsets) { + (None, "busy-receiver-indeterminate") + } else { + classification + } +} + +fn pre_write_activity_is_sustained(change_offsets: &[Duration]) -> bool { + matches!(change_offsets, [_, _, ..]) +} + +fn post_submit_activity_is_sustained(change_offsets: &[Duration]) -> bool { + matches!( + change_offsets, + [first, .., last] + if last.saturating_sub(*first) >= SUBMIT_CONFIRMATION_MIN_SPAN + ) +} + +async fn observe_pre_write_activity(state: &AppState, agent_id: &str) -> PreWriteActivityBaseline { + let observation_started = Instant::now(); + let observation_deadline = observation_started + INJECTION_ACTIVITY_OBSERVATION_WINDOW; + let Some(mut previous) = state.pty_manager.read().recent_output(agent_id) else { + return PreWriteActivityBaseline { + pty_observation_available: false, + change_offsets: Vec::new(), + observation_elapsed_ms: 0, + }; + }; + let mut change_offsets = Vec::new(); + let mut pty_observation_available = true; + + loop { + let remaining = observation_deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + break; + } + tokio::time::sleep(INJECTION_ACTIVITY_POLL_INTERVAL.min(remaining)).await; + let Some(current) = state.pty_manager.read().recent_output(agent_id) else { + pty_observation_available = false; + break; + }; + if current != previous { + change_offsets.push(observation_started.elapsed()); + previous = current; + } + } + + PreWriteActivityBaseline { + pty_observation_available, + change_offsets, + observation_elapsed_ms: observation_started.elapsed().as_millis() as u64, } } @@ -146,6 +219,7 @@ async fn observe_submit( agent_id: &str, submit_keystroke_issued: bool, pty_output_after_write: Option<&str>, + pre_write_change_offsets: &[Duration], ) -> InjectionObservation { if !submit_keystroke_issued { return InjectionObservation::unobserved( @@ -189,7 +263,7 @@ async fn observe_submit( } output_after = Some(current); // Once the sustained-activity criterion is met, the verdict cannot change. - if classify_submit_confirmation(&change_offsets).0 == Some(true) { + if post_submit_activity_is_sustained(&change_offsets) { break; } } @@ -197,7 +271,7 @@ async fn observe_submit( let (submit_confirmed, submit_confirmation_basis) = if ring_lost { (None, "pty-unobservable") } else { - classify_submit_confirmation(&change_offsets) + classify_submit_confirmation(&change_offsets, pre_write_change_offsets) }; // The legacy activity fields keep their pre-#256 meaning: did the ring change within @@ -226,12 +300,14 @@ async fn observe_injection_with_retry( state: &AppState, agent_id: &str, receipt: &InjectionReceipt, + pre_write_baseline: PreWriteActivityBaseline, ) -> InjectionMeasurement { let mut observation = observe_submit( state, agent_id, receipt.submit_keystroke_issued, receipt.pty_output_after_write.as_deref(), + &pre_write_baseline.change_offsets, ) .await; let mut submit_attempts = usize::from(receipt.submit_keystroke_issued); @@ -262,8 +338,14 @@ async fn observe_injection_with_retry( // (including the Windows test stub) mirror the CR into the ring; observing // from the old baseline would misclassify that self-echo as receiver activity. let retry_baseline = state.pty_manager.read().recent_output(agent_id); - let mut retry_observation = - observe_submit(state, agent_id, true, retry_baseline.as_deref()).await; + let mut retry_observation = observe_submit( + state, + agent_id, + true, + retry_baseline.as_deref(), + &pre_write_baseline.change_offsets, + ) + .await; retry_observation.submit_confirmation_elapsed_ms = first_confirmation_elapsed_ms .saturating_add(retry_observation.submit_confirmation_elapsed_ms); observation = retry_observation; @@ -278,6 +360,7 @@ async fn observe_injection_with_retry( } InjectionMeasurement { + pre_write_baseline, observation, submit_attempts, submit_bytes_written, @@ -308,6 +391,7 @@ fn measured_response( receipt: &InjectionReceipt, measurement: &InjectionMeasurement, ) -> Value { + let pre_write_baseline = &measurement.pre_write_baseline; let observation = &measurement.observation; json!({ "status": "success", @@ -321,6 +405,11 @@ fn measured_response( "submit_attempts": measurement.submit_attempts, "submit_retry_failed": measurement.submit_retry_failure.is_some(), "submit_retry_failure": measurement.submit_retry_failure, + "pre_write_pty_observation_available": pre_write_baseline.pty_observation_available, + "pre_write_activity_change_count": pre_write_baseline.change_offsets.len(), + "receiver_busy_before_write": pre_write_baseline.receiver_busy(), + "pre_write_observation_window_ms": INJECTION_ACTIVITY_OBSERVATION_WINDOW.as_millis() as u64, + "pre_write_observation_elapsed_ms": pre_write_baseline.observation_elapsed_ms, "pty_observation_available": observation.pty_observation_available, "pty_output_bytes_before": receipt.pty_output_before.as_ref().map(String::len), "pty_output_bytes_after_write": receipt.pty_output_after_write.as_ref().map(String::len), @@ -373,6 +462,7 @@ pub async fn operator_inject( let target_agent_id = payload.target_agent_id.clone(); let transaction_lock = injection_transaction_lock(&state, &target_agent_id); let _transaction_guard = transaction_lock.lock().await; + let pre_write_baseline = observe_pre_write_activity(&state, &target_agent_id).await; let injection_target_agent_id = target_agent_id.clone(); let manager = Arc::clone(&state.injection_manager); let injection_session_id = id.clone(); @@ -387,7 +477,8 @@ pub async fn operator_inject( .await .map_err(|error| ApiError::internal(format!("Injection task failed: {error}")))?; let receipt = injection_result.map_err(|error| ApiError::internal(error.to_string()))?; - let measurement = observe_injection_with_retry(&state, &target_agent_id, &receipt).await; + let measurement = + observe_injection_with_retry(&state, &target_agent_id, &receipt, pre_write_baseline).await; record_injection_submission(&state, &id, &target_agent_id, &receipt); Ok(Json(measured_response( @@ -409,6 +500,7 @@ pub async fn queen_inject( let target_worker_id = payload.target_worker_id.clone(); let transaction_lock = injection_transaction_lock(&state, &target_worker_id); let _transaction_guard = transaction_lock.lock().await; + let pre_write_baseline = observe_pre_write_activity(&state, &target_worker_id).await; let injection_target_worker_id = target_worker_id.clone(); let manager = Arc::clone(&state.injection_manager); let injection_session_id = id.clone(); @@ -424,7 +516,8 @@ pub async fn queen_inject( .await .map_err(|error| ApiError::internal(format!("Injection task failed: {error}")))?; let receipt = injection_result.map_err(map_injection_error)?; - let measurement = observe_injection_with_retry(&state, &target_worker_id, &receipt).await; + let measurement = + observe_injection_with_retry(&state, &target_worker_id, &receipt, pre_write_baseline).await; record_injection_submission(&state, &id, &target_worker_id, &receipt); Ok(Json(measured_response( @@ -446,6 +539,7 @@ pub async fn evaluator_inject( let target_agent_id = payload.target_agent_id.clone(); let transaction_lock = injection_transaction_lock(&state, &target_agent_id); let _transaction_guard = transaction_lock.lock().await; + let pre_write_baseline = observe_pre_write_activity(&state, &target_agent_id).await; let injection_target_agent_id = target_agent_id.clone(); let manager = Arc::clone(&state.injection_manager); let injection_session_id = id.clone(); @@ -461,7 +555,8 @@ pub async fn evaluator_inject( .await .map_err(|error| ApiError::internal(format!("Injection task failed: {error}")))?; let receipt = injection_result.map_err(map_injection_error)?; - let measurement = observe_injection_with_retry(&state, &target_agent_id, &receipt).await; + let measurement = + observe_injection_with_retry(&state, &target_agent_id, &receipt, pre_write_baseline).await; record_injection_submission(&state, &id, &target_agent_id, &receipt); Ok(Json(measured_response( @@ -697,6 +792,16 @@ mod tests { assert_eq!(response["submit_attempts"], 2); assert_eq!(response["submit_retry_failed"], false); assert_eq!(response["submit_retry_failure"], Value::Null); + assert_eq!(response["pre_write_pty_observation_available"], true); + assert_eq!(response["pre_write_activity_change_count"], 0); + assert_eq!(response["receiver_busy_before_write"], false); + assert_eq!(response["pre_write_observation_window_ms"], 250); + assert!( + response["pre_write_observation_elapsed_ms"] + .as_u64() + .unwrap() + >= 240 + ); assert_eq!(response["pty_observation_available"], true); assert_eq!(response["pty_output_bytes_before"], 0); // The stub mirrors stdin into the ring: 6-byte start marker + 5-byte payload @@ -868,6 +973,19 @@ mod tests { assert_eq!(response["submit_attempts"], 2, "{path}"); assert_eq!(response["submit_retry_failed"], false, "{path}"); assert_eq!(response["submit_retry_failure"], Value::Null, "{path}"); + assert_eq!( + response["pre_write_pty_observation_available"], true, + "{path}" + ); + assert!( + response["pre_write_activity_change_count"].is_number(), + "{path}" + ); + assert!( + response["receiver_busy_before_write"].is_boolean(), + "{path}" + ); + assert_eq!(response["pre_write_observation_window_ms"], 250, "{path}"); assert_eq!(response["pty_observation_available"], true, "{path}"); assert!(response["pty_output_bytes_before"].is_number(), "{path}"); assert!( @@ -980,9 +1098,9 @@ mod tests { let pty_manager = Arc::clone(&state.pty_manager); let repainter = tokio::spawn(async move { - // The initial submit holds a 50 ms adapter gap before capturing its receipt - // baseline. Wait well past that gap so this is post-submit receiver activity. - tokio::time::sleep(std::time::Duration::from_millis(200)).await; + // Wait past the 250 ms pre-write baseline and the 50 ms adapter gap so this + // remains one isolated post-submit receiver repaint. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; pty_manager .read() .record_output_for_test(AGENT_ID, b"one-frame;"); @@ -1039,10 +1157,15 @@ mod tests { ) .await }); - tokio::time::timeout(Duration::from_secs(2), retry_ready_rx) - .await - .expect("initial quiet confirmation window") - .expect("retry hook signal"); + tokio::time::timeout( + SUBMIT_CONFIRMATION_WINDOW + + INJECTION_ACTIVITY_OBSERVATION_WINDOW + + Duration::from_secs(1), + retry_ready_rx, + ) + .await + .expect("initial quiet confirmation window") + .expect("retry hook signal"); let staged = tokio::spawn(async move { post_inject( @@ -1125,16 +1248,18 @@ mod tests { )); } - /// #256: sustained post-submit ring activity flips submit_confirmed to true. A - /// background task plays the role of the child TUI repainting after it accepted - /// the Enter. + /// #256/#260: a receiver that stays quiet through the pre-write baseline can still + /// produce a confident positive from sustained post-submit activity. #[tokio::test] - async fn operator_inject_confirms_submit_on_sustained_post_submit_activity() { + async fn operator_inject_confirms_submit_when_quiet_before_sustained_activity() { let (_temp_dir, app, state) = setup_test_app(); let pty_manager = Arc::clone(&state.pty_manager); let repainter = tokio::spawn(async move { - for tick in 0..12u32 { + // Let the 250 ms pre-write baseline finish before simulating the child TUI's + // sustained reaction to Enter. + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + for tick in 0..8u32 { tokio::time::sleep(std::time::Duration::from_millis(50)).await; pty_manager .read() @@ -1151,6 +1276,8 @@ mod tests { repainter.abort(); assert_eq!(status, StatusCode::OK); + assert_eq!(response["pre_write_activity_change_count"], 0); + assert_eq!(response["receiver_busy_before_write"], false); assert_eq!(response["submit_confirmed"], true); assert_eq!( response["submit_confirmation_basis"], @@ -1173,32 +1300,119 @@ mod tests { ); } + /// #260: output already streaming before the payload write makes an otherwise + /// confident positive indeterminate. The retry remains keyed strictly to false, + /// so this busy receiver must receive exactly one Enter. + #[tokio::test] + async fn operator_inject_downgrades_sustained_activity_from_busy_receiver() { + let (_temp_dir, app, state) = setup_test_app(); + + let pty_manager = Arc::clone(&state.pty_manager); + let repainter = tokio::spawn(async move { + for tick in 0..16u32 { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + pty_manager + .read() + .record_output_for_test(AGENT_ID, format!("busy-frame-{tick};").as_bytes()); + } + }); + + let (status, response) = post_inject( + app, + "/api/sessions/inject-test/inject", + json!({ "target_agent_id": AGENT_ID, "message": "go" }), + ) + .await; + repainter.abort(); + + assert_eq!(status, StatusCode::OK); + assert!( + response["pre_write_activity_change_count"] + .as_u64() + .unwrap() + >= 2 + ); + assert_eq!(response["receiver_busy_before_write"], true); + assert_eq!(response["submit_confirmed"], Value::Null); + assert_eq!( + response["submit_confirmation_basis"], + "busy-receiver-indeterminate" + ); + assert_eq!( + response["submit_attempts"], 1, + "an indeterminate busy receiver must not trigger the Some(false)-only retry" + ); + assert_eq!(response["submit_bytes_written"], 1); + let writes = state + .pty_manager + .read() + .write_records_for_test(AGENT_ID) + .unwrap(); + assert_eq!( + writes + .iter() + .filter(|write| write.as_slice() == b"\r") + .count(), + 1 + ); + } + #[test] fn submit_confirmation_classifier_distinguishes_all_three_verdicts() { use std::time::Duration; assert_eq!( - classify_submit_confirmation(&[]), + classify_submit_confirmation(&[], &[]), (Some(false), "no-post-submit-activity") ); assert_eq!( - classify_submit_confirmation(&[Duration::from_millis(40)]), + classify_submit_confirmation(&[Duration::from_millis(40)], &[]), (None, "ambiguous-post-submit-activity"), "a single repaint also matches a swallowed Enter" ); assert_eq!( - classify_submit_confirmation(&[Duration::from_millis(40), Duration::from_millis(120)]), + classify_submit_confirmation( + &[Duration::from_millis(40), Duration::from_millis(120)], + &[] + ), (None, "ambiguous-post-submit-activity"), "a short burst is not sustained activity" ); assert_eq!( - classify_submit_confirmation(&[ - Duration::from_millis(40), - Duration::from_millis(120), - Duration::from_millis(290) - ]), + classify_submit_confirmation( + &[ + Duration::from_millis(40), + Duration::from_millis(120), + Duration::from_millis(290) + ], + &[] + ), (Some(true), "sustained-post-submit-activity") ); + + let busy_baseline = [Duration::from_millis(40), Duration::from_millis(120)]; + assert_eq!( + classify_submit_confirmation(&[], &busy_baseline), + (Some(false), "no-post-submit-activity"), + "a busy baseline does not change a confident negative" + ); + assert_eq!( + classify_submit_confirmation(&[Duration::from_millis(40)], &busy_baseline), + (None, "ambiguous-post-submit-activity"), + "a busy baseline does not change an ambiguous verdict" + ); + assert_eq!( + classify_submit_confirmation( + &[ + Duration::from_millis(40), + Duration::from_millis(120), + Duration::from_millis(290) + ], + &busy_baseline + ), + (None, "busy-receiver-indeterminate"), + "pre-write streaming invalidates only the would-be confident positive" + ); } #[test] diff --git a/src-tauri/src/http/handlers/work_graph.rs b/src-tauri/src/http/handlers/work_graph.rs index d1cd49a0..b5d39a7e 100644 --- a/src-tauri/src/http/handlers/work_graph.rs +++ b/src-tauri/src/http/handlers/work_graph.rs @@ -7,12 +7,18 @@ use axum::Json; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use crate::coordination::StateManager; +use crate::coordination::queue_manager::CompletionProvenance; +use crate::coordination::{HierarchyNode, StateManager}; use crate::http::error::ApiError; use crate::http::state::AppState; use crate::orchestrator::work_graph::archive::{list_archives, read_archive, WorkGraphArchive}; +use crate::orchestrator::work_graph::completion_ledger::{ + read_node_completion_facts, NodeCompletionProvenance, +}; use crate::orchestrator::work_graph::divergence::{compute_divergence, DivergenceSummary}; -use crate::orchestrator::work_graph::runtime::{mutation_log_snapshot, RuntimeOutcome}; +use crate::orchestrator::work_graph::runtime::{ + mutation_log_snapshot, CompletionEvidenceClass, RuntimeOutcome, +}; use crate::orchestrator::work_graph::{ topological_sort, BindingRef, CompositeExpansion, EdgeKind, EdgeProvenance, NodeContract, NodeKind, NodeStatus, TaskGraph, TaskId, WorkGraphOmission, WorkGraphOmissionReason, @@ -107,7 +113,10 @@ pub struct WorkGraphResponse { pub edges: Vec, pub waves: Vec>, pub status_by_node: BTreeMap, + pub completion_provenance: BTreeMap, + pub completion_source_refs: BTreeMap>, pub lane_assignment: BTreeMap, + pub agents_by_lane: BTreeMap>, pub critical_path: Vec, pub provenance_by_edge: Vec, pub divergence: Option, @@ -131,7 +140,15 @@ pub async fn get_work_graph( ))); } - let (source, graph, divergence, progress_by_node, omissions) = match query.source { + let ( + source, + graph, + divergence, + progress_by_node, + completion_provenance, + completion_source_refs, + omissions, + ) = match query.source { WorkGraphSourceSelector::Live => { let plan = read_live_graph(&session_dir)?.ok_or_else(|| { ApiError::not_found(format!("Live work graph not found: {session_id}")) @@ -145,7 +162,27 @@ pub async fn get_work_graph( graph_from_archive(&archive, query.view) } WorkGraphSourceSelector::Auto => { - if let Some(plan) = read_live_graph(&session_dir)? { + let is_terminal = state + .session_controller + .read() + .get_session(&session_id) + .is_some_and(|session| session.state.is_terminal()); + if is_terminal { + if let Some(archive) = latest_archive(&session_dir, &session_id)? { + graph_from_archive(&archive, query.view) + } else { + let plan = read_live_graph(&session_dir)?.ok_or_else(|| { + ApiError::not_found(format!("Work graph not found: {session_id}")) + })?; + let mut live = graph_from_live_state(&state, &session_id, query.view, plan)?; + live.6.push(WorkGraphOmission::new( + WorkGraphOmissionReason::SourceUnreadable, + 1, + vec!["archive:missing".to_string()], + )); + live + } + } else if let Some(plan) = read_live_graph(&session_dir)? { graph_from_live_state(&state, &session_id, query.view, plan)? } else { let archive = latest_archive(&session_dir, &session_id)?.ok_or_else(|| { @@ -156,12 +193,19 @@ pub async fn get_work_graph( } }; + let hierarchy = StateManager::new(session_dir) + .read_hierarchy() + .map_err(|error| ApiError::internal(format!("Failed to read agent hierarchy: {error}")))?; + let response = project_graph( query.view, source, graph, divergence, &progress_by_node, + completion_provenance, + completion_source_refs, + &hierarchy, omissions, )?; Ok(Json(response)) @@ -211,6 +255,8 @@ fn graph_from_archive( TaskGraph, Option, BTreeMap, + BTreeMap, + BTreeMap>, Vec, ) { match view { @@ -222,6 +268,8 @@ fn graph_from_archive( .unwrap_or_else(|| archive.runtime_graph.clone()), None, BTreeMap::new(), + BTreeMap::new(), + BTreeMap::new(), Vec::new(), ), WorkGraphView::Runtime => ( @@ -229,6 +277,8 @@ fn graph_from_archive( archive.runtime_graph.clone(), None, archive_progress_by_node(archive), + archive_completion_provenance(archive), + archive_completion_source_refs(archive), Vec::new(), ), WorkGraphView::Divergence => ( @@ -236,11 +286,86 @@ fn graph_from_archive( archive.runtime_graph.clone(), Some(archive.divergence.clone()), archive_progress_by_node(archive), + archive_completion_provenance(archive), + archive_completion_source_refs(archive), Vec::new(), ), } } +fn archive_completion_source_refs(archive: &WorkGraphArchive) -> BTreeMap> { + let structural_node_ids: BTreeSet<&str> = archive + .runtime_graph + .nodes + .iter() + .map(|node| node.id.as_str()) + .collect(); + let mut refs_by_node = BTreeMap::>::new(); + for outcome in archive + .outcomes + .iter() + .filter(|outcome| outcome.completion_evidence.is_some()) + { + let task_id = if structural_node_ids.contains(outcome.subject_id.as_str()) { + Some(outcome.subject_id.clone()) + } else { + outcome + .task_id + .clone() + .filter(|task_id| structural_node_ids.contains(task_id.as_str())) + }; + let Some(task_id) = task_id else { + continue; + }; + refs_by_node + .entry(task_id) + .or_default() + .extend(outcome.source_refs.iter().cloned()); + } + for source_refs in refs_by_node.values_mut() { + source_refs.sort(); + source_refs.dedup(); + } + refs_by_node +} + +fn archive_completion_provenance( + archive: &WorkGraphArchive, +) -> BTreeMap { + let structural_node_ids: BTreeSet<&str> = archive + .runtime_graph + .nodes + .iter() + .map(|node| node.id.as_str()) + .collect(); + let mut provenance = BTreeMap::new(); + for outcome in &archive.outcomes { + let Some(class) = outcome.completion_evidence else { + continue; + }; + let task_id = if structural_node_ids.contains(outcome.subject_id.as_str()) { + Some(outcome.subject_id.clone()) + } else { + outcome + .task_id + .clone() + .filter(|task_id| structural_node_ids.contains(task_id.as_str())) + }; + let Some(task_id) = task_id else { + continue; + }; + let candidate = match class { + CompletionEvidenceClass::Observed => CompletionProvenance::Observed, + CompletionEvidenceClass::Inferred => CompletionProvenance::Inferred, + }; + let existing = provenance.get(&task_id).copied(); + if existing != Some(CompletionProvenance::Observed) { + provenance.insert(task_id, candidate); + } + } + provenance +} + fn archive_progress_by_node(archive: &WorkGraphArchive) -> BTreeMap { let structural_node_ids: BTreeSet<&str> = archive .runtime_graph @@ -301,6 +426,8 @@ fn graph_from_live_state( TaskGraph, Option, BTreeMap, + BTreeMap, + BTreeMap>, Vec, ), ApiError, @@ -311,17 +438,64 @@ fn graph_from_live_state( plan, None, BTreeMap::new(), + BTreeMap::new(), + BTreeMap::new(), Vec::new(), )); } - let runtime = state + let (mut runtime, mut completion_provenance) = state .queue_manager - .project_queue_statuses(session_id, &plan) + .project_queue_statuses_for_view(session_id, &plan) .map_err(|error| { ApiError::internal(format!("Failed to project live work-graph status: {error}")) })?; - let (divergence, omissions) = if matches!(view, WorkGraphView::Divergence) { + let (completion_facts, mut ledger_source_omissions) = + match read_node_completion_facts(&state.storage.session_dir(session_id)) { + Ok(facts) => (facts, Vec::new()), + Err(error) => { + let mut omission = WorkGraphOmission::new( + WorkGraphOmissionReason::SourceUnreadable, + 1, + vec!["state/work-graph-completions.jsonl".to_string()], + ); + omission.detail = format!( + "declared node completions could not be read and were omitted: {error}" + ); + (Vec::new(), vec![omission]) + } + }; + let mut completion_source_refs = BTreeMap::>::new(); + let mut declared_progress = BTreeMap::::new(); + let mut declared_omissions = Vec::new(); + for fact in completion_facts { + if let Some(node) = runtime + .nodes + .iter_mut() + .find(|node| node.id == fact.task_id) + { + node.status = NodeStatus::Completed; + completion_provenance.insert(fact.task_id.clone(), CompletionProvenance::Declared); + completion_source_refs + .entry(fact.task_id.clone()) + .or_default() + .push(fact.source_ref()); + declared_progress.insert( + fact.task_id, + WorkGraphNodeProgress { + started_at: None, + finished_at: Some(fact.completed_at), + attempts: 1, + agent_id: Some(fact.agent_id), + last_heartbeat_at: (fact.provenance == NodeCompletionProvenance::Heartbeat) + .then_some(fact.completed_at), + }, + ); + } else { + declared_omissions.push(format!("{}:task:{}", fact.source_ref(), fact.task_id)); + } + } + let (divergence, mut omissions) = if matches!(view, WorkGraphView::Divergence) { let mutation_snapshot = mutation_log_snapshot(session_id); let omissions = (!mutation_snapshot.tracked) .then(|| { @@ -346,12 +520,27 @@ fn graph_from_live_state( } else { (None, Vec::new()) }; - let progress = live_progress_by_node(state, session_id)?; + omissions.append(&mut ledger_source_omissions); + if !declared_omissions.is_empty() { + let mut omission = WorkGraphOmission::new( + WorkGraphOmissionReason::CompletionUnresolved, + declared_omissions.len(), + declared_omissions, + ); + omission.detail = + "declared completion referenced a node absent from the current live work graph" + .to_string(); + omissions.push(omission); + } + let mut progress = live_progress_by_node(state, session_id)?; + progress.extend(declared_progress); Ok(( WorkGraphSource::Live, runtime, divergence, progress, + completion_provenance, + completion_source_refs, omissions, )) } @@ -426,6 +615,9 @@ fn project_graph( graph: TaskGraph, divergence: Option, progress_by_node: &BTreeMap, + completion_provenance: BTreeMap, + completion_source_refs: BTreeMap>, + hierarchy: &[HierarchyNode], supplemental_omissions: Vec, ) -> Result { let order = topological_sort(&graph).map_err(|error| { @@ -470,10 +662,48 @@ fn project_graph( .iter() .map(|node| (node.id.clone(), node.status)) .collect(); + let principal_by_agent: BTreeMap<&str, &str> = hierarchy + .iter() + .filter_map(|agent| { + agent + .principal + .as_deref() + .map(|principal| (agent.id.as_str(), principal)) + }) + .collect(); + let mut agents_by_lane = BTreeMap::>::new(); + for agent in hierarchy { + if let Some(principal) = agent.principal.as_deref() { + agents_by_lane + .entry(principal.to_string()) + .or_default() + .push(agent.id.clone()); + } + } + for agents in agents_by_lane.values_mut() { + agents.sort(); + agents.dedup(); + } let lane_assignment = graph .nodes .iter() - .map(|node| (node.id.clone(), node.binding.clone())) + .map(|node| { + let principal = binding_value(&node.binding); + let progress_agent = progress_by_node + .get(&node.id) + .and_then(|progress| progress.agent_id.as_deref()) + .filter(|agent_id| principal_by_agent.get(agent_id).copied() == Some(principal)); + let observed_agent = progress_agent.or_else(|| { + agents_by_lane + .get(principal) + .and_then(|agents| agents.first()) + .map(String::as_str) + }); + let assignment = observed_agent + .map(|agent_id| BindingRef::Role(agent_id.to_string())) + .unwrap_or_else(|| node.binding.clone()); + (node.id.clone(), assignment) + }) .collect(); let provenance_by_edge = graph .edges @@ -493,7 +723,10 @@ fn project_graph( edges, waves, status_by_node, + completion_provenance, + completion_source_refs, lane_assignment, + agents_by_lane, critical_path, provenance_by_edge, divergence, @@ -501,6 +734,12 @@ fn project_graph( }) } +fn binding_value(binding: &BindingRef) -> &str { + match binding { + BindingRef::Role(value) | BindingRef::Zone(value) => value, + } +} + fn topological_waves(graph: &TaskGraph, order: &[TaskId]) -> Vec> { let known: BTreeSet<&str> = graph.nodes.iter().map(|node| node.id.as_str()).collect(); let mut dependents: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); diff --git a/src-tauri/src/http/tests_wg_api.rs b/src-tauri/src/http/tests_wg_api.rs index 08dfd133..a83941e5 100644 --- a/src-tauri/src/http/tests_wg_api.rs +++ b/src-tauri/src/http/tests_wg_api.rs @@ -11,7 +11,7 @@ use serde_json::{json, Value}; use tempfile::TempDir; use tower::ServiceExt; -use crate::coordination::{InjectionManager, QueueManager, StateManager}; +use crate::coordination::{HierarchyNode, InjectionManager, QueueManager, StateManager}; use crate::domain::HiveExecutionPolicy; use crate::events::EventBus; use crate::http::routes::create_router; @@ -21,7 +21,8 @@ use crate::orchestrator::work_graph::archive::{ }; use crate::orchestrator::work_graph::divergence::DivergenceSummary; use crate::orchestrator::work_graph::runtime::{ - record_graph_change, GraphMutationType, RuntimeOutcome, RuntimeOutcomeStatus, + record_graph_change, CompletionEvidenceClass, GraphMutationType, RuntimeOutcome, + RuntimeOutcomeStatus, }; use crate::orchestrator::work_graph::{ BindingRef, CompositeExpansion, EdgeKind, EdgeProvenance, NodeContract, NodeKind, NodeStatus, @@ -380,7 +381,7 @@ async fn node_payload_shape_is_identical_across_live_views_and_archive() { } #[tokio::test] -async fn source_selectors_choose_live_archive_and_auto_prefers_live() { +async fn source_selectors_choose_live_archive_and_auto_tracks_session_lifecycle() { const SESSION_ID: &str = "wg-api-source-selector"; let app = test_app().await; let session_dir = app @@ -415,6 +416,10 @@ async fn source_selectors_choose_live_archive_and_auto_prefers_live() { state .write_work_graph(&live_graph) .expect("persisted newer live graph"); + app.state + .session_controller + .read() + .insert_test_session(running_session_with_agent(SESSION_ID, "selector-queen")); for (selector, expected_source, expected_node) in [ ("source=live", "live", "live-node"), @@ -436,6 +441,26 @@ async fn source_selectors_choose_live_archive_and_auto_prefers_live() { ); } + let mut terminal = app + .state + .session_controller + .read() + .get_session(SESSION_ID) + .expect("running selector session"); + terminal.state = SessionState::Completed; + app.state + .session_controller + .read() + .insert_test_session(terminal); + let (status, body, response) = get( + &app.router, + &format!("/api/sessions/{SESSION_ID}/work-graph?view=plan&source=auto"), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(response["source"], "archive"); + assert_eq!(response["nodes"][0]["id"], "archived-node"); + std::fs::remove_file(session_dir.join("state").join("work-graph.json")) .expect("remove only the temporary live graph fixture"); let (status, body, response) = get( @@ -448,6 +473,51 @@ async fn source_selectors_choose_live_archive_and_auto_prefers_live() { assert_eq!(response["nodes"][0]["id"], "archived-node"); } +#[tokio::test] +async fn terminal_session_without_archive_serves_live_with_typed_omission() { + const SESSION_ID: &str = "wg-api-terminal-missing-archive"; + let app = test_app().await; + let session_dir = app + .storage() + .create_session_dir(SESSION_ID) + .expect("session directory"); + StateManager::new(session_dir) + .write_work_graph(&TaskGraph::new( + vec![node( + "live-terminal-node", + BindingRef::Role("backend".to_string()), + NodeStatus::Completed, + "live-terminal", + )], + vec![], + )) + .expect("persisted live terminal graph"); + let mut session = running_session_with_agent(SESSION_ID, "terminal-queen"); + session.state = SessionState::Failed("expected test failure".to_string()); + app.state + .session_controller + .read() + .insert_test_session(session); + + let (status, body, response) = get( + &app.router, + &format!("/api/sessions/{SESSION_ID}/work-graph?view=plan&source=auto"), + ) + .await; + + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(response["source"], "live"); + assert_eq!(response["nodes"][0]["id"], "live-terminal-node"); + assert!(response["omissions"] + .as_array() + .expect("typed omissions") + .iter() + .any(|omission| { + omission["reason"] == "source_unreadable" + && omission["examples"] == json!(["archive:missing"]) + })); +} + #[tokio::test] async fn runtime_graph_projects_a_blocked_subtree_from_the_durable_queue() { const SESSION_ID: &str = "wg-api-mid-flight"; @@ -548,6 +618,111 @@ async fn runtime_graph_projects_a_blocked_subtree_from_the_durable_queue() { ); } +#[tokio::test] +async fn live_view_preserves_unbacked_terminal_status_and_reports_provenance() { + const SESSION_ID: &str = "wg-api-view-provenance"; + let app = test_app().await; + let session_dir = app + .storage() + .create_session_dir(SESSION_ID) + .expect("session directory"); + let graph = TaskGraph::new( + vec![ + node( + "plan-completed", + BindingRef::Role("backend".to_string()), + NodeStatus::Completed, + "plan-completed", + ), + node( + "queue-completed", + BindingRef::Role("backend".to_string()), + NodeStatus::Pending, + "queue-completed", + ), + node( + "dependent", + BindingRef::Role("reviewer".to_string()), + NodeStatus::Pending, + "dependent", + ), + ], + vec![WorkEdge::new( + "plan-completed", + "dependent", + EdgeKind::DependsOn, + EdgeProvenance::Planner, + )], + ); + let state_manager = StateManager::new(session_dir); + state_manager + .write_work_graph(&graph) + .expect("persisted live graph"); + state_manager + .update_hierarchy(&[ + HierarchyNode { + id: "worker-queue-completed".to_string(), + role: "Worker-1".to_string(), + principal: Some("backend".to_string()), + parent_id: Some("view-queen".to_string()), + children: Vec::new(), + }, + HierarchyNode { + id: "view-queen".to_string(), + role: "Queen".to_string(), + principal: None, + parent_id: None, + children: vec!["worker-queue-completed".to_string()], + }, + ]) + .expect("persisted observed principal mapping"); + app.state + .queue_manager + .repo() + .enqueue(&queue_row( + SESSION_ID, + "run-queue-completed", + "queue-completed", + QueueStatus::Finalized, + 1, + )) + .expect("finalized queue evidence"); + + let (status, body, response) = get( + &app.router, + &format!("/api/sessions/{SESSION_ID}/work-graph?view=runtime&source=live"), + ) + .await; + + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(response["status_by_node"]["plan-completed"], "completed"); + assert_eq!(response["status_by_node"]["queue-completed"], "completed"); + assert_eq!( + response["status_by_node"]["dependent"], "pending", + "the view projection must not feed persisted completion into readiness promotion" + ); + assert_eq!(response["completion_provenance"]["plan-completed"], "plan"); + assert_eq!(response["completion_provenance"]["queue-completed"], "queue"); + assert_eq!( + response["lane_assignment"]["queue-completed"], + json!({"kind":"role","value":"worker-queue-completed"}) + ); + assert_eq!( + response["agents_by_lane"]["backend"], + json!(["worker-queue-completed"]) + ); + assert!(response["omissions"] + .as_array() + .expect("typed omissions") + .iter() + .any(|omission| { + omission["reason"] == "resolution_incomplete" + && omission["examples"] + .as_array() + .is_some_and(|examples| examples.iter().any(|example| example == "queue:plan-completed")) + })); +} + #[tokio::test] async fn runtime_and_divergence_progress_preserve_queue_evidence_and_null_timing() { const SESSION_ID: &str = "wg-api-live-progress"; @@ -764,6 +939,7 @@ async fn archived_progress_keeps_target_and_expansion_outcomes_distinct_in_any_o attempt_count: 1, effects: Vec::new(), source_refs: vec!["event:review-a".to_string()], + completion_evidence: Some(CompletionEvidenceClass::Inferred), }; let task_outcome = RuntimeOutcome { subject_id: "task-a".to_string(), @@ -775,6 +951,7 @@ async fn archived_progress_keeps_target_and_expansion_outcomes_distinct_in_any_o attempt_count: 3, effects: Vec::new(), source_refs: vec!["event:task-a".to_string()], + completion_evidence: Some(CompletionEvidenceClass::Observed), }; let fixture_orders = [ ( @@ -829,6 +1006,16 @@ async fn archived_progress_keeps_target_and_expansion_outcomes_distinct_in_any_o .find(|node| node["id"] == "review-a") .expect("review node")["progress"]; assert_eq!(task_progress["attempts"], 3, "{body}"); + assert_eq!(response["completion_provenance"]["task-a"], "observed"); + assert_eq!(response["completion_provenance"]["review-a"], "inferred"); + assert_eq!( + response["completion_source_refs"]["task-a"], + json!(["event:task-a"]) + ); + assert_eq!( + response["completion_source_refs"]["review-a"], + json!(["event:review-a"]) + ); assert_eq!(task_progress["agent_id"], "agent-task", "{body}"); assert_eq!(task_progress["started_at"], json!(task_started), "{body}"); assert_eq!(task_progress["finished_at"], json!(task_finished), "{body}"); @@ -898,10 +1085,16 @@ async fn completed_session_falls_back_to_archive_with_divergence() { let completion = archive_completed_session(app.storage().base_dir(), None, SESSION_ID) .expect("completed archive"); assert!(completion.created); + let mut session = running_session_with_agent(SESSION_ID, "archived-queen"); + session.state = SessionState::Completed; + app.state + .session_controller + .read() + .insert_test_session(session); let (status, body, response) = get( &app.router, - &format!("/api/sessions/{SESSION_ID}/work-graph?view=divergence&source=archive"), + &format!("/api/sessions/{SESSION_ID}/work-graph?view=divergence&source=auto"), ) .await; diff --git a/src-tauri/src/http/tests_wg_queue.rs b/src-tauri/src/http/tests_wg_queue.rs index fdd7aa49..0104e204 100644 --- a/src-tauri/src/http/tests_wg_queue.rs +++ b/src-tauri/src/http/tests_wg_queue.rs @@ -1,7 +1,7 @@ //! Readiness-queue tests for issue #212, owned by WS-4. -use std::path::Path; use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Barrier}; @@ -13,8 +13,8 @@ use serde_json::json; use tower::ServiceExt; use crate::coordination::queue_manager::{ - ClaimOutcome, QueueManager, FIRST_HEARTBEAT_GRACE_MS, - FIRST_HEARTBEAT_LATENCY_MS_BY_CLI, STUCK_CUTOFF_MS, + ClaimOutcome, QueueManager, FIRST_HEARTBEAT_GRACE_MS, FIRST_HEARTBEAT_LATENCY_MS_BY_CLI, + STUCK_CUTOFF_MS, }; use crate::coordination::{InjectionManager, StateManager}; use crate::domain::event::{EventType, Severity}; @@ -24,18 +24,22 @@ use crate::http::handlers::heartbeats::PostHeartbeatRequest; use crate::http::handlers::workers::AddWorkerRequest; use crate::http::routes::create_router; use crate::http::state::AppState; +use crate::orchestrator::work_graph::completion_ledger::{ + read_node_completion_facts, NodeCompletionProvenance, +}; +use crate::orchestrator::work_graph::plan_parse::promote_initial_ready_nodes; use crate::orchestrator::work_graph::review::checkpoint_aware_claimable_nodes; use crate::orchestrator::work_graph::{ - BindingRef, EdgeKind, EdgeProvenance, NodeContract, NodeKind, NodeStatus, TaskGraph, - WorkEdge, WorkNode, + BindingRef, EdgeKind, EdgeProvenance, NodeContract, NodeKind, NodeStatus, TaskGraph, WorkEdge, + WorkNode, }; use crate::pty::{AgentConfig, AgentRole, AgentStatus, PtyManager}; use crate::session::{ AgentInfo, AuthStrategy, Session, SessionController, SessionState, SessionType, }; use crate::storage::queue::{ - QueueConflictAction, QueueConflictCoverage, QueueConflictRow, QueueResolutionUpdate, - QueueRow, QueueStatus, + QueueConflictAction, QueueConflictCoverage, QueueConflictRow, QueueResolutionUpdate, QueueRow, + QueueStatus, }; use crate::storage::{ApplicationStateDb, QueueRepo, SessionStorage}; @@ -49,13 +53,7 @@ fn queue_repo() -> QueueRepo { } fn queued_row(id: &str, worker_id: &str, task_id: Option<&str>, created_at: i64) -> QueueRow { - queue_row( - id, - worker_id, - task_id, - QueueStatus::Queued, - created_at, - ) + queue_row(id, worker_id, task_id, QueueStatus::Queued, created_at) } fn queue_row( @@ -143,13 +141,8 @@ fn race_bound_claims( let worker_id = worker_id.to_string(); std::thread::spawn(move || { barrier.wait(); - repo.try_claim_for_worker( - &id, - Some(&worker_id), - now_ms - 90_000, - now_ms, - ) - .unwrap() + repo.try_claim_for_worker(&id, Some(&worker_id), now_ms - 90_000, now_ms) + .unwrap() }) }) .collect::>(); @@ -164,10 +157,7 @@ fn race_bound_claims( async fn slow_spawn_reservation_prevents_stale_reclaim_and_second_spawn() { let temp = tempfile::tempdir().unwrap(); let repo = Arc::new(queue_repo()); - let manager = QueueManager::new( - Arc::clone(&repo), - EventBus::new(temp.path().to_path_buf()), - ); + let manager = QueueManager::new(Arc::clone(&repo), EventBus::new(temp.path().to_path_buf())); manager .enqueue_worker( "run-slow", @@ -233,7 +223,9 @@ async fn slow_spawn_reservation_prevents_stale_reclaim_and_second_spawn() { "a slow epoch-1 spawn must not be reclaimed and double-spawned as epoch 2" ); assert_eq!(protected.heartbeat_at, Some(claimed_heartbeat + 2)); - assert!(manager.complete_spawn_handoff("run-slow", 1, "worker-1").unwrap()); + assert!(manager + .complete_spawn_handoff("run-slow", 1, "worker-1") + .unwrap()); assert!(manager.spawn_in_flight("run-slow").is_none()); } @@ -241,10 +233,7 @@ async fn slow_spawn_reservation_prevents_stale_reclaim_and_second_spawn() { async fn stale_epoch_one_cannot_release_or_reacquire_after_epoch_two_exists() { let temp = tempfile::tempdir().unwrap(); let repo = Arc::new(queue_repo()); - let manager = QueueManager::new( - Arc::clone(&repo), - EventBus::new(temp.path().to_path_buf()), - ); + let manager = QueueManager::new(Arc::clone(&repo), EventBus::new(temp.path().to_path_buf())); manager .enqueue_worker( "run-epoch", @@ -287,7 +276,9 @@ async fn stale_epoch_one_cannot_release_or_reacquire_after_epoch_two_exists() { .unwrap(), crate::coordination::ReleaseAfterFailure::NotHeld ); - assert!(!manager.complete_spawn_handoff("run-epoch", 1, "worker-1").unwrap()); + assert!(!manager + .complete_spawn_handoff("run-epoch", 1, "worker-1") + .unwrap()); assert_eq!( manager.spawn_in_flight("run-epoch").unwrap().epoch, 2, @@ -314,9 +305,16 @@ async fn stale_epoch_one_cannot_release_or_reacquire_after_epoch_two_exists() { let row = repo.get_row("run-epoch").unwrap().unwrap(); assert_eq!( (reclaimed, retry, row.attempts, row.worker_id.as_str()), - (Vec::::new(), ClaimOutcome::AlreadyClaimed, 2, "worker-2") + ( + Vec::::new(), + ClaimOutcome::AlreadyClaimed, + 2, + "worker-2" + ) ); - assert!(manager.complete_spawn_handoff("run-epoch", 2, "worker-2").unwrap()); + assert!(manager + .complete_spawn_handoff("run-epoch", 2, "worker-2") + .unwrap()); // Manual recovery resets only the independent spawn-failure budget, never the fencing // epoch. Reuse worker-1 deliberately: even matching the old worker ID cannot make the stale @@ -338,7 +336,9 @@ async fn stale_epoch_one_cannot_release_or_reacquire_after_epoch_two_exists() { assert!(!repo .refresh_claimed_spawn("run-epoch", 1, "worker-1", 2) .unwrap()); - assert!(!manager.complete_spawn_handoff("run-epoch", 1, "worker-1").unwrap()); + assert!(!manager + .complete_spawn_handoff("run-epoch", 1, "worker-1") + .unwrap()); assert_eq!( manager .release_after_failed_spawn(SESSION_ID, "worker-1", "run-epoch", 1) @@ -353,10 +353,7 @@ async fn stale_epoch_one_cannot_release_or_reacquire_after_epoch_two_exists() { async fn spawn_failure_budget_resets_without_resetting_claim_epoch() { let temp = tempfile::tempdir().unwrap(); let repo = Arc::new(queue_repo()); - let manager = QueueManager::new( - Arc::clone(&repo), - EventBus::new(temp.path().to_path_buf()), - ); + let manager = QueueManager::new(Arc::clone(&repo), EventBus::new(temp.path().to_path_buf())); manager .enqueue_worker( "run-budget", @@ -373,12 +370,7 @@ async fn spawn_failure_budget_resets_without_resetting_claim_epoch() { for epoch in 1..=3 { assert_eq!( manager - .claim_and_reserve_spawn( - "run-budget", - SESSION_ID, - "worker-1", - Some("BUDGET"), - ) + .claim_and_reserve_spawn("run-budget", SESSION_ID, "worker-1", Some("BUDGET"),) .await .unwrap(), ClaimOutcome::Claimed { epoch } @@ -397,7 +389,10 @@ async fn spawn_failure_budget_resets_without_resetting_claim_epoch() { } } let exhausted = repo.get_row("run-budget").unwrap().unwrap(); - assert_eq!((exhausted.status, exhausted.attempts), (QueueStatus::Failed, 3)); + assert_eq!( + (exhausted.status, exhausted.attempts), + (QueueStatus::Failed, 3) + ); assert_eq!( manager.release_claim(SESSION_ID, "worker-1").await.unwrap(), @@ -407,12 +402,7 @@ async fn spawn_failure_budget_resets_without_resetting_claim_epoch() { ); assert_eq!( manager - .claim_and_reserve_spawn( - "run-budget", - SESSION_ID, - "worker-1", - Some("BUDGET"), - ) + .claim_and_reserve_spawn("run-budget", SESSION_ID, "worker-1", Some("BUDGET"),) .await .unwrap(), ClaimOutcome::Claimed { epoch: 4 }, @@ -433,10 +423,7 @@ async fn spawn_failure_budget_resets_without_resetting_claim_epoch() { async fn in_flight_spawn_denies_manual_and_same_process_reconcile_but_drops_on_restart() { let temp = tempfile::tempdir().unwrap(); let repo = Arc::new(queue_repo()); - let manager = QueueManager::new( - Arc::clone(&repo), - EventBus::new(temp.path().to_path_buf()), - ); + let manager = QueueManager::new(Arc::clone(&repo), EventBus::new(temp.path().to_path_buf())); manager .enqueue_worker( "run-restart", @@ -466,15 +453,15 @@ async fn in_flight_spawn_denies_manual_and_same_process_reconcile_but_drops_on_r crate::coordination::ReleaseOutcome::SpawnInFlight { epoch: 1 } ); assert!(manager.reconcile(SESSION_ID, &[]).await.unwrap().is_empty()); - assert_eq!(repo.get_row("run-restart").unwrap().unwrap().status, QueueStatus::Running); + assert_eq!( + repo.get_row("run-restart").unwrap().unwrap().status, + QueueStatus::Running + ); // A new manager models process restart: its in-memory marker set is empty, so the normal // durable startup reconciliation repairs the orphaned running row. drop(manager); - let restarted = QueueManager::new( - Arc::clone(&repo), - EventBus::new(temp.path().to_path_buf()), - ); + let restarted = QueueManager::new(Arc::clone(&repo), EventBus::new(temp.path().to_path_buf())); assert_eq!( restarted.reconcile(SESSION_ID, &[]).await.unwrap(), vec!["run-restart".to_string()] @@ -525,7 +512,10 @@ fn diamond_claims_are_dependency_gated_atomic_and_eventually_unblock() { let same_ready_row = race_claims(&repo, &["run-d", "run-d"], 90); assert_eq!( - same_ready_row.iter().filter(|epoch| epoch.is_some()).count(), + same_ready_row + .iter() + .filter(|epoch| epoch.is_some()) + .count(), 1, "exactly one concurrent claimer may win the same ready row: {same_ready_row:?}" ); @@ -535,19 +525,16 @@ fn diamond_claims_are_dependency_gated_atomic_and_eventually_unblock() { } fn serialized_conflict_rows(session_id: &str) -> Vec { - [ - ("T1", "T2"), - ("T2", "T1"), - ] - .into_iter() - .map(|(task_id, conflicting_task_id)| QueueConflictRow { - session_id: session_id.to_string(), - task_id: task_id.to_string(), - conflicting_task_id: conflicting_task_id.to_string(), - action: QueueConflictAction::Serialize, - reason: "T1 and T2 overlap src/shared.rs".to_string(), - }) - .collect() + [("T1", "T2"), ("T2", "T1")] + .into_iter() + .map(|(task_id, conflicting_task_id)| QueueConflictRow { + session_id: session_id.to_string(), + task_id: task_id.to_string(), + conflicting_task_id: conflicting_task_id.to_string(), + action: QueueConflictAction::Serialize, + reason: "T1 and T2 overlap src/shared.rs".to_string(), + }) + .collect() } #[test] @@ -596,8 +583,14 @@ fn materialized_serialize_conflict_is_atomic_and_partial_coverage_stays_visible( repo.record_heartbeat(SESSION_ID, winner_worker, "completed", 30) .unwrap(); assert!(repo.try_claim(loser_id, -90_000, 40).unwrap().is_some()); - assert_eq!(repo.get_row(winner_id).unwrap().unwrap().status, QueueStatus::Finalized); - assert_eq!(repo.snapshot(SESSION_ID).unwrap().conflict_coverage, Some(coverage)); + assert_eq!( + repo.get_row(winner_id).unwrap().unwrap().status, + QueueStatus::Finalized + ); + assert_eq!( + repo.snapshot(SESSION_ID).unwrap().conflict_coverage, + Some(coverage) + ); } #[test] @@ -737,7 +730,13 @@ fn unresolved_binding_blocks_until_exact_queued_intent_is_reconciled() { ) .unwrap(); assert_eq!(repo.try_claim("run-unknown", -90_000, 10).unwrap(), None); - assert_eq!(repo.snapshot(SESSION_ID).unwrap().resolution_incomplete.len(), 1); + assert_eq!( + repo.snapshot(SESSION_ID) + .unwrap() + .resolution_incomplete + .len(), + 1 + ); // A wrong binding under the same queue id cannot clear or rewrite the omission. let wrong = queued_row("run-unknown", "pending:run-unknown", Some("OTHER"), 2); @@ -753,17 +752,13 @@ fn unresolved_binding_blocks_until_exact_queued_intent_is_reconciled() { assert!(repo.resolution_issue("run-unknown").unwrap().is_some()); // Reconciliation is allowed only for the exact still-queued intent. - repo.enqueue_with_scheduling( - &row, - &[], - &QueueResolutionUpdate::Resolved, - &[], - None, - None, - ) - .unwrap(); + repo.enqueue_with_scheduling(&row, &[], &QueueResolutionUpdate::Resolved, &[], None, None) + .unwrap(); assert!(repo.resolution_issue("run-unknown").unwrap().is_none()); - assert!(repo.try_claim("run-unknown", -90_000, 20).unwrap().is_some()); + assert!(repo + .try_claim("run-unknown", -90_000, 20) + .unwrap() + .is_some()); } #[test] @@ -809,11 +804,10 @@ fn cancellation_blocks_root_and_transitive_descendants_with_reason() { for id in &blocked { let row = repo.get_row(id).unwrap().unwrap(); assert_eq!(row.status, QueueStatus::Blocked); - assert!( - row.blocked_reason - .as_deref() - .is_some_and(|reason| reason.contains("A") && reason.contains("cancelled")) - ); + assert!(row + .blocked_reason + .as_deref() + .is_some_and(|reason| reason.contains("A") && reason.contains("cancelled"))); assert_eq!(repo.try_claim(id, -90_000, 20).unwrap(), None); } assert!(repo.try_claim("run-e", -90_000, 20).unwrap().is_some()); @@ -936,10 +930,16 @@ fn assignment_ids_advance_across_claim_rebind_and_release_paths() { assert!(repo.fail_claimed("run-a", epoch_2, 10).unwrap()); let failed = repo.get_row("run-a").unwrap().unwrap().assignment_id; - assert!(failed > claim_2, "terminal claim failure invalidates its identity"); + assert!( + failed > claim_2, + "terminal claim failure invalidates its identity" + ); assert!(repo.release_claim_manual("run-a", 10).unwrap()); let manual_release = repo.get_row("run-a").unwrap().unwrap().assignment_id; - assert!(manual_release > failed, "manual recovery mints a new identity"); + assert!( + manual_release > failed, + "manual recovery mints a new identity" + ); repo.try_claim_for_worker("run-a", Some("worker-1"), -90_000, 10) .unwrap() @@ -950,7 +950,10 @@ fn assignment_ids_advance_across_claim_rebind_and_release_paths() { vec!["run-a".to_string()] ); let reclaimed = repo.get_row("run-a").unwrap().unwrap().assignment_id; - assert!(reclaimed > before_reclaim, "stale reclaim invalidates the assignment"); + assert!( + reclaimed > before_reclaim, + "stale reclaim invalidates the assignment" + ); let epoch_4 = repo .try_claim_for_worker("run-a", Some("worker-1"), -90_000, 10) @@ -1052,7 +1055,11 @@ fn spawned_identity_survives_every_worker_sentinel_rewrite() { let released = repo.get_row(&id).unwrap().unwrap(); assert_eq!(released.status, QueueStatus::Queued, "{release_path}"); - assert_eq!(released.worker_id, format!("pending:{id}"), "{release_path}"); + assert_eq!( + released.worker_id, + format!("pending:{id}"), + "{release_path}" + ); assert!( repo.record_heartbeat(SESSION_ID, &worker_id, "completed", 30) .unwrap(), @@ -1168,10 +1175,7 @@ async fn first_heartbeat_grace_is_separate_from_steady_state_cutoff() { repo.enqueue(&awaiting_first).unwrap(); repo.enqueue(&steady).unwrap(); - let manager = QueueManager::new( - Arc::clone(&repo), - EventBus::new(temp.path().to_path_buf()), - ); + let manager = QueueManager::new(Arc::clone(&repo), EventBus::new(temp.path().to_path_buf())); assert_eq!( manager .reclaim_stuck_at(2, STUCK_CUTOFF_MS + 2) @@ -1233,12 +1237,7 @@ fn new_assignment_resets_first_heartbeat_grace_across_every_retry_route() { .unwrap() .expect("first worker claims the row"); assert!(repo - .record_heartbeat( - SESSION_ID, - &first_worker, - "working", - FIRST_HEARTBEAT_AT, - ) + .record_heartbeat(SESSION_ID, &first_worker, "working", FIRST_HEARTBEAT_AT,) .unwrap()); assert_eq!( repo.get_row(&id).unwrap().unwrap().last_status.as_deref(), @@ -1264,14 +1263,8 @@ fn new_assignment_resets_first_heartbeat_grace_across_every_retry_route() { assert!(repo.requeue_claimed(&id, first_epoch, RELEASE_AT).unwrap()) } "release_failed_spawn" => assert_eq!( - repo.release_failed_spawn( - &id, - first_epoch, - &first_worker, - 3, - RELEASE_AT, - ) - .unwrap(), + repo.release_failed_spawn(&id, first_epoch, &first_worker, 3, RELEASE_AT,) + .unwrap(), SpawnFailureRelease::Requeued { failures: 1 } ), "release_claim_manual" => { @@ -1303,8 +1296,8 @@ fn new_assignment_resets_first_heartbeat_grace_across_every_retry_route() { let just_past_steady_cutoff = RETRY_CLAIM_AT + STUCK_CUTOFF_MS + 1; let reclaimable_worker = vec![(id.clone(), retry.worker_id.clone())]; - assert!(repo - .reclaim_stuck_with_grace( + assert!( + repo.reclaim_stuck_with_grace( just_past_steady_cutoff - STUCK_CUTOFF_MS, just_past_steady_cutoff - FIRST_HEARTBEAT_GRACE_MS, just_past_steady_cutoff, @@ -1361,8 +1354,13 @@ fn heartbeat_updates_only_the_current_finalized_assignment_for_a_reused_slot() { repo.record_heartbeat(SESSION_ID, "worker-1", "completed", 20) .unwrap(); - repo.enqueue(&queued_row("aa-current", "pending:aa-current", Some("CURRENT"), 2)) - .unwrap(); + repo.enqueue(&queued_row( + "aa-current", + "pending:aa-current", + Some("CURRENT"), + 2, + )) + .unwrap(); repo.try_claim_for_worker("aa-current", Some("worker-1"), -90_000, 30) .unwrap() .unwrap(); @@ -1386,7 +1384,10 @@ fn heartbeat_updates_only_the_current_finalized_assignment_for_a_reused_slot() { (old_before.heartbeat_at, old_before.updated_at), "the sibling's historical liveness bytes must remain unchanged" ); - assert_eq!((current_after.heartbeat_at, current_after.updated_at), (Some(999), 999)); + assert_eq!( + (current_after.heartbeat_at, current_after.updated_at), + (Some(999), 999) + ); let impossible_assignment = current_after.assignment_id + 100; assert_eq!( @@ -1427,8 +1428,13 @@ fn heartbeat_body_keeps_assignment_identity_optional_for_legacy_prompts() { async fn worker_finalized_event_uses_the_current_assignment_row_under_slot_reuse() { let temp = tempfile::tempdir().unwrap(); let repo = Arc::new(queue_repo()); - repo.enqueue(&queued_row("old-run", "pending:old-run", Some("old-task"), 1)) - .unwrap(); + repo.enqueue(&queued_row( + "old-run", + "pending:old-run", + Some("old-task"), + 1, + )) + .unwrap(); repo.try_claim_for_worker("old-run", Some("worker-1"), -90_000, 10) .unwrap() .unwrap(); @@ -1436,16 +1442,17 @@ async fn worker_finalized_event_uses_the_current_assignment_row_under_slot_reuse .unwrap(); let old_assignment = repo.get_row("old-run").unwrap().unwrap().assignment_id; - repo.enqueue(&queued_row("current-run", "pending:current-run", Some("current-task"), 2)) - .unwrap(); + repo.enqueue(&queued_row( + "current-run", + "pending:current-run", + Some("current-task"), + 2, + )) + .unwrap(); repo.try_claim_for_worker("current-run", Some("worker-1"), -90_000, 30) .unwrap() .unwrap(); - let current_assignment = repo - .get_row("current-run") - .unwrap() - .unwrap() - .assignment_id; + let current_assignment = repo.get_row("current-run").unwrap().unwrap().assignment_id; assert!(current_assignment > old_assignment); let event_bus = EventBus::new(temp.path().to_path_buf()); @@ -1462,6 +1469,21 @@ async fn worker_finalized_event_uses_the_current_assignment_row_under_slot_reuse .unwrap(); assert_eq!(event.event_type, EventType::WorkerFinalized); assert_eq!(event.payload["task_id"], "current-task"); + let completion_event = tokio::time::timeout(std::time::Duration::from_secs(1), events.recv()) + .await + .expect("queue completion must emit WorkNodeCompleted promptly") + .unwrap(); + assert_eq!(completion_event.event_type, EventType::WorkNodeCompleted); + assert_eq!(completion_event.payload["task_id"], "current-task"); + let completion_facts = + read_node_completion_facts(&temp.path().join("sessions").join(SESSION_ID)).unwrap(); + assert_eq!(completion_facts.len(), 1); + assert_eq!(completion_facts[0].task_id, "current-task"); + assert_eq!(completion_facts[0].agent_id, "worker-1"); + assert_eq!( + completion_facts[0].provenance, + NodeCompletionProvenance::QueueFinalize + ); assert_eq!( repo.get_row("current-run").unwrap().unwrap().status, QueueStatus::Finalized @@ -1500,10 +1522,7 @@ async fn recovery_prioritizes_live_claim_over_older_terminal_slot_history() { 3, )) .unwrap(); - let manager = QueueManager::new( - Arc::clone(&repo), - EventBus::new(temp.path().to_path_buf()), - ); + let manager = QueueManager::new(Arc::clone(&repo), EventBus::new(temp.path().to_path_buf())); assert_eq!( manager.release_claim(SESSION_ID, "worker-1").await.unwrap(), @@ -1511,9 +1530,18 @@ async fn recovery_prioritizes_live_claim_over_older_terminal_slot_history() { previous: QueueStatus::Running } ); - assert_eq!(repo.get_row("live-running").unwrap().unwrap().status, QueueStatus::Queued); - assert_eq!(repo.get_row("old-failed").unwrap().unwrap().status, QueueStatus::Failed); - assert_eq!(repo.get_row("old-finalized").unwrap().unwrap().status, QueueStatus::Finalized); + assert_eq!( + repo.get_row("live-running").unwrap().unwrap().status, + QueueStatus::Queued + ); + assert_eq!( + repo.get_row("old-failed").unwrap().unwrap().status, + QueueStatus::Failed + ); + assert_eq!( + repo.get_row("old-finalized").unwrap().unwrap().status, + QueueStatus::Finalized + ); assert_eq!( manager.release_claim(SESSION_ID, "worker-1").await.unwrap(), @@ -1521,7 +1549,10 @@ async fn recovery_prioritizes_live_claim_over_older_terminal_slot_history() { previous: QueueStatus::Failed } ); - assert_eq!(repo.get_row("old-failed").unwrap().unwrap().status, QueueStatus::Queued); + assert_eq!( + repo.get_row("old-failed").unwrap().unwrap().status, + QueueStatus::Queued + ); assert_eq!( manager.release_claim(SESSION_ID, "worker-1").await.unwrap(), crate::coordination::ReleaseOutcome::Terminal { @@ -1592,7 +1623,10 @@ async fn typed_pending_outcome_retries_to_real_claim_and_events_keep_task_id() { .unwrap(), ClaimOutcome::Claimed { epoch: 1 } )); - assert_eq!(repo.get_row("run-b").unwrap().unwrap().task_id.as_deref(), Some("B")); + assert_eq!( + repo.get_row("run-b").unwrap().unwrap().task_id.as_deref(), + Some("B") + ); let mut observed = Vec::new(); for _ in 0..6 { @@ -1607,7 +1641,9 @@ async fn typed_pending_outcome_retries_to_real_claim_and_events_keep_task_id() { .expect("normal dependency wait event"); assert_eq!(pending.severity, Severity::Info); assert_eq!(pending.payload["task_id"], "B"); - assert!(observed.iter().all(|event| event.payload.get("task_id").is_some())); + assert!(observed + .iter() + .all(|event| event.payload.get("task_id").is_some())); } #[test] @@ -1640,10 +1676,7 @@ fn checkpoint_projection_and_queue_sql_agree_on_claimability() { ); let temp = tempfile::tempdir().unwrap(); let repo = Arc::new(queue_repo()); - let manager = QueueManager::new( - Arc::clone(&repo), - EventBus::new(temp.path().to_path_buf()), - ); + let manager = QueueManager::new(Arc::clone(&repo), EventBus::new(temp.path().to_path_buf())); repo.enqueue(&queue_row( "run-a", "worker-a", @@ -1674,6 +1707,53 @@ fn checkpoint_projection_and_queue_sql_agree_on_claimability() { assert!(repo.try_claim("run-b", -90_000, 30).unwrap().is_some()); } +#[test] +fn operational_queue_projection_preserves_worker_conflict_claimability() { + let node = |id: &str, status: NodeStatus| { + WorkNode::new( + id, + NodeKind::Task, + id, + NodeContract::default(), + BindingRef::Role("worker".to_string()), + status, + ) + }; + let graph = TaskGraph::new( + vec![ + node("persisted-completed", NodeStatus::Completed), + node("dependent", NodeStatus::Pending), + ], + vec![WorkEdge::new( + "persisted-completed", + "dependent", + EdgeKind::DependsOn, + EdgeProvenance::Planner, + )], + ); + let mut legacy_operational_projection = graph.clone(); + legacy_operational_projection.nodes[0].status = NodeStatus::Pending; + promote_initial_ready_nodes(&mut legacy_operational_projection); + let before = checkpoint_aware_claimable_nodes(&legacy_operational_projection); + + let temp = tempfile::tempdir().unwrap(); + let manager = QueueManager::new( + Arc::new(queue_repo()), + EventBus::new(temp.path().to_path_buf()), + ); + let after = checkpoint_aware_claimable_nodes( + &manager + .project_queue_statuses(SESSION_ID, &graph) + .expect("operational projection"), + ); + + assert_eq!(before, vec!["persisted-completed"]); + assert_eq!( + after, before, + "workers.rs conflict analysis must retain the pre-split claimable set" + ); +} + #[test] fn add_worker_request_task_id_is_explicit_or_null_and_backward_compatible() { let legacy: AddWorkerRequest = serde_json::from_value(json!({ @@ -1786,6 +1866,128 @@ async fn post_heartbeat_with_assignment( .unwrap() } +async fn post_heartbeat_with_completed_nodes( + app: &axum::Router, + session_id: &str, + agent_id: &str, + completed_nodes: &[&str], +) -> axum::response::Response { + app.clone() + .oneshot( + Request::builder() + .method("POST") + .uri(format!("/api/sessions/{session_id}/heartbeat")) + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&json!({ + "agent_id": agent_id, + "status": "completed", + "summary": "declared node completion", + "completed_nodes": completed_nodes, + })) + .unwrap(), + )) + .unwrap(), + ) + .await + .unwrap() +} + +#[tokio::test] +async fn completed_nodes_are_all_or_nothing_and_project_into_the_live_view() { + let temp = tempfile::tempdir().unwrap(); + let (app, state, controller) = dependency_http_fixture(&temp); + let session_id = "declared-heartbeat-session"; + let worker_id = format!("{session_id}-worker-1"); + let project = temp.path().join("declared-project"); + std::fs::create_dir_all(&project).unwrap(); + let session_dir = state.storage.create_session_dir(session_id).unwrap(); + StateManager::new(session_dir.clone()) + .write_work_graph(&TaskGraph::new( + vec![ + queue_test_node("T1", NodeStatus::Pending), + queue_test_node("T2", NodeStatus::Pending), + queue_test_node("T3", NodeStatus::Pending), + ], + Vec::new(), + )) + .unwrap(); + let mut session = quiet_hive_session(session_id, &project); + session.agents.push(AgentInfo { + id: worker_id.clone(), + role: AgentRole::Worker { + index: 1, + parent: Some(format!("{session_id}-queen")), + }, + status: AgentStatus::Running, + config: AgentConfig::default(), + parent_id: Some(format!("{session_id}-queen")), + commit_sha: None, + base_commit_sha: None, + role_definition_id: None, + role_definition_version: None, + }); + controller.read().insert_test_session(session); + + let rejected = + post_heartbeat_with_completed_nodes(&app, session_id, "worker-1", &["T2", "UNKNOWN"]).await; + assert_eq!(rejected.status(), StatusCode::BAD_REQUEST); + let rejected_body = String::from_utf8( + to_bytes(rejected.into_body(), usize::MAX) + .await + .unwrap() + .to_vec(), + ) + .unwrap(); + assert!(rejected_body.contains("UNKNOWN")); + assert!(StateManager::new(session_dir.clone()) + .read_node_completion_facts() + .unwrap() + .is_empty()); + assert!(!controller + .read() + .get_heartbeat_info(session_id) + .contains_key(&worker_id)); + + let accepted = + post_heartbeat_with_completed_nodes(&app, session_id, "worker-1", &["T2", "T3"]).await; + assert_eq!(accepted.status(), StatusCode::OK); + let facts = StateManager::new(session_dir) + .read_node_completion_facts() + .unwrap(); + assert_eq!(facts.len(), 2); + assert!(facts.iter().all(|fact| { + fact.agent_id == worker_id && fact.provenance == NodeCompletionProvenance::Heartbeat + })); + + let response = app + .clone() + .oneshot( + Request::builder() + .uri(format!( + "/api/sessions/{session_id}/work-graph?view=runtime&source=live" + )) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body: serde_json::Value = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()).unwrap(); + for task_id in ["T2", "T3"] { + let node = body["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["id"] == task_id) + .unwrap(); + assert_eq!(node["status"], "completed"); + assert_eq!(node["progress"]["agent_id"], worker_id); + assert_eq!(body["completion_provenance"][task_id], "declared"); + } +} + async fn post_queen_injection( app: &axum::Router, session_id: &str, @@ -2022,17 +2224,17 @@ async fn http_manual_release_fails_closed_before_touching_an_in_flight_spawn() { .oneshot( Request::builder() .method("POST") - .uri(format!("/api/sessions/{session_id}/workers/{worker_id}/release")) + .uri(format!( + "/api/sessions/{session_id}/workers/{worker_id}/release" + )) .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::CONFLICT); - let body: serde_json::Value = serde_json::from_slice( - &to_bytes(response.into_body(), usize::MAX).await.unwrap(), - ) - .unwrap(); + let body: serde_json::Value = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()).unwrap(); assert_eq!(body["reason"], "spawn_in_flight"); assert_eq!(body["epoch"], 1); let protected = state @@ -2045,7 +2247,12 @@ async fn http_manual_release_fails_closed_before_touching_an_in_flight_spawn() { .unwrap(); assert_eq!(protected.status, QueueStatus::Running); assert_eq!(protected.attempts, 1); - assert!(controller.read().get_session(session_id).unwrap().agents.is_empty()); + assert!(controller + .read() + .get_session(session_id) + .unwrap() + .agents + .is_empty()); } #[tokio::test] @@ -2079,23 +2286,38 @@ async fn http_retry_spawns_dependent_without_reusing_another_tasks_queue_identit let waiting_b = post_task_worker(&app, "B").await; let waiting_status = waiting_b.status(); - let waiting_body: serde_json::Value = serde_json::from_slice( - &to_bytes(waiting_b.into_body(), usize::MAX).await.unwrap(), - ) - .unwrap(); + let waiting_body: serde_json::Value = + serde_json::from_slice(&to_bytes(waiting_b.into_body(), usize::MAX).await.unwrap()) + .unwrap(); assert_eq!( waiting_status, StatusCode::CONFLICT, "unexpected dependency wait response: {waiting_body}" ); assert_eq!(waiting_body["reason"], "dependencies_pending"); - assert_eq!(controller.read().get_session(session_id).unwrap().agents.len(), 0); + assert_eq!( + controller + .read() + .get_session(session_id) + .unwrap() + .agents + .len(), + 0 + ); // C is ready and must not collide with B's dependency-pending queue intent even though // both requests initially reserve worker-1. let spawned_c = post_task_worker(&app, "C").await; assert_eq!(spawned_c.status(), StatusCode::CREATED); - assert_eq!(controller.read().get_session(session_id).unwrap().agents.len(), 1); + assert_eq!( + controller + .read() + .get_session(session_id) + .unwrap() + .agents + .len(), + 1 + ); let after_c = state.queue_manager.queue_snapshot(session_id).unwrap(); assert!(after_c .rows @@ -2143,10 +2365,9 @@ async fn http_retry_spawns_dependent_without_reusing_another_tasks_queue_identit // the roster's current worker-2 slot, and reaches the real controller/PTy spawn path. let spawned_b = post_task_worker(&app, "B").await; let spawned_b_status = spawned_b.status(); - let spawned_b_body: serde_json::Value = serde_json::from_slice( - &to_bytes(spawned_b.into_body(), usize::MAX).await.unwrap(), - ) - .unwrap(); + let spawned_b_body: serde_json::Value = + serde_json::from_slice(&to_bytes(spawned_b.into_body(), usize::MAX).await.unwrap()) + .unwrap(); assert_eq!(spawned_b_status, StatusCode::CREATED); let task_file = spawned_b_body["task_file"].as_str().unwrap(); let task_file_body = std::fs::read_to_string(task_file).unwrap(); @@ -2154,7 +2375,10 @@ async fn http_retry_spawns_dependent_without_reusing_another_tasks_queue_identit assert!(task_file_body.contains("\"B\"")); let session = controller.read().get_session(session_id).unwrap(); assert_eq!(session.agents.len(), 2); - assert!(session.agents.iter().any(|agent| agent.id.ends_with("worker-2"))); + assert!(session + .agents + .iter() + .any(|agent| agent.id.ends_with("worker-2"))); let snapshot = state.queue_manager.queue_snapshot(session_id).unwrap(); let b = snapshot @@ -2204,13 +2428,20 @@ async fn http_unknown_task_is_persisted_nonclaimable_then_reconciles_and_spawns( let unresolved = post_task_worker(&app, "UNKNOWN").await; assert_eq!(unresolved.status(), StatusCode::CONFLICT); - let body: serde_json::Value = serde_json::from_slice( - &to_bytes(unresolved.into_body(), usize::MAX).await.unwrap(), - ) - .unwrap(); + let body: serde_json::Value = + serde_json::from_slice(&to_bytes(unresolved.into_body(), usize::MAX).await.unwrap()) + .unwrap(); assert_eq!(body["reason"], "resolution_incomplete"); assert_eq!(body["task_id"], "UNKNOWN"); - assert_eq!(controller.read().get_session(session_id).unwrap().agents.len(), 0); + assert_eq!( + controller + .read() + .get_session(session_id) + .unwrap() + .agents + .len(), + 0 + ); let pending = state.queue_manager.queue_snapshot(session_id).unwrap(); assert_eq!(pending.queued, 1); assert_eq!(pending.resolution_incomplete.len(), 1); @@ -2224,7 +2455,15 @@ async fn http_unknown_task_is_persisted_nonclaimable_then_reconciles_and_spawns( let reconciled = state.queue_manager.queue_snapshot(session_id).unwrap(); assert!(reconciled.resolution_incomplete.is_empty()); assert_eq!(reconciled.running, 1); - assert_eq!(controller.read().get_session(session_id).unwrap().agents.len(), 1); + assert_eq!( + controller + .read() + .get_session(session_id) + .unwrap() + .agents + .len(), + 1 + ); } #[tokio::test] @@ -2253,8 +2492,14 @@ async fn http_materializes_known_code_conflict_then_spawns_after_peer_finalizes( available: true, artifact_languages: BTreeSet::from(["rust".to_string()]), touches: BTreeMap::from([ - ("T1".to_string(), BTreeSet::from(["src/shared.rs".to_string()])), - ("T2".to_string(), BTreeSet::from(["src/shared.rs".to_string()])), + ( + "T1".to_string(), + BTreeSet::from(["src/shared.rs".to_string()]), + ), + ( + "T2".to_string(), + BTreeSet::from(["src/shared.rs".to_string()]), + ), ]), unresolved_task_ids: Vec::new(), module_node_count: 1, @@ -2274,20 +2519,29 @@ async fn http_materializes_known_code_conflict_then_spawns_after_peer_finalizes( .write_graph_composition_state(&composition) .unwrap(); - assert_eq!(post_task_worker(&app, "T1").await.status(), StatusCode::CREATED); + assert_eq!( + post_task_worker(&app, "T1").await.status(), + StatusCode::CREATED + ); let waiting = post_task_worker(&app, "T2").await; assert_eq!(waiting.status(), StatusCode::CONFLICT); - let body: serde_json::Value = serde_json::from_slice( - &to_bytes(waiting.into_body(), usize::MAX).await.unwrap(), - ) - .unwrap(); + let body: serde_json::Value = + serde_json::from_slice(&to_bytes(waiting.into_body(), usize::MAX).await.unwrap()).unwrap(); assert_eq!(body["reason"], "conflicts_pending"); assert_eq!(body["blocking_task_ids"], json!(["T1"])); assert!(body["conflict_reasons"][0] .as_str() .unwrap() .contains("src/shared.rs")); - assert_eq!(controller.read().get_session(session_id).unwrap().agents.len(), 1); + assert_eq!( + controller + .read() + .get_session(session_id) + .unwrap() + .agents + .len(), + 1 + ); assert_eq!( state .queue_manager @@ -2304,8 +2558,19 @@ async fn http_materializes_known_code_conflict_then_spawns_after_peer_finalizes( .record_heartbeat(session_id, "http-dependency-session-worker-1", "completed") .await .unwrap(); - assert_eq!(post_task_worker(&app, "T2").await.status(), StatusCode::CREATED); - assert_eq!(controller.read().get_session(session_id).unwrap().agents.len(), 2); + assert_eq!( + post_task_worker(&app, "T2").await.status(), + StatusCode::CREATED + ); + assert_eq!( + controller + .read() + .get_session(session_id) + .unwrap() + .agents + .len(), + 2 + ); } #[tokio::test] @@ -2336,9 +2601,15 @@ async fn http_staggered_readiness_serializes_against_an_already_running_peer() { let composition = queue_test_composition( graph, BTreeMap::from([ - ("T1".to_string(), BTreeSet::from(["src/shared.rs".to_string()])), + ( + "T1".to_string(), + BTreeSet::from(["src/shared.rs".to_string()]), + ), ("DEP".to_string(), BTreeSet::new()), - ("T2".to_string(), BTreeSet::from(["src/shared.rs".to_string()])), + ( + "T2".to_string(), + BTreeSet::from(["src/shared.rs".to_string()]), + ), ]), Vec::new(), ); @@ -2346,11 +2617,16 @@ async fn http_staggered_readiness_serializes_against_an_already_running_peer() { .write_graph_composition_state(&composition) .unwrap(); - assert_eq!(post_task_worker(&app, "T1").await.status(), StatusCode::CREATED); + assert_eq!( + post_task_worker(&app, "T1").await.status(), + StatusCode::CREATED + ); let dependency_wait = post_task_worker(&app, "T2").await; assert_eq!(dependency_wait.status(), StatusCode::CONFLICT); let dependency_body: serde_json::Value = serde_json::from_slice( - &to_bytes(dependency_wait.into_body(), usize::MAX).await.unwrap(), + &to_bytes(dependency_wait.into_body(), usize::MAX) + .await + .unwrap(), ) .unwrap(); assert_eq!(dependency_body["reason"], "dependencies_pending"); @@ -2387,7 +2663,9 @@ async fn http_staggered_readiness_serializes_against_an_already_running_peer() { let conflict_wait = post_task_worker(&app, "T2").await; assert_eq!(conflict_wait.status(), StatusCode::CONFLICT); let conflict_body: serde_json::Value = serde_json::from_slice( - &to_bytes(conflict_wait.into_body(), usize::MAX).await.unwrap(), + &to_bytes(conflict_wait.into_body(), usize::MAX) + .await + .unwrap(), ) .unwrap(); assert_eq!(conflict_body["reason"], "conflicts_pending"); @@ -2398,7 +2676,10 @@ async fn http_staggered_readiness_serializes_against_an_already_running_peer() { .record_heartbeat(session_id, "http-dependency-session-worker-1", "completed") .await .unwrap(); - assert_eq!(post_task_worker(&app, "T2").await.status(), StatusCode::CREATED); + assert_eq!( + post_task_worker(&app, "T2").await.status(), + StatusCode::CREATED + ); } #[tokio::test] @@ -2469,13 +2750,19 @@ async fn complete_edgeless_graph_rejects_unknown_explicit_task_without_spawning( let response = post_task_worker(&app, "UNKNOWN").await; assert_eq!(response.status(), StatusCode::CONFLICT); - let body: serde_json::Value = serde_json::from_slice( - &to_bytes(response.into_body(), usize::MAX).await.unwrap(), - ) - .unwrap(); + let body: serde_json::Value = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()).unwrap(); assert_eq!(body["reason"], "resolution_incomplete"); assert_eq!(body["task_id"], "UNKNOWN"); - assert_eq!(controller.read().get_session(session_id).unwrap().agents.len(), 0); + assert_eq!( + controller + .read() + .get_session(session_id) + .unwrap() + .agents + .len(), + 0 + ); let snapshot = state.queue_manager.queue_snapshot(session_id).unwrap(); assert_eq!(snapshot.queued, 1); assert_eq!(snapshot.running, 0); @@ -2500,11 +2787,22 @@ async fn complete_edgeless_graph_admits_known_explicit_task() { )) .unwrap(); - assert_eq!(post_task_worker(&app, "KNOWN").await.status(), StatusCode::CREATED); + assert_eq!( + post_task_worker(&app, "KNOWN").await.status(), + StatusCode::CREATED + ); let snapshot = state.queue_manager.queue_snapshot(session_id).unwrap(); assert!(snapshot.resolution_incomplete.is_empty()); assert_eq!(snapshot.running, 1); - assert_eq!(controller.read().get_session(session_id).unwrap().agents.len(), 1); + assert_eq!( + controller + .read() + .get_session(session_id) + .unwrap() + .agents + .len(), + 1 + ); } #[tokio::test] @@ -2519,28 +2817,34 @@ async fn degraded_empty_graph_rejects_unknown_and_retains_its_omission() { .read() .insert_test_session(quiet_hive_session(session_id, &project)); let mut degraded = TaskGraph::default(); - degraded.omissions.push( - crate::orchestrator::work_graph::WorkGraphOmission::new( + degraded + .omissions + .push(crate::orchestrator::work_graph::WorkGraphOmission::new( crate::orchestrator::work_graph::WorkGraphOmissionReason::ResolutionIncomplete, 1, vec!["planner metadata malformed".to_string()], - ), - ); + )); let state_manager = StateManager::new(state.storage.session_dir(session_id)); state_manager.write_work_graph(°raded).unwrap(); let response = post_task_worker(&app, "UNKNOWN").await; assert_eq!(response.status(), StatusCode::CONFLICT); - let body: serde_json::Value = serde_json::from_slice( - &to_bytes(response.into_body(), usize::MAX).await.unwrap(), - ) - .unwrap(); + let body: serde_json::Value = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()).unwrap(); assert_eq!(body["reason"], "resolution_incomplete"); assert_eq!(body["task_id"], "UNKNOWN"); assert_eq!(state_manager.read_work_graph().unwrap(), Some(degraded)); let snapshot = state.queue_manager.queue_snapshot(session_id).unwrap(); assert_eq!(snapshot.resolution_incomplete.len(), 1); - assert_eq!(controller.read().get_session(session_id).unwrap().agents.len(), 0); + assert_eq!( + controller + .read() + .get_session(session_id) + .unwrap() + .agents + .len(), + 0 + ); } #[tokio::test] @@ -2554,7 +2858,11 @@ async fn unreadable_authoritative_graph_fails_closed_before_enqueue() { controller .read() .insert_test_session(quiet_hive_session(session_id, &project)); - std::fs::write(session_dir.join("state").join("work-graph.json"), "not-json").unwrap(); + std::fs::write( + session_dir.join("state").join("work-graph.json"), + "not-json", + ) + .unwrap(); let response = post_task_worker(&app, "B").await; assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); @@ -2666,9 +2974,7 @@ fn quiet_fusion_session(session_id: &str, project_path: &Path) -> Session { fn mark_session_completed_schedules_archive_after_persistence() { let temp = tempfile::tempdir().unwrap(); let session_id = "completion-archive"; - let storage = Arc::new( - SessionStorage::new_with_base(temp.path().join("storage")).unwrap(), - ); + let storage = Arc::new(SessionStorage::new_with_base(temp.path().join("storage")).unwrap()); let session_dir = storage.create_session_dir(session_id).unwrap(); let project = temp.path().join("project"); std::fs::create_dir_all(&project).unwrap(); diff --git a/src-tauri/src/http/tests_wg_retro.rs b/src-tauri/src/http/tests_wg_retro.rs index 78f9f46b..0744759c 100644 --- a/src-tauri/src/http/tests_wg_retro.rs +++ b/src-tauri/src/http/tests_wg_retro.rs @@ -96,6 +96,7 @@ fn outcome( subject_id: id.to_string(), task_id: Some(id.to_string()), agent_ids: vec![format!("agent-{id}")], + completion_evidence: None, status, started_at, finished_at, diff --git a/src-tauri/src/http/tests_wg_roles.rs b/src-tauri/src/http/tests_wg_roles.rs index 4d8e2d62..81bbf902 100644 --- a/src-tauri/src/http/tests_wg_roles.rs +++ b/src-tauri/src/http/tests_wg_roles.rs @@ -547,6 +547,7 @@ async fn a16_role_construction_paths_produce_explicit_template_keys() { id: "coordination-reviewer".to_string(), role: "Reviewer".to_string(), parent_id: Some("coordination-queen".to_string()), + principal: None, children: Vec::new(), }]) .expect("write hierarchy fixture"); diff --git a/src-tauri/src/http/tests_wg_runtime.rs b/src-tauri/src/http/tests_wg_runtime.rs index 3190848b..43aef13e 100644 --- a/src-tauri/src/http/tests_wg_runtime.rs +++ b/src-tauri/src/http/tests_wg_runtime.rs @@ -7,37 +7,32 @@ use chrono::Utc; use serde_json::json; use tempfile::TempDir; -use crate::coordination::StateManager; +use crate::coordination::{HierarchyNode, StateManager}; use crate::domain::event::{Event, EventType, Severity}; -use crate::domain::run_journal::{ - Confidence, LedgerEntry, RunJournalEntry, StepKind, StepStatus, -}; +use crate::domain::run_journal::{Confidence, LedgerEntry, RunJournalEntry, StepKind, StepStatus}; use crate::events::EventBus; use crate::orchestrator::work_graph::archive::{ - archive_completed_session, list_archives, read_archive, - schedule_completed_session_archive, ArchiveSourceKind, ArchiveSourceReport, - WorkGraphArchive, - WORK_GRAPH_ARCHIVE_SCHEMA_VERSION, -}; -use crate::orchestrator::work_graph::divergence::{ - compute_divergence, DivergenceKind, + archive_completed_session, list_archives, read_archive, schedule_completed_session_archive, + ArchiveSourceKind, ArchiveSourceReport, WorkGraphArchive, WORK_GRAPH_ARCHIVE_SCHEMA_VERSION, }; -use crate::orchestrator::work_graph::review::{ - instantiate_review_templates, ReviewTemplate, +use crate::orchestrator::work_graph::completion_ledger::{ + append_node_completion_facts, NodeCompletionFact, NodeCompletionProvenance, }; +use crate::orchestrator::work_graph::divergence::{compute_divergence, DivergenceKind}; use crate::orchestrator::work_graph::retro::{ evaluate_archives, IndependentEvaluator, RetroRunInput, }; +use crate::orchestrator::work_graph::review::{instantiate_review_templates, ReviewTemplate}; use crate::orchestrator::work_graph::runtime::{ - derive_runtime_graph, instantiate_review_templates_and_record, - mutate_and_record, mutation_log, reconstruct_structural_history, - record_graph_change, record_review_verdict_and_record, - route_failed_verdict_and_record, GraphMutationDelta, GraphMutationType, ReviewVerdict, - RuntimeOutcomeStatus, + derive_runtime_graph, derive_runtime_graph_with_completion_facts, + derive_runtime_graph_with_principals, instantiate_review_templates_and_record, + mutate_and_record, mutation_log, reconstruct_structural_history, record_graph_change, + record_review_verdict_and_record, route_failed_verdict_and_record, CompletionEvidenceClass, + GraphMutationDelta, GraphMutationType, ReviewVerdict, RuntimeOutcomeStatus, }; use crate::orchestrator::work_graph::{ - BindingRef, EdgeKind, EdgeProvenance, NodeContract, NodeKind, NodeStatus, - TaskGraph, WorkEdge, WorkGraphOmissionReason, WorkNode, + BindingRef, EdgeKind, EdgeProvenance, NodeContract, NodeKind, NodeStatus, TaskGraph, WorkEdge, + WorkGraphOmissionReason, WorkNode, }; use crate::storage::{ApplicationStateDb, RunJournalStore, SessionStorage}; @@ -152,7 +147,10 @@ fn retry_records_total_attempts_and_retro_reports_one_additional_attempt() { .iter() .find(|outcome| outcome.task_id.as_deref() == Some("task-a")) .unwrap(); - assert_eq!(outcome.attempt_count, 2, "attempt_count stores total attempts"); + assert_eq!( + outcome.attempt_count, 2, + "attempt_count stores total attempts" + ); let report = evaluate_runtime_retro(&plan, derived, events.len()); let node_metrics = report.runs[0].nodes.value().unwrap(); @@ -263,7 +261,10 @@ fn journal_observation_interrupted_status_remains_blocked() { #[test] fn lane_completion_records_event_backed_outcomes_and_reaches_retro() { let plan = TaskGraph::new( - vec![task("queue-backed", &["root"]), task("in-lane", &["follow-up"])], + vec![ + task("queue-backed", &["root"]), + task("in-lane", &["follow-up"]), + ], vec![WorkEdge::new( "queue-backed", "in-lane", @@ -302,7 +303,10 @@ fn lane_completion_records_event_backed_outcomes_and_reaches_retro() { ); } assert!( - outcome.source_refs.iter().any(|source| source == "event:lane-complete"), + outcome + .source_refs + .iter() + .any(|source| source == "event:lane-complete"), "{task_id} outcome is not event-backed by the real lane completion" ); } @@ -336,39 +340,248 @@ fn lane_completion_records_event_backed_outcomes_and_reaches_retro() { } #[test] -fn unresolved_completion_is_reported_as_typed_omission() { - let plan = TaskGraph::new(vec![task("unresolved-task", &["code"])], Vec::new()); - let events = vec![event( - "unresolved-complete", +fn anchored_lane_fanout_records_one_observed_and_eight_inferred_through_archive() { + const SESSION_ID: &str = "archive-lane-provenance"; + let plan = TaskGraph::new( + (1..=9) + .map(|index| { + WorkNode::new( + format!("T{index}"), + NodeKind::Task, + format!("Task T{index}"), + NodeContract::default(), + BindingRef::Role("P1".to_string()), + NodeStatus::Pending, + ) + }) + .collect(), + Vec::new(), + ); + let mut completion = event( + "single-lane-completion", EventType::AgentCompleted, - Some("unmapped-agent"), + Some("worker-1"), + json!({"task_id":"T1"}), + ); + completion.session_id = SESSION_ID.to_string(); + + let derived = derive_runtime_graph( + Some(&plan), + std::slice::from_ref(&completion), + &[], + &[], + &[], + ); + assert_eq!( + derived + .outcomes + .iter() + .filter(|outcome| { + outcome.completion_evidence == Some(CompletionEvidenceClass::Observed) + && outcome.task_id.is_some() + }) + .count(), + 1 + ); + assert_eq!( + derived + .outcomes + .iter() + .filter(|outcome| { + outcome.completion_evidence == Some(CompletionEvidenceClass::Inferred) + && outcome.task_id.is_some() + }) + .count(), + 8 + ); + + let temp = TempDir::new().unwrap(); + let storage = SessionStorage::new_with_base(temp.path().to_path_buf()).unwrap(); + let session_dir = storage.create_session_dir(SESSION_ID).unwrap(); + StateManager::new(session_dir) + .write_work_graph(&plan) + .unwrap(); + let event_bus = EventBus::new(temp.path().to_path_buf()); + tokio::runtime::Runtime::new() + .unwrap() + .block_on(event_bus.publish(completion)) + .unwrap(); + drop(event_bus); + + let completion = archive_completed_session(temp.path(), None, SESSION_ID).unwrap(); + let reread = read_archive(&completion.path).unwrap(); + assert_eq!( + reread + .outcomes + .iter() + .filter(|outcome| { + outcome.completion_evidence == Some(CompletionEvidenceClass::Observed) + && outcome.task_id.is_some() + }) + .count(), + 1 + ); + assert_eq!( + reread + .outcomes + .iter() + .filter(|outcome| { + outcome.completion_evidence == Some(CompletionEvidenceClass::Inferred) + && outcome.task_id.is_some() + }) + .count(), + 8 + ); +} + +#[test] +fn recorded_principal_resolves_completion_without_agent_id_suffix_guessing() { + const SESSION_ID: &str = "recorded-principal-runtime"; + const AGENT_ID: &str = "recorded-principal-runtime-worker-3"; + let plan = TaskGraph::new( + vec![WorkNode::new( + "T1", + NodeKind::Task, + "Principal task", + NodeContract::default(), + BindingRef::Role("P1".to_string()), + NodeStatus::Pending, + )], + Vec::new(), + ); + let mut completion = event( + "principal-completion", + EventType::AgentCompleted, + Some(AGENT_ID), json!({}), - )]; + ); + completion.session_id = SESSION_ID.to_string(); + let principals = std::collections::BTreeMap::from([(AGENT_ID.to_string(), "P1".to_string())]); - let derived = derive_runtime_graph(Some(&plan), &events, &[], &[], &[]); - let omission = derived - .runtime_graph - .omissions + let derived = derive_runtime_graph_with_principals( + Some(&plan), + std::slice::from_ref(&completion), + &[], + &[], + &[], + &principals, + ); + let task_outcome = derived + .outcomes .iter() - .find(|omission| { - omission.reason == WorkGraphOmissionReason::CompletionUnresolved - }) - .expect("unresolvable completion must not be reported as an ordinary pending task"); - assert_eq!(omission.count, 1); + .find(|outcome| outcome.task_id.as_deref() == Some("T1")) + .expect("recorded principal resolves the task"); + assert_eq!(task_outcome.status, RuntimeOutcomeStatus::Completed); assert_eq!( - omission.examples, - vec!["event:unresolved-complete:agent:unmapped-agent"] + task_outcome.completion_evidence, + Some(CompletionEvidenceClass::Inferred) ); - assert!(!derived.outcomes.iter().any(|outcome| { - outcome.task_id.as_deref() == Some("unresolved-task") + assert!(!derived.runtime_graph.omissions.iter().any(|omission| { + matches!( + omission.reason, + WorkGraphOmissionReason::CompletionUnresolved + | WorkGraphOmissionReason::ResolutionIncomplete + ) })); - let agent_outcome = derived - .outcomes - .iter() - .find(|outcome| outcome.subject_id == "agent:unmapped-agent") + + let temp = TempDir::new().unwrap(); + let storage = SessionStorage::new_with_base(temp.path().to_path_buf()).unwrap(); + let session_dir = storage.create_session_dir(SESSION_ID).unwrap(); + let state = StateManager::new(session_dir); + state.write_work_graph(&plan).unwrap(); + state + .update_hierarchy(&[HierarchyNode { + id: AGENT_ID.to_string(), + role: "Worker-3".to_string(), + principal: Some("P1".to_string()), + parent_id: Some(format!("{SESSION_ID}-queen")), + children: Vec::new(), + }]) .unwrap(); - assert_eq!(agent_outcome.started_at, None); - assert!(agent_outcome.finished_at.is_some()); + let event_bus = EventBus::new(temp.path().to_path_buf()); + tokio::runtime::Runtime::new() + .unwrap() + .block_on(event_bus.publish(completion)) + .unwrap(); + drop(event_bus); + + let archived = archive_completed_session(temp.path(), None, SESSION_ID).unwrap(); + let reread = read_archive(&archived.path).unwrap(); + assert_eq!( + reread + .outcomes + .iter() + .find(|outcome| outcome.task_id.as_deref() == Some("T1")) + .expect("hierarchy binding survives into archive attribution") + .status, + RuntimeOutcomeStatus::Completed + ); + + let legacy = TaskGraph::new( + vec![WorkNode::new( + "legacy", + NodeKind::Task, + "Legacy worker binding", + NodeContract::default(), + BindingRef::Role("worker-8".to_string()), + NodeStatus::Pending, + )], + Vec::new(), + ); + let legacy_agent = "legacy-session-worker-8"; + let legacy_derived = derive_runtime_graph_with_principals( + Some(&legacy), + &[event( + "legacy-completion", + EventType::AgentCompleted, + Some(legacy_agent), + json!({}), + )], + &[], + &[], + &[], + &std::collections::BTreeMap::from([(legacy_agent.to_string(), "worker-8".to_string())]), + ); + assert_eq!( + legacy_derived.runtime_graph.nodes[0].status, + NodeStatus::Completed + ); +} + +#[test] +fn missing_principal_binding_is_expected_typed_absence_without_guessing() { + let plan = TaskGraph::new(vec![task("unresolved-task", &["code"])], Vec::new()); + for agent_id in ["session-queen", "session-evaluator", "session-judge-1"] { + let derived = derive_runtime_graph( + Some(&plan), + &[event( + &format!("unresolved-{agent_id}"), + EventType::AgentCompleted, + Some(agent_id), + json!({}), + )], + &[], + &[], + &[], + ); + let omission = derived + .runtime_graph + .omissions + .iter() + .find(|omission| omission.reason == WorkGraphOmissionReason::ResolutionIncomplete) + .expect("an unbound supervisory agent is expected typed absence"); + assert_eq!(omission.count, 1, "{agent_id}"); + assert_eq!(omission.examples, vec![format!("binding:{agent_id}")]); + assert!(!derived + .runtime_graph + .omissions + .iter() + .any(|omission| { omission.reason == WorkGraphOmissionReason::CompletionUnresolved })); + assert!(!derived + .outcomes + .iter() + .any(|outcome| { outcome.task_id.as_deref() == Some("unresolved-task") })); + } } #[test] @@ -420,18 +633,13 @@ fn claims_resolve_by_task_id_and_null_task_ids_report_omissions() { && edge.kind == EdgeKind::Consumes }); assert!(resolved_edge.is_some()); - assert_eq!( - resolved_edge.unwrap().provenance, - EdgeProvenance::Runtime - ); + assert_eq!(resolved_edge.unwrap().provenance, EdgeProvenance::Runtime); let omission = derived .runtime_graph .omissions .iter() - .find(|omission| { - omission.reason == WorkGraphOmissionReason::ResolutionIncomplete - }) + .find(|omission| omission.reason == WorkGraphOmissionReason::ResolutionIncomplete) .unwrap(); assert_eq!(omission.count, 1); assert!(omission.examples[0].contains("null-task-id")); @@ -458,8 +666,7 @@ fn claims_resolve_by_task_id_and_null_task_ids_report_omissions() { .unwrap(); assert_eq!(task_outcome.status, RuntimeOutcomeStatus::Completed); assert!(task_outcome.effects.iter().any(|effect| { - effect.kind == "artifact" - && effect.reference.as_deref() == Some("artifacts/task-a.json") + effect.kind == "artifact" && effect.reference.as_deref() == Some("artifacts/task-a.json") })); } @@ -489,24 +696,25 @@ fn review_and_remediation_mutations_are_append_only_and_reconstructable() { .expect("failed review verdict changes the graph"); let expansion = expansions.first_mut().unwrap(); - let (_, second_delta) = route_failed_verdict_and_record( - session_id, - &mut graph, - &template, - expansion, - ) - .unwrap(); + let (_, second_delta) = + route_failed_verdict_and_record(session_id, &mut graph, &template, expansion).unwrap(); let second_delta = second_delta.expect("failed verdict adds remediation"); assert_eq!(first_delta.sequence, 1); assert_eq!(verdict_delta.sequence, 2); assert_eq!(second_delta.sequence, 3); - assert_eq!(first_delta.mutation_type, GraphMutationType::ReviewRoundAdded); + assert_eq!( + first_delta.mutation_type, + GraphMutationType::ReviewRoundAdded + ); assert_eq!( verdict_delta.mutation_type, GraphMutationType::ReviewVerdictRecorded ); - assert_eq!(second_delta.mutation_type, GraphMutationType::RemediationDetour); + assert_eq!( + second_delta.mutation_type, + GraphMutationType::RemediationDetour + ); assert_eq!(first_delta.before, initial); assert_eq!(first_delta.after, verdict_delta.before); assert_eq!(verdict_delta.after, second_delta.before); @@ -536,11 +744,7 @@ fn review_and_remediation_mutations_are_append_only_and_reconstructable() { broken_sequence.sequence = 4; assert!(reconstruct_structural_history( &graph, - &[ - first_delta.clone(), - verdict_delta.clone(), - broken_sequence, - ], + &[first_delta.clone(), verdict_delta.clone(), broken_sequence,], ) .is_err()); assert_eq!( @@ -557,15 +761,10 @@ fn review_and_remediation_mutations_are_append_only_and_reconstructable() { &[], &[], &[], - &[ - first_delta, - verdict_delta, - second_delta.clone(), - ], + &[first_delta, verdict_delta, second_delta.clone()], ); assert!(derived.outcomes.iter().any(|outcome| { - outcome.subject_id == verdict_id - && outcome.status == RuntimeOutcomeStatus::Failed + outcome.subject_id == verdict_id && outcome.status == RuntimeOutcomeStatus::Failed })); let before_failed_mutation = history.last().unwrap().clone(); @@ -634,11 +833,8 @@ fn structural_history_rejects_shape_divergence_and_duplicates_but_accepts_status }; assert_eq!(duplicate_ids.nodes.len(), duplicate_delta.after.nodes.len()); assert_eq!( - reconstruct_structural_history( - &duplicate_ids, - std::slice::from_ref(&duplicate_delta), - ) - .unwrap_err(), + reconstruct_structural_history(&duplicate_ids, std::slice::from_ref(&duplicate_delta),) + .unwrap_err(), "final runtime graph does not match mutation delta 1" ); @@ -728,12 +924,7 @@ fn archive_round_trip_preserves_corpus_and_never_mutates_runtime_sources() { .record_step_finished(session_id, &spawn_step, StepStatus::Completed) .unwrap(); let commit_step = journal - .record_step_started( - session_id, - StepKind::GitCommit, - 1, - Some("worker-a"), - ) + .record_step_started(session_id, StepKind::GitCommit, 1, Some("worker-a")) .unwrap(); journal .record_ledger( @@ -745,12 +936,7 @@ fn archive_round_trip_preserves_corpus_and_never_mutates_runtime_sources() { ) .unwrap(); journal - .confirm_ledger( - session_id, - &commit_step, - Some("abc123"), - Confidence::High, - ) + .confirm_ledger(session_id, &commit_step, Some("abc123"), Confidence::High) .unwrap(); journal .record_step_finished(session_id, &commit_step, StepStatus::Completed) @@ -818,10 +1004,7 @@ fn archive_round_trip_preserves_corpus_and_never_mutates_runtime_sources() { .get(&GraphMutationType::CompositeExpanded), Some(&1) ); - assert!(archived - .divergence - .count(DivergenceKind::NodeAdded) - > 0); + assert!(archived.divergence.count(DivergenceKind::NodeAdded) > 0); assert!(archived.outcomes.iter().any(|outcome| { outcome.task_id.as_deref() == Some("task-a") && outcome.status == RuntimeOutcomeStatus::Completed @@ -840,12 +1023,7 @@ fn archive_round_trip_preserves_corpus_and_never_mutates_runtime_sources() { reread.reconstruct_structural_history().unwrap(), vec![plan, runtime_structure] ); - let repeated = archive_completed_session( - temp.path(), - Some(&journal), - session_id, - ) - .unwrap(); + let repeated = archive_completed_session(temp.path(), Some(&journal), session_id).unwrap(); assert!(!repeated.created); assert_eq!(repeated.path, paths[0]); assert_eq!(repeated.archive, archived); @@ -901,7 +1079,123 @@ fn legacy_session_without_graph_or_sources_archives_cleanly_with_omissions() { .find(|source| source.kind == ArchiveSourceKind::MutationLog) .unwrap(); assert!(!mutation_source.available); - assert_eq!(mutation_source.omissions[0].reason, WorkGraphOmissionReason::ResolutionIncomplete); + assert_eq!( + mutation_source.omissions[0].reason, + WorkGraphOmissionReason::ResolutionIncomplete + ); +} + +#[test] +fn declared_completion_facts_resolve_null_task_events_and_archive_task_ids() { + let temp = TempDir::new().unwrap(); + let session_id = "declared-completion-093d-shape"; + let storage = SessionStorage::new_with_base(temp.path().to_path_buf()).unwrap(); + let session_dir = storage.create_session_dir(session_id).unwrap(); + let plan = TaskGraph::new( + (1..=10) + .map(|index| task(&format!("T{index}"), &[])) + .collect(), + Vec::new(), + ); + StateManager::new(session_dir.clone()) + .write_work_graph(&plan) + .unwrap(); + + let declared = vec![ + NodeCompletionFact::new( + "T2", + format!("{session_id}-worker-2"), + NodeCompletionProvenance::Heartbeat, + ), + NodeCompletionFact::new( + "T3", + format!("{session_id}-worker-3"), + NodeCompletionProvenance::Heartbeat, + ), + ]; + append_node_completion_facts(&session_dir, &declared).unwrap(); + + let events = (1..=5) + .map(|index| Event { + id: format!("completed-{index}"), + session_id: session_id.to_string(), + cell_id: None, + agent_id: Some(format!("{session_id}-worker-{index}")), + event_type: EventType::AgentCompleted, + timestamp: Utc::now(), + payload: json!({}), + severity: Severity::Info, + }) + .collect::>(); + let event_dir = temp.path().join(session_id); + fs::create_dir_all(&event_dir).unwrap(); + fs::write( + event_dir.join("events.jsonl"), + events + .iter() + .map(|event| serde_json::to_string(event).unwrap()) + .collect::>() + .join("\n"), + ) + .unwrap(); + + let direct = derive_runtime_graph_with_completion_facts( + Some(&plan), + &events, + &[], + &[], + &[], + &declared, + &Default::default(), + ); + for task_id in ["T2", "T3"] { + assert_eq!( + direct + .runtime_graph + .nodes + .iter() + .find(|node| node.id == task_id) + .unwrap() + .status, + NodeStatus::Completed + ); + let outcome = direct + .outcomes + .iter() + .find(|outcome| outcome.subject_id == task_id) + .unwrap(); + assert_eq!(outcome.task_id.as_deref(), Some(task_id)); + assert_eq!( + outcome.completion_evidence, + Some(CompletionEvidenceClass::Observed) + ); + } + assert!(direct.runtime_graph.omissions.iter().all(|omission| { + omission.reason != WorkGraphOmissionReason::CompletionUnresolved + || omission.examples.iter().all(|example| { + !example.contains(&format!("{session_id}-worker-2")) + && !example.contains(&format!("{session_id}-worker-3")) + }) + })); + + let archived = archive_completed_session(temp.path(), None, session_id) + .unwrap() + .archive; + for task_id in ["T2", "T3"] { + assert_eq!( + archived + .runtime_graph + .nodes + .iter() + .find(|node| node.id == task_id) + .unwrap() + .status, + NodeStatus::Completed + ); + assert!(archived.outcomes.iter().any(|outcome| { + outcome.subject_id == task_id && outcome.task_id.as_deref() == Some(task_id) + })); + } } #[test] @@ -915,13 +1209,7 @@ fn orphan_ledger_effect_is_preserved_and_reported_incomplete() { confidence: Confidence::High, recorded_at: Utc::now(), }; - let derived = derive_runtime_graph( - Some(&TaskGraph::default()), - &[], - &[], - &[effect], - &[], - ); + let derived = derive_runtime_graph(Some(&TaskGraph::default()), &[], &[], &[effect], &[]); assert!(derived.outcomes.iter().any(|outcome| { outcome.effects.iter().any(|effect| { effect.kind == "git_commit" && effect.reference.as_deref() == Some("deadbeef") @@ -929,7 +1217,10 @@ fn orphan_ledger_effect_is_preserved_and_reported_incomplete() { })); assert!(derived.runtime_graph.omissions.iter().any(|omission| { omission.reason == WorkGraphOmissionReason::ResolutionIncomplete - && omission.examples.iter().any(|example| example == "orphan-ledger-step:missing-step") + && omission + .examples + .iter() + .any(|example| example == "orphan-ledger-step:missing-step") })); } @@ -952,7 +1243,10 @@ fn unreadable_and_cross_session_event_lines_are_reported() { ); fs::write( event_dir.join("events.jsonl"), - format!("{{not-json}}\n{}\n", serde_json::to_string(&foreign_event).unwrap()), + format!( + "{{not-json}}\n{}\n", + serde_json::to_string(&foreign_event).unwrap() + ), ) .unwrap(); @@ -965,12 +1259,14 @@ fn unreadable_and_cross_session_event_lines_are_reported() { .unwrap(); assert!(event_source.available); assert_eq!(event_source.record_count, 0); - assert!(event_source.omissions.iter().any(|omission| { - omission.reason == WorkGraphOmissionReason::SourceUnreadable - })); - assert!(event_source.omissions.iter().any(|omission| { - omission.reason == WorkGraphOmissionReason::ResolutionIncomplete - })); + assert!(event_source + .omissions + .iter() + .any(|omission| { omission.reason == WorkGraphOmissionReason::SourceUnreadable })); + assert!(event_source + .omissions + .iter() + .any(|omission| { omission.reason == WorkGraphOmissionReason::ResolutionIncomplete })); } #[test] diff --git a/src-tauri/src/http/tests_wg_verifier.rs b/src-tauri/src/http/tests_wg_verifier.rs index f17b17ce..d76808cb 100644 --- a/src-tauri/src/http/tests_wg_verifier.rs +++ b/src-tauri/src/http/tests_wg_verifier.rs @@ -337,6 +337,7 @@ fn role_archive( subject_id: "verification-task".to_string(), task_id: Some("verification-task".to_string()), agent_ids: vec![agent_id.to_string()], + completion_evidence: None, status: RuntimeOutcomeStatus::Completed, started_at: Some(timestamp), finished_at: Some(timestamp), diff --git a/src-tauri/src/orchestrator/work_graph/archive.rs b/src-tauri/src/orchestrator/work_graph/archive.rs index c3d99828..716e8840 100644 --- a/src-tauri/src/orchestrator/work_graph/archive.rs +++ b/src-tauri/src/orchestrator/work_graph/archive.rs @@ -1,5 +1,6 @@ //! Immutable work-graph instance archives for issue #214, owned by WS-5. +use std::collections::BTreeMap; use std::error::Error; use std::fmt; use std::fs; @@ -14,19 +15,17 @@ use uuid::Uuid; use crate::coordination::StateManager; use crate::domain::event::Event; use crate::domain::run_journal::{LedgerEntry, RunJournalEntry}; -use crate::storage::{ - LearningSubmission, RunJournalStore, SessionStorage, -}; +use crate::storage::{LearningSubmission, RunJournalStore, SessionStorage}; +use super::completion_ledger::read_node_completion_facts; use super::divergence::{compute_divergence, DivergenceSummary}; use super::retro::{ - evaluate_archive_paths, evaluate_completed_session, IndependentEvaluator, - RetroArchivePath, RetroOmission, RetroOmissionReason, RetroReport, - UNREVIEWED_OUTCOME, + evaluate_archive_paths, evaluate_completed_session, IndependentEvaluator, RetroArchivePath, + RetroOmission, RetroOmissionReason, RetroReport, UNREVIEWED_OUTCOME, }; use super::runtime::{ - derive_runtime_graph, mutation_log_snapshot, reconstruct_structural_history, - record_omission, GraphMutationDelta, RuntimeOutcome, + derive_runtime_graph_with_completion_facts, mutation_log_snapshot, + reconstruct_structural_history, record_omission, GraphMutationDelta, RuntimeOutcome, }; use super::schema::{TaskGraph, WorkGraph, WorkGraphOmission, WorkGraphOmissionReason}; @@ -353,6 +352,21 @@ pub fn archive_completed_session( let event_log_path = storage_base_dir.join(session_id).join("events.jsonl"); let (events, event_report) = read_event_log(&event_log_path, session_id); + let (completion_facts, completion_ledger_omission) = + match read_node_completion_facts(&session_dir) { + Ok(facts) => (facts, None), + Err(error) => { + let mut omission = WorkGraphOmission::new( + WorkGraphOmissionReason::SourceUnreadable, + 1, + vec!["state/work-graph-completions.jsonl".to_string()], + ); + omission.detail = format!( + "declared node completions could not be read and were omitted: {error}" + ); + (Vec::new(), Some(omission)) + } + }; let (journal, journal_report, ledger, ledger_report) = read_run_sources(run_journal, session_id); let mutation_snapshot = mutation_log_snapshot(session_id); @@ -391,12 +405,20 @@ pub fn archive_completed_session( // Source reporting owns the legacy-plan omission in an archive. Passing an // explicit empty derivation base avoids counting the same absent file twice. let derivation_base = plan_graph.clone().unwrap_or_default(); - let mut derivation = derive_runtime_graph( + let agent_principals: BTreeMap = state + .read_hierarchy() + .unwrap_or_default() + .into_iter() + .filter_map(|node| node.principal.map(|principal| (node.id, principal))) + .collect(); + let mut derivation = derive_runtime_graph_with_completion_facts( Some(&derivation_base), &events, &journal, &ledger, &deltas, + &completion_facts, + &agent_principals, ); let sources = vec![ plan_report.clone(), @@ -410,11 +432,10 @@ pub fn archive_completed_session( merge_omission(&mut derivation.runtime_graph, omission); } } - let divergence = compute_divergence( - plan_graph.as_ref(), - &derivation.runtime_graph, - &deltas, - ); + if let Some(omission) = completion_ledger_omission.as_ref() { + merge_omission(&mut derivation.runtime_graph, omission); + } + let divergence = compute_divergence(plan_graph.as_ref(), &derivation.runtime_graph, &deltas); let archive = WorkGraphArchive { schema_version: WORK_GRAPH_ARCHIVE_SCHEMA_VERSION, archive_id, @@ -469,11 +490,7 @@ pub fn complete_session_archive_and_retro( let storage = SessionStorage::new_with_base(storage_base_dir.to_path_buf()) .map_err(|error| ArchiveError::Storage(error.to_string()))?; let session_dir = storage.session_dir(session_id); - let archive_result = archive_completed_session( - storage_base_dir, - run_journal, - session_id, - ); + let archive_result = archive_completed_session(storage_base_dir, run_journal, session_id); let archive_id = archive_result .as_ref() .map(|completion| completion.archive.archive_id.clone()) @@ -482,7 +499,9 @@ pub fn complete_session_archive_and_retro( let provenance_path = retro_provenance_path(&session_dir); let provenance = fs::read_to_string(&provenance_path) .map_err(ArchiveError::Io) - .and_then(|json| serde_json::from_str::(&json).map_err(ArchiveError::Json)); + .and_then(|json| { + serde_json::from_str::(&json).map_err(ArchiveError::Json) + }); let mut report = match (&archive_result, provenance) { (Ok(_), Ok(provenance)) => match IndependentEvaluator::new( @@ -490,62 +509,71 @@ pub fn complete_session_archive_and_retro( provenance.planner_agent_ids.clone(), provenance.supervisor_agent_ids.clone(), ) { - Ok(evaluator) => evaluate_archive_corpus( - &evaluator, - &storage, - &session_dir, - &provenance, - ) - .unwrap_or_else(|error| { - tracing::warn!(session_id, "Post-run work-graph evaluation failed: {error}"); + Ok(evaluator) => { + evaluate_archive_corpus(&evaluator, &storage, &session_dir, &provenance) + .unwrap_or_else(|error| { + tracing::warn!( + session_id, + "Post-run work-graph evaluation failed: {error}" + ); + RetroReport::unavailable( + provenance.evaluator_id, + RetroOmission::new( + RetroOmissionReason::ArchiveUnreadable, + "retro_completion", + format!("post-run evaluation failed: {error}"), + vec![session_id.to_string()], + ) + .for_archive(&archive_id), + ) + }) + } + Err(error) => { + tracing::warn!( + session_id, + "Retro evaluator provenance is not independent: {error}" + ); RetroReport::unavailable( provenance.evaluator_id, RetroOmission::new( - RetroOmissionReason::ArchiveUnreadable, + RetroOmissionReason::EvaluatorProvenanceUnavailable, "retro_completion", - format!("post-run evaluation failed: {error}"), - vec![session_id.to_string()], + error.to_string(), + vec![provenance_path.display().to_string()], ) .for_archive(&archive_id), ) - }), - Err(error) => { - tracing::warn!(session_id, "Retro evaluator provenance is not independent: {error}"); - RetroReport::unavailable( - provenance.evaluator_id, - RetroOmission::new( - RetroOmissionReason::EvaluatorProvenanceUnavailable, - "retro_completion", - error.to_string(), - vec![provenance_path.display().to_string()], - ) - .for_archive(&archive_id), - ) } }, (Ok(_), Err(error)) => { - tracing::warn!(session_id, "Retro evaluator provenance could not be read: {error}"); + tracing::warn!( + session_id, + "Retro evaluator provenance could not be read: {error}" + ); RetroReport::unavailable( "unavailable", RetroOmission::new( - RetroOmissionReason::EvaluatorProvenanceUnavailable, - "retro_completion", - format!("retro evaluator provenance could not be read: {error}"), - vec![provenance_path.display().to_string()], - ) + RetroOmissionReason::EvaluatorProvenanceUnavailable, + "retro_completion", + format!("retro evaluator provenance could not be read: {error}"), + vec![provenance_path.display().to_string()], + ) .for_archive(&archive_id), ) } (Err(error), _) => { - tracing::warn!(session_id, "Work-graph archival failed before retro evaluation: {error}"); + tracing::warn!( + session_id, + "Work-graph archival failed before retro evaluation: {error}" + ); RetroReport::unavailable( "unavailable", RetroOmission::new( - RetroOmissionReason::ArchiveUnreadable, - "retro_completion", - format!("work-graph archival failed before retro evaluation: {error}"), - vec![session_id.to_string()], - ) + RetroOmissionReason::ArchiveUnreadable, + "retro_completion", + format!("work-graph archival failed before retro evaluation: {error}"), + vec![session_id.to_string()], + ) .for_archive(&archive_id), ) } @@ -584,21 +612,19 @@ pub fn complete_session_archive_and_retro( insight: submission.insight, files_touched: submission.files_touched, }; - match storage.submit_learning_session_with_id( - &request.session, - &request, - &learning_id, - ) { + match storage.submit_learning_session_with_id(&request.session, &request, &learning_id) { Ok(result) => submitted_learning_ids.push(result.learning_id), Err(error) => { tracing::warn!(session_id, "Retro learning submission failed: {error}"); - report.omissions.push(RetroOmission::new( - RetroOmissionReason::LearningSubmissionFailed, - "learning_submission", - format!("unreviewed retro learning was not submitted: {error}"), - vec![learning_id], - ) - .for_archive(&archive_id)); + report.omissions.push( + RetroOmission::new( + RetroOmissionReason::LearningSubmissionFailed, + "learning_submission", + format!("unreviewed retro learning was not submitted: {error}"), + vec![learning_id], + ) + .for_archive(&archive_id), + ); } } } @@ -622,15 +648,20 @@ pub fn schedule_completed_session_archive_and_retro( session_id: String, ) { let thread_name = format!("work-graph-archive-retro-{session_id}"); - let spawn = std::thread::Builder::new().name(thread_name).spawn(move || { - if let Err(error) = complete_session_archive_and_retro( - &storage_base_dir, - run_journal.as_ref(), - &session_id, - ) { - tracing::warn!(session_id, "Failed to archive completed work graph: {error}"); - } - }); + let spawn = std::thread::Builder::new() + .name(thread_name) + .spawn(move || { + if let Err(error) = complete_session_archive_and_retro( + &storage_base_dir, + run_journal.as_ref(), + &session_id, + ) { + tracing::warn!( + session_id, + "Failed to archive completed work graph: {error}" + ); + } + }); if let Err(error) = spawn { tracing::warn!("Failed to schedule completed work-graph archive: {error}"); } @@ -643,11 +674,7 @@ pub fn schedule_completed_session_archive( run_journal: Option, session_id: String, ) { - schedule_completed_session_archive_and_retro( - storage_base_dir, - run_journal, - session_id, - ); + schedule_completed_session_archive_and_retro(storage_base_dir, run_journal, session_id); } pub fn read_archive(path: &Path) -> Result { @@ -679,12 +706,7 @@ pub fn list_archives(session_dir: &Path) -> Result, ArchiveError> { } fn read_event_log(path: &Path, session_id: &str) -> (Vec, ArchiveSourceReport) { - let mut report = source_report( - ArchiveSourceKind::EventLog, - "events.jsonl", - false, - 0, - ); + let mut report = source_report(ArchiveSourceKind::EventLog, "events.jsonl", false, 0); let file = match fs::File::open(path) { Ok(file) => file, Err(_) => { @@ -774,9 +796,8 @@ fn read_run_sources( WorkGraphOmissionReason::SourceUnreadable, "run_journal:read-failed", ); - journal_report.omissions[0].detail = format!( - "run journal could not be read; runtime evidence is incomplete: {error}" - ); + journal_report.omissions[0].detail = + format!("run journal could not be read; runtime evidence is incomplete: {error}"); Vec::new() } }; @@ -792,9 +813,8 @@ fn read_run_sources( WorkGraphOmissionReason::SourceUnreadable, "run_ledger:read-failed", ); - ledger_report.omissions[0].detail = format!( - "run ledger could not be read; runtime evidence is incomplete: {error}" - ); + ledger_report.omissions[0].detail = + format!("run ledger could not be read; runtime evidence is incomplete: {error}"); Vec::new() } }; @@ -827,17 +847,13 @@ fn source_omission( .find(|omission| omission.reason == reason) { omission.count = omission.count.saturating_add(1); - if omission.examples.len() < 5 - && !omission.examples.iter().any(|seen| seen == example) - { + if omission.examples.len() < 5 && !omission.examples.iter().any(|seen| seen == example) { omission.examples.push(example.to_string()); } } else { - report.omissions.push(WorkGraphOmission::new( - reason, - 1, - vec![example.to_string()], - )); + report + .omissions + .push(WorkGraphOmission::new(reason, 1, vec![example.to_string()])); } } diff --git a/src-tauri/src/orchestrator/work_graph/completion_ledger.rs b/src-tauri/src/orchestrator/work_graph/completion_ledger.rs new file mode 100644 index 00000000..49bef8b2 --- /dev/null +++ b/src-tauri/src/orchestrator/work_graph/completion_ledger.rs @@ -0,0 +1,175 @@ +//! Durable, node-addressed completion facts for work-graph projection. +//! +//! The effect ledger in the run journal records side effects of execution steps. This +//! separate JSONL ledger records the first-class fact that a specific work-graph node was +//! completed by a resolved agent identity. + +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use tempfile::NamedTempFile; + +use crate::domain::event::{Event, EventType, Severity}; + +use super::TaskId; + +pub const NODE_COMPLETION_LEDGER: &str = "state/work-graph-completions.jsonl"; + +static APPEND_LOCK: Mutex<()> = Mutex::new(()); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NodeCompletionProvenance { + QueueFinalize, + Heartbeat, + EvaluatorVerdict, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeCompletionFact { + pub id: String, + pub task_id: TaskId, + pub agent_id: String, + pub provenance: NodeCompletionProvenance, + pub completed_at: DateTime, +} + +impl NodeCompletionFact { + pub fn new( + task_id: impl Into, + agent_id: impl Into, + provenance: NodeCompletionProvenance, + ) -> Self { + Self { + id: uuid::Uuid::new_v4().to_string(), + task_id: task_id.into(), + agent_id: agent_id.into(), + provenance, + completed_at: Utc::now(), + } + } + + pub fn source_ref(&self) -> String { + format!("completion-fact:{}", self.id) + } + + pub fn event(&self, session_id: &str) -> Event { + Event { + id: uuid::Uuid::new_v4().to_string(), + session_id: session_id.to_string(), + cell_id: None, + agent_id: Some(self.agent_id.clone()), + event_type: EventType::WorkNodeCompleted, + timestamp: self.completed_at, + payload: serde_json::json!({ + "task_id": self.task_id, + "fact_id": self.id, + "provenance": self.provenance, + }), + severity: Severity::Info, + } + } +} + +pub fn ledger_path(session_dir: &Path) -> PathBuf { + session_dir.join(NODE_COMPLETION_LEDGER) +} + +/// Append facts without ever exposing a partially written JSONL file. The existing bytes and +/// new records are written to a sibling temporary file, then atomically persisted over the old +/// path. Existing records are preserved byte-for-byte. +pub fn append_node_completion_facts( + session_dir: &Path, + facts: &[NodeCompletionFact], +) -> Result<(), std::io::Error> { + if facts.is_empty() { + return Ok(()); + } + let _guard = APPEND_LOCK + .lock() + .map_err(|_| std::io::Error::other("completion-ledger append lock was poisoned"))?; + let path = ledger_path(session_dir); + let parent = path + .parent() + .ok_or_else(|| std::io::Error::other("completion ledger has no parent directory"))?; + fs::create_dir_all(parent)?; + + let existing = match fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(error) => return Err(error), + }; + let mut temp = NamedTempFile::new_in(parent)?; + temp.write_all(&existing)?; + if !existing.is_empty() && existing.last() != Some(&b'\n') { + temp.write_all(b"\n")?; + } + for fact in facts { + serde_json::to_writer(&mut temp, fact).map_err(std::io::Error::other)?; + temp.write_all(b"\n")?; + } + temp.as_file().sync_all()?; + temp.persist(&path).map_err(|error| error.error)?; + Ok(()) +} + +pub fn read_node_completion_facts( + session_dir: &Path, +) -> Result, std::io::Error> { + let path = ledger_path(session_dir); + let file = match fs::File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error), + }; + BufReader::new(file) + .lines() + .enumerate() + .filter_map(|(index, line)| match line { + Ok(line) if line.trim().is_empty() => None, + other => Some((index, other)), + }) + .map(|(index, line)| { + let line = line?; + serde_json::from_str(&line).map_err(|error| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("invalid completion fact at line {}: {error}", index + 1), + ) + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn atomic_append_preserves_prior_completion_facts() { + let temp = TempDir::new().unwrap(); + let first = NodeCompletionFact::new( + "T1", + "session-worker-1", + NodeCompletionProvenance::Heartbeat, + ); + let second = NodeCompletionFact::new( + "T2", + "session-worker-2", + NodeCompletionProvenance::QueueFinalize, + ); + + append_node_completion_facts(temp.path(), std::slice::from_ref(&first)).unwrap(); + append_node_completion_facts(temp.path(), std::slice::from_ref(&second)).unwrap(); + + assert_eq!( + read_node_completion_facts(temp.path()).unwrap(), + vec![first, second] + ); + } +} diff --git a/src-tauri/src/orchestrator/work_graph/mod.rs b/src-tauri/src/orchestrator/work_graph/mod.rs index 576331e7..51673f19 100644 --- a/src-tauri/src/orchestrator/work_graph/mod.rs +++ b/src-tauri/src/orchestrator/work_graph/mod.rs @@ -3,6 +3,7 @@ pub mod archetypes; pub mod archive; pub mod codegraph; +pub mod completion_ledger; pub mod context; pub mod divergence; pub mod plan_parse; @@ -14,8 +15,7 @@ pub mod toposort; pub mod validate; pub use schema::{ - BindingRef, CompositeExpansion, EdgeKind, EdgeProvenance, NodeContract, NodeKind, - NodeStatus, TaskGraph, TaskId, WorkEdge, WorkGraph, WorkGraphOmission, - WorkGraphOmissionReason, WorkNode, + BindingRef, CompositeExpansion, EdgeKind, EdgeProvenance, NodeContract, NodeKind, NodeStatus, + TaskGraph, TaskId, WorkEdge, WorkGraph, WorkGraphOmission, WorkGraphOmissionReason, WorkNode, }; pub use toposort::{topological_sort, CycleError}; diff --git a/src-tauri/src/orchestrator/work_graph/plan_parse.rs b/src-tauri/src/orchestrator/work_graph/plan_parse.rs index 9fa616e2..9907e9e5 100644 --- a/src-tauri/src/orchestrator/work_graph/plan_parse.rs +++ b/src-tauri/src/orchestrator/work_graph/plan_parse.rs @@ -2,11 +2,11 @@ use crate::actions::coordination::SessionPlan; +use super::review::JUDGE_PRINCE_REMEDIATION_TEMPLATE; use super::schema::{ BindingRef, EdgeKind, EdgeProvenance, NodeContract, NodeKind, NodeStatus, TaskGraph, TaskId, WorkEdge, WorkGraphOmission, WorkGraphOmissionReason, WorkNode, }; -use super::review::JUDGE_PRINCE_REMEDIATION_TEMPLATE; /// Convert parsed plan tasks into schedulable nodes and planner-provenance edges. /// @@ -89,6 +89,26 @@ pub fn task_graph_from_plan(plan: &SessionPlan) -> TaskGraph { examples, )); } + let unrecognized_bindings = graph_tasks + .iter() + .filter(|task| task.assignee.is_some() && !task.assignee_recognized) + .collect::>(); + if !unrecognized_bindings.is_empty() { + graph.omissions.push(WorkGraphOmission::new( + WorkGraphOmissionReason::ResolutionIncomplete, + unrecognized_bindings.len(), + unrecognized_bindings + .iter() + .map(|task| { + format!( + "task {} preserved unrecognized principal binding {}", + task.id, + task.assignee.as_deref().unwrap_or("unassigned") + ) + }) + .collect(), + )); + } graph } @@ -107,9 +127,10 @@ pub fn promote_initial_ready_nodes(graph: &mut TaskGraph) -> Vec { .iter() .filter(|node| node.status == NodeStatus::Pending) .filter(|node| { - !node.expansion.as_ref().is_some_and(|expansion| { - expansion.template == JUDGE_PRINCE_REMEDIATION_TEMPLATE - }) + !node + .expansion + .as_ref() + .is_some_and(|expansion| expansion.template == JUDGE_PRINCE_REMEDIATION_TEMPLATE) }) .filter(|node| { graph diff --git a/src-tauri/src/orchestrator/work_graph/runtime.rs b/src-tauri/src/orchestrator/work_graph/runtime.rs index 1711433e..404e32cc 100644 --- a/src-tauri/src/orchestrator/work_graph/runtime.rs +++ b/src-tauri/src/orchestrator/work_graph/runtime.rs @@ -14,32 +14,30 @@ use serde::{Deserialize, Serialize}; use std::path::Path; use crate::domain::event::{Event, EventType}; -use crate::domain::run_journal::{ - Confidence, LedgerEntry, RunJournalEntry, StepKind, StepStatus, -}; +use crate::domain::run_journal::{Confidence, LedgerEntry, RunJournalEntry, StepKind, StepStatus}; use crate::orchestrator::org_graph::adjudication::{ - adjudicate_contradiction, AdjudicationDeclaration, AdjudicationError, - AdjudicationRecord, AdjudicationResolution, SourceVerdict, SourceVerdictValue, + adjudicate_contradiction, AdjudicationDeclaration, AdjudicationError, AdjudicationRecord, + AdjudicationResolution, SourceVerdict, SourceVerdictValue, }; -use super::review::{ - instantiate_checkpoint_wave, instantiate_review_templates, route_failed_verdict, - CheckpointWave, ReviewExpansion, ReviewGraphError, ReviewRoundExpansion, - ReviewExpansionSidecar, ReviewTemplate, -}; use super::archetypes::{ instantiate_named_archetype_into, reconcile_planner_graph, stamp_checkpoint_waves, ArchetypeError, ArchetypeLineage, RepoShapeFactsProvider, }; use super::codegraph::{derive_codegraph_touches, CodegraphDerivationReport}; +use super::completion_ledger::NodeCompletionFact; use super::context::{ derive_project_context_from_coverage, ContextDerivationReport, TouchesResolver, }; use super::plan_parse::promote_initial_ready_nodes; +use super::review::{ + instantiate_checkpoint_wave, instantiate_review_templates, route_failed_verdict, + CheckpointWave, ReviewExpansion, ReviewExpansionSidecar, ReviewGraphError, + ReviewRoundExpansion, ReviewTemplate, +}; use super::schema::{ - BindingRef, EdgeKind, EdgeProvenance, NodeContract, NodeKind, NodeStatus, - TaskGraph, TaskId, WorkEdge, WorkGraph, WorkGraphOmission, - WorkGraphOmissionReason, WorkNode, + BindingRef, EdgeKind, EdgeProvenance, NodeContract, NodeKind, NodeStatus, TaskGraph, TaskId, + WorkEdge, WorkGraph, WorkGraphOmission, WorkGraphOmissionReason, WorkNode, }; pub const RUNTIME_OBSERVATION_PREFIX: &str = "runtime:"; @@ -65,7 +63,9 @@ pub enum GraphCompositionError { impl fmt::Display for GraphCompositionError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Archetype(error) => write!(formatter, "graph archetype composition failed: {error}"), + Self::Archetype(error) => { + write!(formatter, "graph archetype composition failed: {error}") + } Self::Review(error) => write!(formatter, "graph review composition failed: {error}"), } } @@ -98,30 +98,28 @@ pub fn compose_initial_work_graph( where R: TouchesResolver + RepoShapeFactsProvider, { - let (mut graph, lineage, review_templates, checkpoints) = if let Some(archetype_id) = archetype_id { - let stage = instantiate_named_archetype_into( - base_graph, - project_path, - institutional_wiki_root, - archetype_id, - session_parameters, - resolver, - )?; - ( - stage.instance.graph, - Some(stage.instance.lineage), - stage.review_templates, - stage.checkpoints, - ) - } else { - (base_graph, None, Vec::new(), Vec::new()) - }; + let (mut graph, lineage, review_templates, checkpoints) = + if let Some(archetype_id) = archetype_id { + let stage = instantiate_named_archetype_into( + base_graph, + project_path, + institutional_wiki_root, + archetype_id, + session_parameters, + resolver, + )?; + ( + stage.instance.graph, + Some(stage.instance.lineage), + stage.review_templates, + stage.checkpoints, + ) + } else { + (base_graph, None, Vec::new(), Vec::new()) + }; let codegraph = derive_codegraph_touches(&mut graph, resolver); - let context = derive_project_context_from_coverage( - &mut graph, - project_path, - &codegraph.coverage(), - ); + let context = + derive_project_context_from_coverage(&mut graph, project_path, &codegraph.coverage()); let expansions = instantiate_review_templates(&mut graph, &review_templates)?; let reviews = ReviewExpansionSidecar::from_expansions(&review_templates, expansions)?; stamp_checkpoint_waves(&mut graph, &checkpoints)?; @@ -149,7 +147,7 @@ pub fn reconcile_composed_work_graph( Ok(reconciled) } -fn dedupe_graph_omissions(graph: &mut TaskGraph) { +pub(crate) fn dedupe_graph_omissions(graph: &mut TaskGraph) { let mut unique = Vec::new(); for omission in graph.omissions.drain(..) { if !unique.contains(&omission) { @@ -188,8 +186,7 @@ pub struct GraphMutationDelta { pub source_refs: Vec, } -static MUTATION_LOGS: OnceLock>>> = - OnceLock::new(); +static MUTATION_LOGS: OnceLock>>> = OnceLock::new(); fn mutation_logs() -> &'static Mutex>> { MUTATION_LOGS.get_or_init(|| Mutex::new(HashMap::new())) @@ -308,13 +305,7 @@ where } }; mark_new_edges_runtime(&before, graph); - let delta = match record_graph_change( - session_id, - mutation_type, - &before, - graph, - source_refs, - ) { + let delta = match record_graph_change(session_id, mutation_type, &before, graph, source_refs) { Ok(delta) => delta, Err(error) => { *graph = before; @@ -327,12 +318,16 @@ where fn mark_new_edges_runtime(before: &WorkGraph, after: &mut WorkGraph) { let mut consumed = vec![false; before.edges.len()]; for edge in &mut after.edges { - let prior = before.edges.iter().enumerate().position(|(index, candidate)| { - !consumed[index] - && candidate.source == edge.source - && candidate.target == edge.target - && candidate.kind == edge.kind - }); + let prior = before + .edges + .iter() + .enumerate() + .position(|(index, candidate)| { + !consumed[index] + && candidate.source == edge.source + && candidate.target == edge.target + && candidate.kind == edge.kind + }); if let Some(index) = prior { consumed[index] = true; } else { @@ -348,10 +343,8 @@ pub fn instantiate_review_templates_and_record( session_id: &str, graph: &mut WorkGraph, templates: &[ReviewTemplate], -) -> Result< - (Vec, Option), - GraphMutationError, -> { +) -> Result<(Vec, Option), GraphMutationError> +{ let refs = templates .iter() .map(|template| format!("review-template:{}", template.id)) @@ -370,10 +363,8 @@ pub fn route_failed_verdict_and_record( graph: &mut WorkGraph, template: &ReviewTemplate, expansion: &mut ReviewExpansion, -) -> Result< - (ReviewRoundExpansion, Option), - GraphMutationError, -> { +) -> Result<(ReviewRoundExpansion, Option), GraphMutationError> +{ let before_expansion = expansion.clone(); let verdict = expansion .rounds @@ -487,9 +478,7 @@ pub fn route_contradictory_verdicts_and_record( .iter() .position(|node| node.id == review_verdict_id) .ok_or_else(|| { - AdjudicationGraphError::UnknownReviewVerdict( - review_verdict_id.to_string(), - ) + AdjudicationGraphError::UnknownReviewVerdict(review_verdict_id.to_string()) })?; let mut known_ids = graph .nodes @@ -502,10 +491,7 @@ pub fn route_contradictory_verdicts_and_record( .map(|verdict| { ( verdict.clone(), - format!( - "{review_verdict_id}::source-verdict::{}", - verdict.source_id - ), + format!("{review_verdict_id}::source-verdict::{}", verdict.source_id), ) }) .collect::>(); @@ -559,8 +545,7 @@ pub fn route_contradictory_verdicts_and_record( let mut parameters = BTreeMap::new(); parameters.insert( "policy".to_string(), - serde_json::to_string(&record.policy) - .expect("adjudication policy is serializable"), + serde_json::to_string(&record.policy).expect("adjudication policy is serializable"), ); parameters.insert( "adjudicator".to_string(), @@ -578,8 +563,9 @@ pub fn route_contradictory_verdicts_and_record( .expect("source verdict records are serializable"), ); let adjudication_status = match &record.resolution { - AdjudicationResolution::ConsensusPass - | AdjudicationResolution::Findings { .. } => NodeStatus::Completed, + AdjudicationResolution::ConsensusPass | AdjudicationResolution::Findings { .. } => { + NodeStatus::Completed + } AdjudicationResolution::ConsensusFail => NodeStatus::Failed, AdjudicationResolution::ConsensusUnresolved { .. } | AdjudicationResolution::Escalated { .. } @@ -596,7 +582,7 @@ pub fn route_contradictory_verdicts_and_record( .collect(), outputs: vec![format!("{adjudication_id}:resolution")], acceptance: vec![ - "apply the declared policy without arrival-order bias".to_string(), + "apply the declared policy without arrival-order bias".to_string() ], }, BindingRef::Role(record.adjudicator.role_id.clone()), @@ -629,10 +615,7 @@ pub fn instantiate_checkpoint_wave_and_record( session_id: &str, graph: &mut WorkGraph, wave: &CheckpointWave, -) -> Result< - (Option, Option), - GraphMutationError, -> { +) -> Result<(Option, Option), GraphMutationError> { mutate_and_record( session_id, graph, @@ -758,6 +741,13 @@ pub enum RuntimeOutcomeStatus { Finalized, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CompletionEvidenceClass { + Observed, + Inferred, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ReviewVerdict { @@ -789,6 +779,8 @@ pub struct RuntimeOutcome { pub effects: Vec, #[serde(default)] pub source_refs: Vec, + #[serde(default)] + pub completion_evidence: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -805,6 +797,44 @@ pub fn derive_runtime_graph( journal: &[RunJournalEntry], ledger: &[LedgerEntry], deltas: &[GraphMutationDelta], +) -> RuntimeDerivation { + derive_runtime_graph_with_principals( + plan_graph, + events, + journal, + ledger, + deltas, + &BTreeMap::new(), + ) +} + +pub fn derive_runtime_graph_with_principals( + plan_graph: Option<&TaskGraph>, + events: &[Event], + journal: &[RunJournalEntry], + ledger: &[LedgerEntry], + deltas: &[GraphMutationDelta], + agent_principals: &BTreeMap, +) -> RuntimeDerivation { + derive_runtime_graph_with_completion_facts( + plan_graph, + events, + journal, + ledger, + deltas, + &[], + agent_principals, + ) +} + +pub fn derive_runtime_graph_with_completion_facts( + plan_graph: Option<&TaskGraph>, + events: &[Event], + journal: &[RunJournalEntry], + ledger: &[LedgerEntry], + deltas: &[GraphMutationDelta], + completion_facts: &[NodeCompletionFact], + agent_principals: &BTreeMap, ) -> RuntimeDerivation { let mut graph = plan_graph.cloned().unwrap_or_default(); if plan_graph.is_none() { @@ -824,11 +854,14 @@ pub fn derive_runtime_graph( .iter() .map(|node| (node.id.clone(), node.clone())) .collect(); - let structural_ids: std::collections::BTreeSet<_> = - structural_nodes.keys().cloned().collect(); + let structural_ids: std::collections::BTreeSet<_> = structural_nodes.keys().cloned().collect(); let mut last_agent_observation = BTreeMap::::new(); let mut current_task = BTreeMap::::new(); let mut outcomes = BTreeMap::::new(); + let declared_agents = completion_facts + .iter() + .map(|fact| fact.agent_id.as_str()) + .collect::>(); for delta in deltas { update_mutation_outcomes(&mut outcomes, delta); } @@ -839,10 +872,13 @@ pub fn derive_runtime_graph( None => continue, }; let observation_id = format!("{RUNTIME_OBSERVATION_PREFIX}event:{}", event.id); - let agent_id = event - .agent_id - .clone() - .or_else(|| event.payload.get("worker_id").and_then(|v| v.as_str()).map(str::to_string)); + let agent_id = event.agent_id.clone().or_else(|| { + event + .payload + .get("worker_id") + .and_then(|v| v.as_str()) + .map(str::to_string) + }); let task_id = event .payload .get("task_id") @@ -867,9 +903,7 @@ pub fn derive_runtime_graph( EdgeKind::DependsOn, EdgeProvenance::Runtime, ) - .with_rationale(format!( - "ordered runtime observations for agent {agent_id}" - )), + .with_rationale(format!("ordered runtime observations for agent {agent_id}")), ); } last_agent_observation.insert(agent_id.to_string(), observation_id.clone()); @@ -974,29 +1008,49 @@ pub fn derive_runtime_graph( kind, event, ); - if kind == RuntimeObservationKind::Completion { + if kind == RuntimeObservationKind::Completion + && event.event_type != EventType::WorkNodeCompleted + && !declared_agents.contains(agent_id.as_str()) + { let resolved = update_lane_completion_outcomes( &mut outcomes, &structural_nodes, + agent_principals, &agent_id, resolved_event_task.as_deref(), event, ); - if !resolved { - record_omission( + match resolved { + LaneCompletionResolution::Resolved => {} + LaneCompletionResolution::BindingMissing => record_omission( + &mut graph, + WorkGraphOmissionReason::ResolutionIncomplete, + &format!("binding:{agent_id}"), + ), + LaneCompletionResolution::LaneMissing => record_omission( &mut graph, WorkGraphOmissionReason::CompletionUnresolved, &format!("event:{}:agent:{agent_id}", event.id), - ); + ), } } } } - let journal_step_ids: std::collections::BTreeSet<_> = journal - .iter() - .map(|entry| entry.step_id.as_str()) - .collect(); + for fact in completion_facts { + if structural_ids.contains(&fact.task_id) { + update_declared_completion_outcome(&mut outcomes, fact); + } else { + record_omission( + &mut graph, + WorkGraphOmissionReason::CompletionUnresolved, + &format!("{}:task:{}", fact.source_ref(), fact.task_id), + ); + } + } + + let journal_step_ids: std::collections::BTreeSet<_> = + journal.iter().map(|entry| entry.step_id.as_str()).collect(); for entry in journal { let node_id = format!("{RUNTIME_OBSERVATION_PREFIX}journal:{}", entry.step_id); graph.nodes.push(journal_node(&node_id, entry)); @@ -1037,6 +1091,7 @@ pub fn derive_runtime_graph( attempt_count: 1, effects: matched_effects, source_refs: vec![format!("journal:step:{}", entry.step_id)], + completion_evidence: None, }, ); } @@ -1070,6 +1125,7 @@ pub fn derive_runtime_graph( attempt_count: 1, effects: vec![runtime_effect(effect)], source_refs: vec![format!("ledger:step:{}", effect.step_id)], + completion_evidence: None, }, ); } @@ -1091,14 +1147,11 @@ pub fn structural_projection(graph: &WorkGraph) -> WorkGraph { .filter(|node| !is_runtime_observation_node(node)) .cloned() .collect(); - let ids: std::collections::BTreeSet<_> = - nodes.iter().map(|node| node.id.as_str()).collect(); + let ids: std::collections::BTreeSet<_> = nodes.iter().map(|node| node.id.as_str()).collect(); let edges = graph .edges .iter() - .filter(|edge| { - ids.contains(edge.source.as_str()) && ids.contains(edge.target.as_str()) - }) + .filter(|edge| ids.contains(edge.source.as_str()) && ids.contains(edge.target.as_str())) .cloned() .collect(); WorkGraph { @@ -1117,11 +1170,7 @@ fn is_runtime_observation_node(node: &WorkNode) -> bool { ) } -pub fn record_omission( - graph: &mut WorkGraph, - reason: WorkGraphOmissionReason, - example: &str, -) { +pub fn record_omission(graph: &mut WorkGraph, reason: WorkGraphOmissionReason, example: &str) { let omission = if let Some(existing) = graph .omissions .iter_mut() @@ -1154,6 +1203,7 @@ fn observation_kind(event_type: &EventType) -> Option { EventType::AgentCompleted => Some(RuntimeObservationKind::Completion), EventType::AgentFailed => Some(RuntimeObservationKind::Failure), EventType::WorkerFinalized => Some(RuntimeObservationKind::Finalization), + EventType::WorkNodeCompleted => Some(RuntimeObservationKind::Completion), EventType::ArtifactUpdated => Some(RuntimeObservationKind::Artifact), _ => None, } @@ -1164,9 +1214,7 @@ fn observation_status(kind: RuntimeObservationKind) -> NodeStatus { RuntimeObservationKind::Claim | RuntimeObservationKind::Spawn | RuntimeObservationKind::JournalStep => NodeStatus::Running, - RuntimeObservationKind::ClaimFailed | RuntimeObservationKind::Failure => { - NodeStatus::Failed - } + RuntimeObservationKind::ClaimFailed | RuntimeObservationKind::Failure => NodeStatus::Failed, RuntimeObservationKind::Retry => NodeStatus::Ready, RuntimeObservationKind::Completion | RuntimeObservationKind::Finalization @@ -1297,6 +1345,7 @@ fn update_mutation_outcomes( attempt_count: 1, effects: Vec::new(), source_refs: Vec::new(), + completion_evidence: None, }); outcome.status = status; if status == RuntimeOutcomeStatus::Running { @@ -1307,7 +1356,9 @@ fn update_mutation_outcomes( outcome .source_refs .push(format!("mutation:delta:{}", delta.sequence)); - outcome.source_refs.extend(delta.source_refs.iter().cloned()); + outcome + .source_refs + .extend(delta.source_refs.iter().cloned()); } } @@ -1340,6 +1391,7 @@ fn update_event_outcome( attempt_count: 0, effects: Vec::new(), source_refs: Vec::new(), + completion_evidence: None, }); if !outcome.agent_ids.iter().any(|known| known == agent_id) { outcome.agent_ids.push(agent_id.to_string()); @@ -1348,6 +1400,12 @@ fn update_event_outcome( outcome.task_id = task_id; } outcome.source_refs.push(format!("event:{}", event.id)); + if matches!( + kind, + RuntimeObservationKind::Completion | RuntimeObservationKind::Finalization + ) { + merge_completion_evidence(outcome, CompletionEvidenceClass::Observed); + } match kind { RuntimeObservationKind::Claim => { outcome.attempt_count = outcome.attempt_count.saturating_add(1); @@ -1390,24 +1448,27 @@ fn update_event_outcome( source_ref: format!("event:{}", event.id), }); } - RuntimeObservationKind::JournalStep - | RuntimeObservationKind::LedgerEffect => {} + RuntimeObservationKind::JournalStep | RuntimeObservationKind::LedgerEffect => {} } } fn update_lane_completion_outcomes( outcomes: &mut BTreeMap, structural_nodes: &BTreeMap, + agent_principals: &BTreeMap, agent_id: &str, anchor_task_id: Option<&str>, event: &Event, -) -> bool { - let binding = anchor_task_id +) -> LaneCompletionResolution { + let binding = if let Some(binding) = anchor_task_id .and_then(|task_id| structural_nodes.get(task_id)) .map(|node| node.binding.clone()) - .or_else(|| completion_role_binding(agent_id, structural_nodes)); - let Some(binding) = binding else { - return false; + { + binding + } else if let Some(binding) = completion_role_binding(agent_id, agent_principals) { + binding + } else { + return LaneCompletionResolution::BindingMissing; }; let source_ref = format!("event:{}", event.id); @@ -1428,7 +1489,16 @@ fn update_lane_completion_outcomes( attempt_count: 1, effects: Vec::new(), source_refs: Vec::new(), + completion_evidence: None, }); + merge_completion_evidence( + outcome, + if anchor_task_id == Some(node.id.as_str()) { + CompletionEvidenceClass::Observed + } else { + CompletionEvidenceClass::Inferred + }, + ); if !outcome.agent_ids.iter().any(|known| known == agent_id) { outcome.agent_ids.push(agent_id.to_string()); } @@ -1444,22 +1514,74 @@ fn update_lane_completion_outcomes( outcome.source_refs.push(source_ref.clone()); } } - matched + if matched { + LaneCompletionResolution::Resolved + } else { + LaneCompletionResolution::LaneMissing + } +} + +fn update_declared_completion_outcome( + outcomes: &mut BTreeMap, + fact: &NodeCompletionFact, +) { + let outcome = outcomes + .entry(fact.task_id.clone()) + .or_insert_with(|| RuntimeOutcome { + subject_id: fact.task_id.clone(), + task_id: Some(fact.task_id.clone()), + agent_ids: Vec::new(), + status: RuntimeOutcomeStatus::Completed, + started_at: None, + finished_at: Some(fact.completed_at), + attempt_count: 1, + effects: Vec::new(), + source_refs: Vec::new(), + completion_evidence: None, + }); + outcome.task_id = Some(fact.task_id.clone()); + if !outcome + .agent_ids + .iter() + .any(|agent| agent == &fact.agent_id) + { + outcome.agent_ids.push(fact.agent_id.clone()); + } + outcome.status = RuntimeOutcomeStatus::Completed; + outcome.finished_at = Some(fact.completed_at); + if outcome.attempt_count == 0 { + outcome.attempt_count = 1; + } + let source_ref = fact.source_ref(); + if !outcome.source_refs.iter().any(|known| known == &source_ref) { + outcome.source_refs.push(source_ref); + } + merge_completion_evidence(outcome, CompletionEvidenceClass::Observed); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LaneCompletionResolution { + Resolved, + BindingMissing, + LaneMissing, +} + +fn merge_completion_evidence(outcome: &mut RuntimeOutcome, candidate: CompletionEvidenceClass) { + let current = outcome.completion_evidence; + if current == Some(CompletionEvidenceClass::Observed) || current == Some(candidate) { + return; + } + outcome.completion_evidence = Some(candidate); } fn completion_role_binding( agent_id: &str, - structural_nodes: &BTreeMap, + agent_principals: &BTreeMap, ) -> Option { - let mut matches = structural_nodes.values().filter_map(|node| { - let BindingRef::Role(role) = &node.binding else { - return None; - }; - (agent_id == role || agent_id.ends_with(&format!("-{role}"))) - .then(|| node.binding.clone()) - }); - let binding = matches.next()?; - matches.all(|candidate| candidate == binding).then_some(binding) + agent_principals + .get(agent_id) + .cloned() + .map(BindingRef::Role) } fn project_outcome_statuses( diff --git a/src-tauri/src/session/controller.rs b/src-tauri/src/session/controller.rs index f95956f9..dc8a28e7 100644 --- a/src-tauri/src/session/controller.rs +++ b/src-tauri/src/session/controller.rs @@ -15,17 +15,15 @@ use crate::coordination::queue_manager::{heartbeat_cadence_label, STUCK_CUTOFF_S use crate::coordination::{HierarchyNode, StateManager, WorkerStateInfo}; use crate::domain::{ArtifactBundle, HiveExecutionPolicy, HiveLaunchKind, WorkspaceStrategy}; use crate::events::{EventBus, EventEmitter}; -use crate::orchestrator::org_graph::definitions::{ - resolve_role_definition, role_prompt_template, ResolvedRoleDefinition, -}; use crate::orchestrator::org_graph::composition::{ compose_context, render_composed_context, spawn_context_from_work_graph_task, SpawnContext, }; +use crate::orchestrator::org_graph::definitions::{ + resolve_role_definition, role_prompt_template, ResolvedRoleDefinition, +}; use crate::orchestrator::org_graph::RoleDefinition; use crate::orchestrator::session_orchestrator::SessionOrchestrator; -use crate::pty::{ - AgentConfig, AgentRole, AgentStatus, PtyManager, RoleDefinitionRef, WorkerRole, -}; +use crate::pty::{AgentConfig, AgentRole, AgentStatus, PtyManager, RoleDefinitionRef, WorkerRole}; use crate::session::cell_status::{ agent_in_cell, derive_cell_status_name, derive_cell_status_name_for_state, session_cell_ids, variant_to_cell_id, PRIMARY_CELL_ID, RESOLVER_CELL_ID, @@ -329,19 +327,13 @@ impl SessionState { SessionState::WaitingForWorker(_) => SessionStateKind::WaitingForWorker, SessionState::SpawningPlanner(_) => SessionStateKind::SpawningPlanner, SessionState::WaitingForPlanner(_) => SessionStateKind::WaitingForPlanner, - SessionState::SpawningFusionVariant(_) => { - SessionStateKind::SpawningFusionVariant - } - SessionState::WaitingForFusionVariants => { - SessionStateKind::WaitingForFusionVariants - } + SessionState::SpawningFusionVariant(_) => SessionStateKind::SpawningFusionVariant, + SessionState::WaitingForFusionVariants => SessionStateKind::WaitingForFusionVariants, SessionState::SpawningDebateRound(_) => SessionStateKind::SpawningDebateRound, SessionState::WaitingForDebateRound(_) => SessionStateKind::WaitingForDebateRound, SessionState::SpawningJudge => SessionStateKind::SpawningJudge, SessionState::Judging => SessionStateKind::Judging, - SessionState::AwaitingVerdictSelection => { - SessionStateKind::AwaitingVerdictSelection - } + SessionState::AwaitingVerdictSelection => SessionStateKind::AwaitingVerdictSelection, SessionState::MergingWinner => SessionStateKind::MergingWinner, SessionState::SpawningEvaluator => SessionStateKind::SpawningEvaluator, SessionState::QaInProgress { .. } => SessionStateKind::QaInProgress, @@ -373,6 +365,13 @@ impl SessionState { | SessionState::QaInconclusive ) } + + pub fn is_terminal(&self) -> bool { + matches!( + self, + SessionState::Completed | SessionState::Closed | SessionState::Failed(_) + ) + } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -737,6 +736,9 @@ pub struct SessionController { task_watchers: Mutex>, /// session_id -> agent_id -> heartbeat info agent_heartbeats: Arc>>>, + /// Process-local carrier for the plan principal known only at the + /// plan-task-aware spawn site. Both hierarchy persistence sinks consume it. + principal_bindings: Arc>>, /// Session-owned operator shells. These PTYs are deliberately separate from /// `Session::agents` so they never enter worker queues, artifacts, or agent trees. scratch_ptys: Arc>>>, @@ -889,8 +891,7 @@ fn cell_status_changes_for_transition( // Keep cell projection behavior identical even when a transition is not yet // represented. The live mutator logs absent rows, while this lookup makes // the transition table the shared query seam for the status-change path. - let _transition = - super::transitions::transition_for(session.state.kind(), new_state.kind()); + let _transition = super::transitions::transition_for(session.state.kind(), new_state.kind()); let cell_ids = session_cell_ids(session); let before = cell_ids .iter() @@ -985,6 +986,7 @@ impl SessionController { storage: None, task_watchers: Mutex::new(HashMap::new()), agent_heartbeats: Arc::new(RwLock::new(HashMap::new())), + principal_bindings: Arc::new(RwLock::new(HashMap::new())), scratch_ptys: Arc::new(RwLock::new(HashMap::new())), scratch_pty_cleanup_sessions: Arc::new(RwLock::new(HashSet::new())), session_lifecycle_locks: Mutex::new(HashMap::new()), @@ -1686,8 +1688,7 @@ impl SessionController { let session_snapshot = { let mut sessions = self.sessions.write(); sessions.get_mut(session_id).map(|session| { - if Self::status_counts_as_session_activity(status) - && now > session.last_activity_at + if Self::status_counts_as_session_activity(status) && now > session.last_activity_at { session.last_activity_at = now; } @@ -1696,7 +1697,7 @@ impl SessionController { }; if let (Some(storage), Some(session)) = (self.storage.as_ref(), session_snapshot.as_ref()) { - Self::persist_session_snapshot(storage, session, session_id)?; + Self::persist_session_snapshot(storage, session, session_id, &self.principal_bindings)?; } let status_changed = prev_status.as_ref().map(|s| s != status).unwrap_or(true); if status_changed { @@ -1815,11 +1816,9 @@ impl SessionController { let AgentRole::Worker { index, .. } = &agent.role else { return None; }; - let task_file = Self::task_file_path_for_session_worker( - session, - *index as usize, - ) - .ok()?; + let task_file = + Self::task_file_path_for_session_worker(session, *index as usize) + .ok()?; let modified = std::fs::metadata(task_file).ok()?.modified().ok()?; Some((agent.id.clone(), DateTime::::from(modified))) }) @@ -1895,11 +1894,7 @@ impl SessionController { } else { AgentStatus::WaitingForInput("awaiting_injected_turn".to_string()) }; - stalled.push(( - agent_id.clone(), - awaiting_since, - status, - )); + stalled.push((agent_id.clone(), awaiting_since, status)); } } } @@ -2360,11 +2355,7 @@ impl SessionController { session: &mut Session, new_state: SessionState, ) -> Vec<(String, String, String)> { - Self::set_session_state_with_snapshot( - self.storage.as_deref(), - session, - new_state, - ) + Self::set_session_state_with_snapshot(self.storage.as_deref(), session, new_state) } fn set_session_state_with_snapshot( @@ -2509,11 +2500,7 @@ impl SessionController { } #[cfg(test)] - pub(crate) fn insert_scratch_pty_ownership_for_test( - &self, - session_id: &str, - pty_id: &str, - ) { + pub(crate) fn insert_scratch_pty_ownership_for_test(&self, session_id: &str, pty_id: &str) { self.scratch_ptys .write() .entry(session_id.to_string()) @@ -2541,15 +2528,10 @@ impl SessionController { } fn finish_scratch_pty_cleanup(&self, session_id: &str) { - self.scratch_pty_cleanup_sessions - .write() - .remove(session_id); + self.scratch_pty_cleanup_sessions.write().remove(session_id); } - pub(crate) fn scratch_pty_lifecycle_lock( - &self, - pty_id: &str, - ) -> Option>> { + pub(crate) fn scratch_pty_lifecycle_lock(&self, pty_id: &str) -> Option>> { let remainder = pty_id.strip_prefix("scratch:")?; let (session_id, unique_id) = remainder.rsplit_once(':')?; if session_id.is_empty() || unique_id.is_empty() { @@ -2602,8 +2584,7 @@ impl SessionController { if let Err(err) = self.persist_then_emit_session_update(id, changes) { let mut sessions = self.sessions.write(); if let Some(session) = sessions.get_mut(id) { - let _ = - self.set_session_state_with_events(session, previous_session_state); + let _ = self.set_session_state_with_events(session, previous_session_state); session.auth_strategy = previous_auth_strategy; } self.finish_scratch_pty_cleanup(id); @@ -2635,8 +2616,7 @@ impl SessionController { if let Err(err) = self.update_session_storage_checked(session_id) { let mut sessions = self.sessions.write(); if let Some(session) = sessions.get_mut(session_id) { - let _ = self - .set_session_state_with_events(session, previous_session_state); + let _ = self.set_session_state_with_events(session, previous_session_state); session.auth_strategy = previous_auth_strategy; } return Err(CompletionError::storage(err)); @@ -3734,6 +3714,7 @@ Last updated: {timestamp} &agent_id, "working", "Starting fusion variant", + &[], ); let heartbeat_command = heartbeat_snippet( "http://localhost:18800", @@ -3741,6 +3722,7 @@ Last updated: {timestamp} &agent_id, "idle", "Waiting for task activation", + &[], ); let completed_heartbeat = heartbeat_snippet( "http://localhost:18800", @@ -3748,6 +3730,7 @@ Last updated: {timestamp} &agent_id, "completed", "Completed fusion variant", + &[], ); let polling_instructions = get_polling_instructions(cli, &task_file, None, Some(&heartbeat_command)); @@ -4711,10 +4694,7 @@ Hard rule: The Evaluator AND the Prince are created PROGRAMMATICALLY by the back let mut variables = HashMap::new(); variables.insert("qa_worker_index".to_string(), index.to_string()); let qa_worker_agent_id = format!("{}-qa-worker-{}", session_id, index); - variables.insert( - "qa_worker_agent_id".to_string(), - qa_worker_agent_id.clone(), - ); + variables.insert("qa_worker_agent_id".to_string(), qa_worker_agent_id.clone()); variables.insert( "qa_worker_completed_heartbeat".to_string(), heartbeat_snippet( @@ -4723,6 +4703,7 @@ Hard rule: The Evaluator AND the Prince are created PROGRAMMATICALLY by the back &qa_worker_agent_id, "completed", "Completed QA assignment", + &[], ), ); variables.insert( @@ -5695,13 +5676,7 @@ Leading bracket tokens are optional and restricted to `[CRITICAL]`, `[HIGH]`, `[ ), Self::build_swarm_master_planner_prompt("test-swarm", "test objective", 2, &[]), Self::build_smoke_test_prompt("test-hive-smoke", &smoke_workers, false, None), - Self::build_swarm_smoke_test_prompt( - "test-swarm-smoke", - 4, - &[], - false, - None, - ), + Self::build_swarm_smoke_test_prompt("test-swarm-smoke", 4, &[], false, None), ] } @@ -5953,6 +5928,7 @@ This tests that: &format!("{session_id}-worker-1"), "working", "Starting smoke test", + &[], ), smoke_worker_completed_heartbeat = heartbeat_snippet( "http://localhost:18800", @@ -5960,6 +5936,7 @@ This tests that: &format!("{session_id}-worker-1"), "completed", "Smoke test done", + &[], ), ) } @@ -6243,6 +6220,7 @@ This tests that: "queen", "working", "Coordinating researchers", + &[], ), ); variables.insert( @@ -6424,6 +6402,7 @@ This tests that: "queen", "working", "Coordinating managed principals", + &[], ); let queen_completed_heartbeat = heartbeat_snippet( "http://localhost:18800", @@ -6431,6 +6410,7 @@ This tests that: "queen", "completed", "Objective and every configured gate complete", + &[], ); format!( @@ -6714,6 +6694,7 @@ When the objective and every configured gate are complete, send this `completed` &agent_id, "idle", "Waiting for task activation", + &[], ); let polling_instructions = get_polling_instructions( &config.cli, @@ -6727,6 +6708,7 @@ When the objective and every configured gate are complete, send this `completed` &agent_id, "working", "Executing assigned workstream", + &[], ); let completed_heartbeat = heartbeat_snippet( "http://localhost:18800", @@ -6734,6 +6716,7 @@ When the objective and every configured gate are complete, send this `completed` &agent_id, "completed", "Completed assigned workstream", + &[""], ); let role_section = if is_research { @@ -6764,7 +6747,7 @@ When the objective and every configured gate are complete, send this `completed` 1. {validation_and_handoff_rule} 2. Update the authoritative task file at {task_file} to `Status: COMPLETED` and add the evidence summary. -3. Send this completed heartbeat exactly as shown: +3. In the completed heartbeat below, replace `` with every exact node ID completed by this assignment (never a title, label, or agent ID); if no work-graph node was assigned, remove the optional `completed_nodes` field. Then send it: ```bash {completed_heartbeat} ``` @@ -6781,7 +6764,7 @@ When the objective and every configured gate are complete, send this `completed` 1. {validation_and_handoff_rule} 2. Complete the Learnings Protocol below before changing the task status. 3. Update the authoritative task file at {task_file} to `Status: COMPLETED` and add the result summary. -4. Send this completed heartbeat exactly as shown: +4. In the completed heartbeat below, replace `` with every exact node ID completed by this assignment (never a title, label, or agent ID); if no work-graph node was assigned, remove the optional `completed_nodes` field. Then send it: ```bash {completed_heartbeat} ``` @@ -7352,8 +7335,7 @@ Log each iteration to `.hive-manager/{session_id}/coordination.log`: let worker_task_file_example = "".to_string(); let qa_task_file_example = format!(".hive-manager/{}/tasks/qa-worker-N-task.md", session_id); - let worker_one_task_file_example = - "".to_string(); + let worker_one_task_file_example = "".to_string(); // Spawn Worker tool let spawn_worker_tool = format!( @@ -7602,6 +7584,7 @@ curl "http://localhost:18800/api/sessions/{session_id}/workers" "", "completed", "Queen verified completion: replace with concise gate evidence", + &[], ); let mark_worker_status_tool = format!( r#"# Mark Worker Status Tool @@ -7624,6 +7607,7 @@ Content-Type: application/json | agent_id | string | Yes | Exact full agent ID from the roster or worker API, such as `{session_id}-worker-2` or `{session_id}-fusion-1` | | status | string | Yes | `working` = doing work or holding the session open; `idle` = alive and blocked on another actor; `completed` = this actor is finished | | summary | string | No | Concise evidence-backed status summary | +| completed_nodes | string[] | No | Exact work-graph node IDs completed by this agent; valid only with `status: completed`. Omit when no node binding is known. | ## Mark a Verified Completion @@ -8417,9 +8401,7 @@ Last updated: {timestamp} ) .map_err(|error| error.to_string())?; - if config.work_graph_archetype.is_some() - && topology.launch_kind == HiveLaunchKind::Solo - { + if config.work_graph_archetype.is_some() && topology.launch_kind == HiveLaunchKind::Solo { return Err( "A work-graph archetype requires a Hive planning topology and cannot be coerced to Solo" .to_string(), @@ -8842,27 +8824,27 @@ Last updated: {timestamp} config.smoke_test, ) }) - .map_err(|err| { - { - let mut watchers = self.task_watchers.lock(); - let _ = watchers.remove(&session.id); - } - { - let mut heartbeats = self.agent_heartbeats.write(); - heartbeats.remove(&session.id); - } - { - let mut sessions = self.sessions.write(); - sessions.remove(&session.id); - } - self.rollback_launch_allocations( - &project_path, - &session_id, - &created_cells, - &spawned_agent_ids, - ); - err - })?; + .map_err(|err| { + { + let mut watchers = self.task_watchers.lock(); + let _ = watchers.remove(&session.id); + } + { + let mut heartbeats = self.agent_heartbeats.write(); + heartbeats.remove(&session.id); + } + { + let mut sessions = self.sessions.write(); + sessions.remove(&session.id); + } + self.rollback_launch_allocations( + &project_path, + &session_id, + &created_cells, + &spawned_agent_ids, + ); + err + })?; Ok(session) } @@ -9709,11 +9691,8 @@ phases and do EXACTLY this, then stop: role_type: &str, ) -> ResolvedRoleDefinition { let institutional_root = self.configured_institutional_wiki_root(); - let resolved = resolve_role_definition( - project_path, - institutional_root.as_deref(), - role_type, - ); + let resolved = + resolve_role_definition(project_path, institutional_root.as_deref(), role_type); for issue in &resolved.issues { tracing::warn!( role_id = %resolved.requested_id, @@ -9726,10 +9705,7 @@ phases and do EXACTLY this, then stop: resolved } - fn attach_resolved_definition( - role: &mut WorkerRole, - resolved: &ResolvedRoleDefinition, - ) { + fn attach_resolved_definition(role: &mut WorkerRole, resolved: &ResolvedRoleDefinition) { role.prompt_template = Some( resolved .definition @@ -9737,12 +9713,14 @@ phases and do EXACTLY this, then stop: .and_then(|definition| definition.prompt_template.clone()) .unwrap_or_else(|| role_prompt_template(&role.role_type)), ); - role.resolved_definition = resolved.definition.as_ref().map(|definition| { - RoleDefinitionRef { - id: definition.id.clone(), - version: definition.version, - } - }); + role.resolved_definition = + resolved + .definition + .as_ref() + .map(|definition| RoleDefinitionRef { + id: definition.id.clone(), + version: definition.version, + }); } /// Load only the task-scoped knowledge edges for an explicitly identified @@ -9788,6 +9766,40 @@ phases and do EXACTLY this, then stop: spawn_context_from_work_graph_task(&composition.graph, plan_task_id) } + fn principal_binding_for_plan_task( + &self, + session_id: &str, + plan_task_id: Option<&str>, + ) -> Option { + let plan_task_id = plan_task_id.map(str::trim).filter(|id| !id.is_empty())?; + let storage = self.storage.as_ref()?; + let state_manager = StateManager::new(storage.session_dir(session_id)); + let graph = match state_manager.read_work_graph() { + Ok(Some(graph)) => graph, + Ok(None) => { + state_manager + .read_graph_composition_state() + .ok() + .flatten()? + .graph + } + Err(error) => { + tracing::warn!( + session_id, + plan_task_id, + "Failed to read work graph for principal binding: {error}" + ); + return None; + } + }; + let node = graph.nodes.iter().find(|node| node.id == plan_task_id)?; + let principal = match &node.binding { + crate::orchestrator::work_graph::BindingRef::Role(principal) + | crate::orchestrator::work_graph::BindingRef::Zone(principal) => principal.trim(), + }; + (!principal.is_empty()).then(|| principal.to_string()) + } + fn planning_codegraph_artifact_path(project_path: &Path, session_id: &str) -> PathBuf { Self::session_root_path(project_path, session_id) .join("artifacts") @@ -9880,7 +9892,10 @@ phases and do EXACTLY this, then stop: .agents .iter() .filter(|agent| { - matches!(agent.role, AgentRole::MasterPlanner | AgentRole::Planner { .. }) + matches!( + agent.role, + AgentRole::MasterPlanner | AgentRole::Planner { .. } + ) }) .map(|agent| agent.id.clone()) .collect(); @@ -9892,11 +9907,9 @@ phases and do EXACTLY this, then stop: .collect(); if path.exists() { let existing: crate::orchestrator::work_graph::archive::RetroEvaluatorProvenance = - serde_json::from_str( - &std::fs::read_to_string(&path).map_err(|error| { - format!("Failed to read existing retro evaluator provenance: {error}") - })?, - ) + serde_json::from_str(&std::fs::read_to_string(&path).map_err(|error| { + format!("Failed to read existing retro evaluator provenance: {error}") + })?) .map_err(|error| { format!("Failed to parse existing retro evaluator provenance: {error}") })?; @@ -9930,7 +9943,8 @@ phases and do EXACTLY this, then stop: session_id: &str, archetype_id: Option<&str>, session_parameters: &BTreeMap, - ) -> Result, String> { + ) -> Result, String> + { let Some(archetype_id) = archetype_id else { return Ok(None); }; @@ -9947,16 +9961,15 @@ phases and do EXACTLY this, then stop: ) .map_err(|error| format!("Failed to load the planning codegraph artifact: {error}"))?; let institutional_wiki_root = self.configured_institutional_wiki_root(); - let composition = - crate::orchestrator::work_graph::runtime::compose_initial_work_graph( - crate::orchestrator::work_graph::TaskGraph::default(), - project_path, - institutional_wiki_root.as_deref(), - Some(archetype_id), - session_parameters, - &resolver, - ) - .map_err(|error| format!("Failed to compose initial work graph: {error}"))?; + let composition = crate::orchestrator::work_graph::runtime::compose_initial_work_graph( + crate::orchestrator::work_graph::TaskGraph::default(), + project_path, + institutional_wiki_root.as_deref(), + Some(archetype_id), + session_parameters, + &resolver, + ) + .map_err(|error| format!("Failed to compose initial work graph: {error}"))?; StateManager::new(storage.session_dir(session_id)) .write_graph_composition_state(&composition) .map_err(|error| format!("Failed to persist initial work-graph skeleton: {error}"))?; @@ -9999,7 +10012,8 @@ The backend composed and persisted the following authoritative skeleton before l pub(crate) fn reconcile_planner_graph_with_persisted_skeleton( state_manager: &StateManager, planner_graph: &crate::orchestrator::work_graph::TaskGraph, - ) -> Result, String> { + ) -> Result, String> + { let Some(persisted) = state_manager .read_graph_composition_state() .map_err(|error| format!("Failed to read initial work-graph skeleton: {error}"))? @@ -11643,8 +11657,12 @@ The backend composed and persisted the following authoritative skeleton before l }; if let Some(storage) = self.storage.as_ref() { - if let Err(err) = Self::persist_session_snapshot(storage, &updated_session, session_id) - { + if let Err(err) = Self::persist_session_snapshot( + storage, + &updated_session, + session_id, + &self.principal_bindings, + ) { let mut sessions = self.sessions.write(); if let Some(session) = sessions.get_mut(session_id) { *session = previous_session; @@ -11724,8 +11742,12 @@ The backend composed and persisted the following authoritative skeleton before l }; if let Some(storage) = self.storage.as_ref() { - if let Err(err) = Self::persist_session_snapshot(storage, &updated_session, session_id) - { + if let Err(err) = Self::persist_session_snapshot( + storage, + &updated_session, + session_id, + &self.principal_bindings, + ) { let mut sessions = self.sessions.write(); if let Some(session) = sessions.get_mut(session_id) { *session = previous_session; @@ -11819,8 +11841,12 @@ The backend composed and persisted the following authoritative skeleton before l }; if let Some(storage) = self.storage.as_ref() { - if let Err(err) = Self::persist_session_snapshot(storage, &updated_session, session_id) - { + if let Err(err) = Self::persist_session_snapshot( + storage, + &updated_session, + session_id, + &self.principal_bindings, + ) { let mut sessions = self.sessions.write(); if let Some(session) = sessions.get_mut(session_id) { *session = previous_session; @@ -12169,6 +12195,7 @@ The backend composed and persisted the following authoritative skeleton before l let app_handle = self.app_handle.clone(); let event_emitter = self.event_emitter.clone(); let storage = self.storage.clone(); + let principal_bindings = Arc::clone(&self.principal_bindings); let handle = tokio::spawn(async move { tokio::time::sleep(tokio::time::Duration::from_secs(timeout_secs)).await; @@ -12213,6 +12240,7 @@ The backend composed and persisted the following authoritative skeleton before l storage, &updated_session, &sid, + &principal_bindings, ) { tracing::warn!( "Failed to persist QA timeout state for {}: {}", @@ -13388,9 +13416,7 @@ The backend composed and persisted the following authoritative skeleton before l let examples = duplicate_nodes .iter() .map(|task_id| { - format!( - "duplicate task {task_id} was omitted after its first declaration" - ) + format!("duplicate task {task_id} was omitted after its first declaration") }) .collect::>(); tracing::warn!( @@ -13419,9 +13445,7 @@ The backend composed and persisted the following authoritative skeleton before l } let quarantined = - crate::orchestrator::work_graph::validate::quarantine_dangling_dependencies( - &mut graph, - ); + crate::orchestrator::work_graph::validate::quarantine_dangling_dependencies(&mut graph); if !quarantined.is_empty() { let examples = quarantined .iter() @@ -13451,8 +13475,10 @@ The backend composed and persisted the following authoritative skeleton before l tracing::warn!(session_id, warning = %warning, "PlanReady validation warning"); } } - Err(error @ (PlanReadyError::DuplicateTaskIds { .. } - | PlanReadyError::DanglingDependencies { .. })) => { + Err( + error @ (PlanReadyError::DuplicateTaskIds { .. } + | PlanReadyError::DanglingDependencies { .. }), + ) => { let message = error.to_string(); tracing::warn!( session_id, @@ -13465,13 +13491,15 @@ The backend composed and persisted the following authoritative skeleton before l vec![message], )); } - Err(error @ (PlanReadyError::Cycle { .. } - | PlanReadyError::MissingVerificationSignal { .. } - | PlanReadyError::MissingVerificationSignalClass { .. } - | PlanReadyError::InsufficientVerificationIsolation { .. } - | PlanReadyError::InvalidReviewAuthorityMetadata { .. } - | PlanReadyError::MissingAdjudicator { .. } - | PlanReadyError::AdjudicatorLacksAuthority { .. })) => { + Err( + error @ (PlanReadyError::Cycle { .. } + | PlanReadyError::MissingVerificationSignal { .. } + | PlanReadyError::MissingVerificationSignalClass { .. } + | PlanReadyError::InsufficientVerificationIsolation { .. } + | PlanReadyError::InvalidReviewAuthorityMetadata { .. } + | PlanReadyError::MissingAdjudicator { .. } + | PlanReadyError::AdjudicatorLacksAuthority { .. }), + ) => { return Err(error.to_string()); } } @@ -13487,6 +13515,8 @@ The backend composed and persisted the following authoritative skeleton before l tracing::warn!(session_id, omission = message); } + crate::orchestrator::work_graph::runtime::dedupe_graph_omissions(&mut graph); + if let Some(composition) = reconciled_composition.as_mut() { composition.graph = graph.clone(); } @@ -13534,7 +13564,12 @@ The backend composed and persisted the following authoritative skeleton before l let previous_session = session.clone(); let changes = self.set_session_state_with_events(session, SessionState::PlanReady); if let Some(storage) = self.storage.as_ref() { - if let Err(error) = Self::persist_session_snapshot(storage, session, session_id) { + if let Err(error) = Self::persist_session_snapshot( + storage, + session, + session_id, + &self.principal_bindings, + ) { *session = previous_session; Self::emit_work_graph_snapshot_for_session_state( Some(storage.as_ref()), @@ -13762,12 +13797,14 @@ The backend composed and persisted the following authoritative skeleton before l label: pa.config.label.clone().unwrap_or_default(), default_cli: pa.config.cli.clone(), prompt_template: Some(role_prompt_template(rt)), - resolved_definition: pa.role_definition_id.as_ref().zip( - pa.role_definition_version, - ).map(|(id, version)| RoleDefinitionRef { - id: id.clone(), - version, - }), + resolved_definition: pa + .role_definition_id + .as_ref() + .zip(pa.role_definition_version) + .map(|(id, version)| RoleDefinitionRef { + id: id.clone(), + version, + }), }), initial_prompt: pa.config.initial_prompt.clone(), }; @@ -14264,7 +14301,10 @@ The backend composed and persisted the following authoritative skeleton before l AgentRole::Queen | AgentRole::Planner { .. } | AgentRole::Prince ) { return Err(AddWorkerRejection { - error: format!("Agent {} cannot parent a managed principal", explicit_parent), + error: format!( + "Agent {} cannot parent a managed principal", + explicit_parent + ), reason: AddWorkerRejectionReason::ParentCannotParent, current_state: format!("{:?}", session.state), }); @@ -14350,9 +14390,7 @@ The backend composed and persisted the following authoritative skeleton before l let agent = session .agents .iter() - .find(|a| { - a.id == worker_id && matches!(a.role, AgentRole::Worker { .. }) - }) + .find(|a| a.id == worker_id && matches!(a.role, AgentRole::Worker { .. })) .ok_or_else(|| { format!( "Worker {} is not a rostered worker of session {}", @@ -14508,14 +14546,7 @@ The backend composed and persisted the following authoritative skeleton before l parent_id: Option, expected_index: Option, ) -> Result { - self.add_worker_for_plan_task( - session_id, - config, - role, - parent_id, - expected_index, - None, - ) + self.add_worker_for_plan_task(session_id, config, role, parent_id, expected_index, None) } /// Add a worker with an explicit work-graph binding. The ID is transported verbatim; @@ -14536,6 +14567,8 @@ The backend composed and persisted the following authoritative skeleton before l } .ok_or_else(|| format!("Session not found: {}", session_id))?; + let principal_binding = self.principal_binding_for_plan_task(session_id, plan_task_id); + Self::check_add_worker_preconditions(&session, &role, parent_id.as_deref()) .map_err(|rejection| rejection.error)?; @@ -14743,6 +14776,12 @@ The backend composed and persisted the following authoritative skeleton before l base_commit_sha: worker_base_commit_sha, }; + if let Some(principal) = principal_binding { + self.principal_bindings + .write() + .insert(worker_id.clone(), principal); + } + // Update session { let mut sessions = self.sessions.write(); @@ -14821,22 +14860,23 @@ The backend composed and persisted the following authoritative skeleton before l // written `SpawningEvaluator`, so a respawn from `QaFailed { iteration: 2 }` // fell to the catch-all arm and reset the counter to `None`, defeating the // max-retries ceiling. - let (previous_state, spawning_changes) = { - let mut sessions = self.sessions.write(); - if let Some(current) = sessions.get_mut(session_id) { - let previous_state = current.state.clone(); - current.agents.retain(|agent| agent.id != evaluator_id); - ( - Some(previous_state), - Some(self.set_session_state_with_events( - current, - SessionState::SpawningEvaluator, - )), - ) - } else { - (None, None) - } - }; + let (previous_state, spawning_changes) = + { + let mut sessions = self.sessions.write(); + if let Some(current) = sessions.get_mut(session_id) { + let previous_state = current.state.clone(); + current.agents.retain(|agent| agent.id != evaluator_id); + ( + Some(previous_state), + Some(self.set_session_state_with_events( + current, + SessionState::SpawningEvaluator, + )), + ) + } else { + (None, None) + } + }; self.emit_session_update(session_id); self.update_session_storage(session_id); if let Some(changes) = spawning_changes { @@ -15501,6 +15541,7 @@ The backend composed and persisted the following authoritative skeleton before l } // Build hierarchy nodes + let principal_bindings = self.principal_bindings.read(); let hierarchy: Vec = session .agents .iter() @@ -15517,6 +15558,7 @@ The backend composed and persisted the following authoritative skeleton before l HierarchyNode { id: agent.id.clone(), role: role_str, + principal: principal_bindings.get(&agent.id).cloned(), parent_id: agent.parent_id.clone(), children, } @@ -15614,7 +15656,12 @@ The backend composed and persisted the following authoritative skeleton before l session.clone() }; - Self::persist_session_snapshot(storage, &session, session_id)?; + Self::persist_session_snapshot( + storage, + &session, + session_id, + &self.principal_bindings, + )?; } Ok(()) @@ -15624,12 +15671,28 @@ The backend composed and persisted the following authoritative skeleton before l storage: &SessionStorage, session: &Session, session_id: &str, + principal_bindings: &RwLock>, ) -> Result<(), String> { let persisted = Self::session_to_persisted_snapshot(session); storage .save_session(&persisted) .map_err(|e| format!("Failed to update session metadata: {}", e))?; + let state_manager = StateManager::new(storage.session_dir(session_id)); + let persisted_principals: HashMap = match state_manager.read_hierarchy() { + Ok(hierarchy) => hierarchy + .into_iter() + .filter_map(|node| node.principal.map(|principal| (node.id, principal))) + .collect(), + Err(error) => { + tracing::warn!( + session_id, + "Failed to preserve existing hierarchy principals: {error}" + ); + HashMap::new() + } + }; + let principal_bindings = principal_bindings.read(); let hierarchy: Vec = session .agents .iter() @@ -15646,6 +15709,10 @@ The backend composed and persisted the following authoritative skeleton before l HierarchyNode { id: agent.id.clone(), role: role_str, + principal: principal_bindings + .get(&agent.id) + .cloned() + .or_else(|| persisted_principals.get(&agent.id).cloned()), parent_id: agent.parent_id.clone(), children, } @@ -15667,7 +15734,6 @@ The backend composed and persisted the following authoritative skeleton before l }) .collect(); - let state_manager = StateManager::new(storage.session_dir(session_id)); if let Err(e) = state_manager.update_hierarchy(&hierarchy) { tracing::warn!("Failed to update hierarchy: {}", e); } @@ -15897,12 +15963,19 @@ mod tests { SessionError, SessionState, SessionType, }; use super::{heartbeat_cadence_label, CliBehavior, CliRegistry}; + use crate::coordination::StateManager; use crate::domain::{ArtifactBundle, HiveExecutionPolicy, WorkspaceStrategy}; use crate::orchestrator::org_graph::composition::SpawnContext; + use crate::orchestrator::work_graph::{ + BindingRef, NodeContract, NodeKind, NodeStatus, TaskGraph, WorkGraphOmission, + WorkGraphOmissionReason, WorkNode, + }; use crate::pty::{AgentRole, AgentStatus, PtyManager, WorkerRole}; + use crate::storage::SessionStorage; use crate::workspace::git::current_head; use chrono::{Duration, Utc}; use parking_lot::RwLock; + use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use tempfile::TempDir; @@ -15969,6 +16042,198 @@ mod tests { let _failed = SessionState::Failed("error".to_string()); } + #[test] + fn terminal_session_state_excludes_the_closing_transition() { + assert!(SessionState::Completed.is_terminal()); + assert!(SessionState::Closed.is_terminal()); + assert!(SessionState::Failed("failure".to_string()).is_terminal()); + assert!(!SessionState::Closing.is_terminal()); + assert!(!SessionState::Running.is_terminal()); + } + + #[test] + fn plan_task_spawn_persists_principal_only_for_the_spawned_worker() { + const SESSION_ID: &str = "principal-spawn"; + let temp = tempfile::tempdir().expect("temporary principal fixture"); + let project_path = temp.path().join("project"); + std::fs::create_dir_all(&project_path).unwrap(); + let storage = Arc::new( + SessionStorage::new_with_base(temp.path().join("storage")) + .expect("isolated session storage"), + ); + let session_dir = storage + .create_session_dir(SESSION_ID) + .expect("session state directory"); + StateManager::new(session_dir.clone()) + .write_work_graph(&TaskGraph::new( + vec![WorkNode::new( + "T1", + NodeKind::Task, + "Principal-owned task", + NodeContract::default(), + BindingRef::Role("P1".to_string()), + NodeStatus::Ready, + )], + Vec::new(), + )) + .unwrap(); + + let mut controller = SessionController::new(Arc::new(RwLock::new(PtyManager::new()))); + controller.set_storage(Arc::clone(&storage)); + let mut session = + test_completion_session(SESSION_ID, SessionState::Running, Utc::now(), false); + session.session_type = SessionType::Hive { worker_count: 2 }; + session.project_path = project_path.clone(); + session.execution_policy.workspace_strategy = WorkspaceStrategy::SharedCell; + session.worktree_path = Some(project_path.to_string_lossy().to_string()); + session.agents = vec![ + AgentInfo { + id: format!("{SESSION_ID}-queen"), + role: AgentRole::Queen, + status: AgentStatus::Running, + config: AgentConfig::default(), + parent_id: None, + role_definition_id: None, + role_definition_version: None, + commit_sha: None, + base_commit_sha: None, + }, + AgentInfo { + id: format!("{SESSION_ID}-worker-1"), + role: AgentRole::Worker { + index: 1, + parent: Some(format!("{SESSION_ID}-queen")), + }, + status: AgentStatus::Running, + config: AgentConfig::default(), + parent_id: Some(format!("{SESSION_ID}-queen")), + role_definition_id: None, + role_definition_version: None, + commit_sha: None, + base_commit_sha: None, + }, + AgentInfo { + id: format!("{SESSION_ID}-worker-2"), + role: AgentRole::Worker { + index: 2, + parent: Some(format!("{SESSION_ID}-queen")), + }, + status: AgentStatus::Running, + config: AgentConfig::default(), + parent_id: Some(format!("{SESSION_ID}-queen")), + role_definition_id: None, + role_definition_version: None, + commit_sha: None, + base_commit_sha: None, + }, + ]; + controller.insert_test_session(session); + + let spawned = controller + .add_worker_for_plan_task( + SESSION_ID, + AgentConfig::default(), + WorkerRole::new("general", "General", "claude"), + None, + Some(3), + Some("T1"), + ) + .expect("plan-task-aware worker spawn"); + assert_eq!(spawned.id, format!("{SESSION_ID}-worker-3")); + + let hierarchy = StateManager::new(session_dir).read_hierarchy().unwrap(); + assert_eq!( + hierarchy + .iter() + .find(|node| node.id == spawned.id) + .expect("spawned worker hierarchy node") + .principal + .as_deref(), + Some("P1") + ); + assert!(hierarchy + .iter() + .filter(|node| node.id != spawned.id) + .all(|node| node.principal.is_none())); + } + + #[test] + fn mark_plan_ready_dedupes_omissions_appended_after_reconciliation() { + const SESSION_ID: &str = "phase-b-omission-retry"; + let temp = tempfile::tempdir().expect("temporary plan fixture"); + let project_path = temp.path().join("project"); + let project_session_dir = project_path.join(".hive-manager").join(SESSION_ID); + std::fs::create_dir_all(&project_session_dir).unwrap(); + std::fs::write( + project_session_dir.join("plan.md"), + "# Reconciled plan\n\n## Tasks\n- [ ] T1: Planner extension (deps: design, MISSING)\n", + ) + .unwrap(); + + let storage = Arc::new( + SessionStorage::new_with_base(temp.path().join("storage")) + .expect("isolated session storage"), + ); + storage + .create_session_dir(SESSION_ID) + .expect("session state directory"); + let mut controller = SessionController::new(Arc::new(RwLock::new(PtyManager::new()))); + controller.set_storage(Arc::clone(&storage)); + let mut session = + test_completion_session(SESSION_ID, SessionState::Planning, Utc::now(), false); + session.session_type = SessionType::Hive { worker_count: 0 }; + session.project_path = project_path.clone(); + session.no_git = true; + controller.insert_test_session(session); + controller + .prepare_initial_work_graph( + &project_path, + SESSION_ID, + Some("feature-build"), + &BTreeMap::from([("component".to_string(), "billing".to_string())]), + ) + .unwrap(); + + let state_manager = StateManager::new(storage.session_dir(SESSION_ID)); + let duplicate = WorkGraphOmission::new( + WorkGraphOmissionReason::ResolutionIncomplete, + 1, + vec!["T1 depends on unknown MISSING".to_string()], + ); + let mut persisted = state_manager + .read_graph_composition_state() + .unwrap() + .unwrap(); + persisted.graph.omissions.push(duplicate.clone()); + state_manager + .write_graph_composition_state(&persisted) + .unwrap(); + + controller + .mark_plan_ready(SESSION_ID) + .expect("a retry after a partial persist reaches PlanReady"); + + let authoritative = state_manager.read_work_graph().unwrap().unwrap(); + assert_eq!( + authoritative + .omissions + .iter() + .filter(|omission| **omission == duplicate) + .count(), + 1, + "the post-reconciliation quarantine must not duplicate a persisted omission" + ); + assert_eq!( + state_manager + .read_graph_composition_state() + .unwrap() + .unwrap() + .graph, + authoritative, + "both persisted graph representations must contain the same deduped omissions" + ); + } + #[test] fn heartbeat_activity_semantics_only_exempt_completed_status() { assert!(SessionController::status_counts_as_session_activity( @@ -16006,10 +16271,8 @@ mod tests { } drop(heartbeats); - let stalled = controller.get_stalled_agents( - "session-stall", - std::time::Duration::from_secs(30), - ); + let stalled = + controller.get_stalled_agents("session-stall", std::time::Duration::from_secs(30)); assert_eq!(stalled.len(), 1); assert_eq!(stalled[0].0, "session-stall-worker-1"); } @@ -16275,9 +16538,7 @@ mod tests { assert!(prompt.contains("/repo/execution")); assert!(!prompt.contains("UI Tester")); assert!(prompt.contains("## Completion Protocol (MANDATORY)")); - assert!( - prompt.contains(".hive-manager/session-123/tasks/qa-worker-1-task.md") - ); + assert!(prompt.contains(".hive-manager/session-123/tasks/qa-worker-1-task.md")); assert!(prompt.contains(r#""agent_id":"session-123-qa-worker-1""#)); assert!(prompt.contains(r#""status":"completed""#)); assert!(!prompt.contains("{{qa_worker_completed_heartbeat}}")); @@ -16295,10 +16556,8 @@ mod tests { "/repo/execution", ); - let completion = extract_markdown_section( - &prompt, - "## Completion Protocol (MANDATORY)", - ); + let completion = + extract_markdown_section(&prompt, "## Completion Protocol (MANDATORY)"); assert!( completion.contains(r#""agent_id":"session-qa-qa-worker-3""#), "missing exact agent ID for {specialization}" @@ -16781,6 +17040,8 @@ mod tests { assert!(status_content.contains("`working` = doing work or holding the session open")); assert!(status_content.contains("`idle` = alive and blocked on another actor")); assert!(status_content.contains("`completed` = this actor is finished")); + assert!(status_content.contains("| completed_nodes | string[] | No |")); + assert!(status_content.contains("Exact work-graph node IDs")); assert!(status_content.contains( "keeps agent liveness fresh for stall detection but does not extend the session's 10-minute quiescence window" )); @@ -16919,7 +17180,8 @@ mod tests { assert!(prompt.contains("may stay pending up to ~3000 ms across two confirmation windows")); assert!(prompt.contains("pty_activity_observed")); assert!(prompt.contains("do **not** prove that the agent took a turn")); - assert!(prompt.contains("`submit_confirmed: true` is not proof that a busy receiver accepted Enter")); + assert!(prompt + .contains("`submit_confirmed: true` is not proof that a busy receiver accepted Enter")); assert!(!prompt.contains("payload and its newline share one PTY write")); assert!(!prompt.contains("send a second, empty message")); assert!(!prompt.contains("Measured against a live codex agent")); @@ -16975,6 +17237,12 @@ mod tests { assert!(shared_prompt.contains("Completion Protocol (MANDATORY)")); assert!(shared_prompt.contains(r#""agent_id":"session-modern-worker-1""#)); assert!(shared_prompt.contains(r#""status":"completed""#)); + let completion_protocol = + extract_markdown_section(&shared_prompt, "## Completion Protocol (MANDATORY)"); + assert!(completion_protocol.contains(r#""completed_nodes":[""]"#)); + assert!(completion_protocol.contains( + "every exact node ID completed by this assignment (never a title, label, or agent ID)" + )); assert!(shared_prompt.contains("Begin only when Status is ACTIVE")); assert!(shared_prompt.contains("Queue Activation")); assert!(shared_prompt.contains("human-readable mirror")); @@ -16999,8 +17267,9 @@ mod tests { &isolated_policy, ); assert!(isolated_prompt.contains("Commit the completed assignment")); - assert!(isolated_prompt - .contains("commit SHA when applicable plus focused validation evidence")); + assert!( + isolated_prompt.contains("commit SHA when applicable plus focused validation evidence") + ); assert!(isolated_prompt.contains("Do not create or switch branches")); let no_workspace_policy = HiveExecutionPolicy { @@ -17490,12 +17759,8 @@ mod tests { for (session_id, close) in [("scratch-stop", false), ("scratch-close", true)] { let temp_dir = tempfile::tempdir().expect("temp project dir"); let controller = test_controller(); - let mut session = test_completion_session( - session_id, - SessionState::Running, - Utc::now(), - false, - ); + let mut session = + test_completion_session(session_id, SessionState::Running, Utc::now(), false); session.project_path = temp_dir.path().to_path_buf(); controller.insert_test_session(session); @@ -17503,13 +17768,11 @@ mod tests { controller .register_scratch_pty(session_id, pty_id.clone()) .expect("scratch PTY should be owned by its session"); - assert!( - controller - .scratch_ptys - .read() - .get(session_id) - .is_some_and(|ids| ids.contains(&pty_id)) - ); + assert!(controller + .scratch_ptys + .read() + .get(session_id) + .is_some_and(|ids| ids.contains(&pty_id))); if close { controller diff --git a/src-tauri/src/templates/mod.rs b/src-tauri/src/templates/mod.rs index c39da279..53b087e6 100644 --- a/src-tauri/src/templates/mod.rs +++ b/src-tauri/src/templates/mod.rs @@ -62,13 +62,17 @@ pub fn heartbeat_snippet( agent_id: &str, status: &str, summary: &str, + completed_nodes: &[&str], ) -> String { - let body = serde_json::json!({ + let mut body = serde_json::json!({ "agent_id": agent_id, "status": status, "summary": summary, - }) - .to_string(); + }); + if !completed_nodes.is_empty() { + body["completed_nodes"] = serde_json::json!(completed_nodes); + } + let body = body.to_string(); format!( r#"cat <<'JSON' | curl -fsS -X POST "{api_base_url}/api/sessions/{session_id}/heartbeat" \ @@ -2048,6 +2052,7 @@ You are a Planner agent managing the {{domain}} domain in a Swarm session. "queen", "working", "Monitoring workers", + &[], ), ); if rendered.contains("{{generic_heartbeat_snippet}}") { @@ -2062,6 +2067,7 @@ You are a Planner agent managing the {{domain}} domain in a Swarm session. agent_id, heartbeat_status, heartbeat_summary, + &[], ), ); } @@ -2073,6 +2079,7 @@ You are a Planner agent managing the {{domain}} domain in a Swarm session. &format!("{}-evaluator", context.session_id), "idle", "Waiting for milestone handoff", + &[], ), ); @@ -2367,15 +2374,12 @@ mod tests { .render_template(template_name, &context) .expect("render Queen prompt"); - assert!(prompt.contains( - "/api/sessions/session-245/conversations/session-245-queen?since=" - )); - assert!(prompt.contains( - "/api/sessions/session-245/conversations/session-245-worker-N/append" - )); - assert!(prompt.contains( - "/api/sessions/session-245/conversations/shared/append" - )); + assert!( + prompt.contains("/api/sessions/session-245/conversations/session-245-queen?since=") + ); + assert!(prompt + .contains("/api/sessions/session-245/conversations/session-245-worker-N/append")); + assert!(prompt.contains("/api/sessions/session-245/conversations/shared/append")); assert!(prompt.contains("reserved session-wide key `shared`")); assert!(!prompt.contains("/conversations/queen")); assert!(!prompt.contains("/conversations/worker-N")); @@ -2384,7 +2388,12 @@ mod tests { #[test] fn builtin_queen_prompts_require_marking_verified_completions() { - for template_name in ["queen-hive", "queen-research", "queen-fusion", "queen-swarm"] { + for template_name in [ + "queen-hive", + "queen-research", + "queen-fusion", + "queen-swarm", + ] { let prompt = TemplateEngine::default() .render_template( template_name, @@ -2398,9 +2407,7 @@ mod tests { .expect("render Queen prompt"); assert!(prompt.contains("Completion Status (MANDATORY)")); - assert!(prompt.contains( - ".hive-manager/session-123/tools/mark-worker-status.md" - )); + assert!(prompt.contains(".hive-manager/session-123/tools/mark-worker-status.md")); assert!(prompt.contains("UI completion checkoff and stall monitor depend on it")); } } @@ -2519,7 +2526,12 @@ mod tests { let Some(rest) = line.trim().strip_prefix("while [ \"$WAITED\" -lt ") else { continue; }; - let token = rest.split(']').next().unwrap_or("").trim().trim_matches('"'); + let token = rest + .split(']') + .next() + .unwrap_or("") + .trim() + .trim_matches('"'); if let Ok(secs) = token.parse::() { window = Some((token.to_string(), secs)); continue; @@ -2730,7 +2742,10 @@ mod tests { for owner in ["roles/evaluator", "roles/prince"] { let expected = format!("{profile}/{owner}"); // Two clamped loops apiece: evaluator idle + QA poll, prince verdict + fixers. - let found = clamped_sleeps.iter().filter(|got| **got == expected).count(); + let found = clamped_sleeps + .iter() + .filter(|got| **got == expected) + .count(); assert_eq!( found, 2, "expected 2 clamped heartbeat sleeps in {expected}, found {found} — the \ @@ -2797,13 +2812,37 @@ mod tests { "worker-1", "working", "Don't block", + &[], ); assert!(rendered.contains("cat <<'JSON' | curl")); assert!(rendered.contains("curl -fsS -X POST")); assert!(rendered.contains("--data-binary @-")); assert!(rendered.contains(r#""summary":"Don't block""#)); + assert!(!rendered.contains("completed_nodes")); assert!(!rendered.contains(" -d '")); + assert_eq!( + rendered, + "cat <<'JSON' | curl -fsS -X POST \"http://localhost:18800/api/sessions/session-123/heartbeat\" \\\n -H \"Content-Type: application/json\" \\\n --data-binary @-\n{\"agent_id\":\"worker-1\",\"status\":\"working\",\"summary\":\"Don't block\"}\nJSON" + ); + } + + #[test] + fn heartbeat_snippet_emits_completed_nodes_only_when_opted_in() { + let rendered = heartbeat_snippet( + "http://localhost:18800", + "session-123", + "session-123-worker-1", + "completed", + "Finished exact nodes", + &["T2", "T3"], + ); + let body = rendered + .lines() + .find(|line| line.starts_with('{')) + .expect("heartbeat JSON body"); + let parsed: serde_json::Value = serde_json::from_str(body).unwrap(); + assert_eq!(parsed["completed_nodes"], serde_json::json!(["T2", "T3"])); } #[test] diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 4c211c52..ed724992 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Hive Manager", - "version": "0.46.2", + "version": "0.47.0", "identifier": "com.rduff.hive-manager", "build": { "beforeDevCommand": "npm run dev", diff --git a/src/lib/components/workgraph/NodeInspector.svelte b/src/lib/components/workgraph/NodeInspector.svelte index f3910755..8a0bcc18 100644 --- a/src/lib/components/workgraph/NodeInspector.svelte +++ b/src/lib/components/workgraph/NodeInspector.svelte @@ -1,4 +1,17 @@