diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 07482cfffb..7f35a6134a 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -53,15 +53,32 @@ jobs: run: | cargo test --locked -p roboco-harness --lib opencode:: cargo test --locked -p roboco-harness --test opencode + - name: Codex native driver integration tests + # t3-orchestration-v2 ticket 17: the fake app-server suite covers + # the native fork (thread/fork + fallback paste) and provider turn + # refs; it was previously not run by any workflow. + run: cargo test --locked -p roboco-harness --test codex + - name: Claude native driver integration tests + # t3-orchestration-v2 ticket 18: the fake-CLI suite covers the + # native fork (--resume + --fork-session + --resume-session-at, the + # refused-flag and missing-session budgeted pastes) and provider + # turn refs (message uuids); it was previously not run by any + # workflow (unix-gated like the fake CLI itself). + run: cargo test --locked -p roboco-harness --test claude - name: Pi native driver integration tests run: | cargo test --locked -p roboco-harness --test pi - cargo test --locked -p roboco-engine --test pi_resume + cargo test --locked -p roboco-engine --test pi_resume --test pi_steering --test pi_forks - name: Engine-local storage and recovery run: | cargo test --locked -p roboco-sync --lib cargo test --locked -p roboco-update --lib - cargo test --locked -p roboco-engine --lib --test engine_local --test restart_resume --test codex_subagents --test local_profiles --test project_actions_rpc --test project_action_commands --test project_actions_remote --test remote_settings + # t3-orchestration-v2 ticket 23: the Roboco MCP tool tests (the + # fork_chat seam). The crate's unit tests were previously not run + # by any workflow. + cargo test --locked -p roboco-mcp + cargo test --locked -p roboco-engine --lib --test account_failover --test account_sign_in_state --test agent_origin --test codex_subagents --test completion_notices --test continue_on_harness --test engine_local --test harness_switch --test limited_chats --test local_profiles --test project_action_commands --test project_actions_remote --test project_actions_rpc --test remote_settings --test restart_resume --test resume_at_reset --test side_chats + # Mobile Rust core (spec D1, android-app#02): the crates the Android app's # Kotlin painter leans on. Pure Rust, no GPUI system deps. diff --git a/.scratch/upstream-ports-2026-10-wave4/issues/30-pi-native-driver-parity-review.md b/.scratch/upstream-ports-2026-10-wave4/issues/30-pi-native-driver-parity-review.md index 2d41d4f5fe..c62cc5e59e 100644 --- a/.scratch/upstream-ports-2026-10-wave4/issues/30-pi-native-driver-parity-review.md +++ b/.scratch/upstream-ports-2026-10-wave4/issues/30-pi-native-driver-parity-review.md @@ -167,6 +167,20 @@ Verification: reading + `git show` only (ticket 30's budget); no code changed, no cargo. Test execution deferred to the wave-final batched pass (user directive) — nothing here to execute. +- **Reversal (2026-10-16, orchestration v2 ticket 03 — + `.scratch/t3-orchestration-v2/issues/03-pi-steers-mid-turn.md`):** items + 8 and 9 above are superseded. The T3 study measured that our + TurnBoundary classification held every message sent to a working Pi chat + in the engine queue until `agent_settled` — "our steering rides pi's own + steer queue" was true on the wire and false in delivery. Ticket 02 + already moved the driver to upstream's `prompt` + + `streamingBehavior: "steer"` shape; ticket 03 flips the declaration and + the registry descriptor to `StepBoundary` together, selects steering + mode `"all"` at session start unless the user configured one (upstream + #630's overwrite guard now fully applies), and covers both with tests + (fixture `set_steering_mode` handling, an engine-level pi steering test + driving the real queue/steer flow). + - Wave-final batched verification (2026-10-03, merged main `cf94f415`): one batched pass over all lanes — ui lib 1521/1521; engine 529/530 (the one failure is the documented pre-existing diff --git a/CONTEXT.md b/CONTEXT.md index 81049767ce..7207859391 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -44,10 +44,27 @@ _Avoid_: API version, app version (that is `version`) The durable conversation and work history with a harness. Owns its transcript, configuration, and message queue; survives harness restarts. _Avoid_: Conversation, thread, session (when it means the chat) +**Side chat**: +A chat with a parent chat: forked from it in the UI, opened blank next to it, or created by its agent through the Roboco MCP tools. Agent-created side chats record their creator; user-created ones never notify it. Lists under the parent instead of the sidebar's main list. +_Avoid_: Child chat, subagent (that is a harness-spawned agent inside a run) + +**Completion notice**: +The engine-authored system message that tells an agent-created side chat's creator how a turn it started ended: finished, failed (one-line reason), stopped, interrupted (lost to an engine restart without auto-resume), or waiting for input. One merged notice per delivery; it points the parent at `read_chat` rather than carrying the side chat's output. Outcomes the parent already observed (through `wait_for_turn`, a send with wait, `read_chat` or `get_chat`) owe no notice; stopping the parent drops the notices owed to it; at most 25 consecutive notice-started turns may run without a user message in between — past the cap, notices park in the parent's queue, paused, until the user acts. +_Avoid_: Report, callback, acknowledgement + **Chat status**: A chat's live run state: idle, working, awaiting input, or errored. _Avoid_: Session, state (when it means chat status) +**Limited**: +An errored chat whose last run ended on a harness usage limit — the additive limit record (harness, window, reset time, account) on its status row. Offers Resume, Resume at reset, Resume on another account, and Continue on another harness (a one-way harness switch followed by the same continue prompt); queued messages wait behind the continuation. Not a separate wire status: the chat stays errored. + +_Avoid_: Rate-limited (that is the harness event), throttled, capped + +**Resume at reset**: +A pending continuation the engine owns for a Limited chat: at the limit's reset time it sends the continue prompt like a manual Resume. Durable across engine restarts (an overdue one fires after boot); cancelled by a new message, archive, or delete. Armed by the banner action or the "Auto-resume limited chats" engine setting. +_Avoid_: Scheduled task, retry, snooze + **Chat doc**: The chat's durable document — transcript, command ledger, and message queue in one doc, owned by the engine. _Avoid_: Session doc @@ -80,6 +97,10 @@ _Avoid_: Message (a command is not a message), request A user message waiting in the chat's queue, optionally held until the turn ends, delivered in order. _Avoid_: Draft, pending message +**Harness switch**: +A one-way switch of an existing chat to another harness between turns: the engine records a "Switched to " seam in the transcript, drops the chat's harness session, and the next send starts a fresh session carrying the budgeted history of everything above the seam. Switching back is another switch, never a resume. +_Avoid_: Harness handoff, migration, resume (the old session is dropped, not resumed) + ## Harnesses **Harness**: diff --git a/crates/doc/src/commands.rs b/crates/doc/src/commands.rs index f793606eeb..e63f6a47a2 100644 --- a/crates/doc/src/commands.rs +++ b/crates/doc/src/commands.rs @@ -91,6 +91,12 @@ pub struct SessionCommandEntry { pub status: SessionCommandStatus, #[serde(default)] pub resolution: Option, + /// The chat this Run/Steer was sent BY through the Roboco MCP server — + /// structured attribution (additive; ADR 0005). None for user-typed + /// sends and for rows written before the field existed. Executing it + /// stamps the same origin on the user message entry it writes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin_chat_id: Option, } impl SessionCommandEntry { @@ -187,6 +193,7 @@ mod tests { expires_at: None, status: SessionCommandStatus::Pending, resolution: None, + origin_chat_id: None, } } @@ -348,6 +355,39 @@ mod tests { assert!(!can_composer_cancel(&applied, "device-a")); } + #[test] + fn run_and_steer_entries_carry_their_origin_chat_additively() { + for payload in [ + SessionCommandPayload::Run { + request: run_request(), + message_id: "m1".into(), + }, + SessionCommandPayload::Steer { + prompt: "go".into(), + message_id: Some("m2".into()), + }, + ] { + let sent = SessionCommandEntry { + origin_chat_id: Some("chat-origin-1".into()), + ..entry("c1", payload, 1_000) + }; + let value = serde_json::to_value(&sent).unwrap(); + assert_eq!(value["originChatId"], "chat-origin-1", "{value}"); + assert_eq!( + serde_json::from_value::(value).unwrap(), + sent + ); + } + // Rows from older writers (and user-typed sends) carry no origin. + let plain = steer("c1", 1_000); + let value = serde_json::to_value(&plain).unwrap(); + assert!(value.get("originChatId").is_none()); + assert_eq!( + serde_json::from_value::(value).unwrap(), + plain + ); + } + fn run_request() -> RunRequest { RunRequest { prompt: "hello".into(), @@ -362,6 +402,8 @@ mod tests { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } } diff --git a/crates/doc/src/parts.rs b/crates/doc/src/parts.rs index d6c49073a9..98fcab99dc 100644 --- a/crates/doc/src/parts.rs +++ b/crates/doc/src/parts.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; use ts_rs::TS; -use roboco_proto::{AgentEvent, SUBAGENT_INPUT_KEEP, ToolCall, ToolDiff, UserInputQuestion}; +use roboco_proto::{AgentEvent, HarnessId, SUBAGENT_INPUT_KEEP, ToolCall, ToolDiff, UserInputQuestion}; use crate::constants::MSG_INLINE_MAX; @@ -120,6 +120,57 @@ pub enum SubagentStatus { Failed, } +/// What the engine recorded when a notifiable side-chat turn came to rest +/// (orchestration v2). The outcome of one row of a completion notice. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +pub enum NoticeOutcome { + Completed, + Errored, + Stopped, + /// The run was lost to an engine restart without auto-resume (ticket + /// 13): the work never finished — distinct from a stop, which the + /// user chose. + Interrupted, + WaitingForInput, +} + +impl NoticeOutcome { + /// The outcome's verb phrase — the ONE vocabulary the engine's notice + /// text (`"title" (id) `) and every client's notice card share, so + /// the card and the prose an agent reads never drift (the web mirrors + /// it in `noticeOutcomeVerb`). + pub fn verb(self) -> &'static str { + match self { + Self::Completed => "finished", + Self::Errored => "failed", + Self::Stopped => "was stopped", + Self::Interrupted => "was interrupted", + Self::WaitingForInput => "is waiting for input", + } + } +} + +/// One side chat a completion notice reports: its id and title, the +/// recorded outcome, and the one-line reason (error text, restart reason or +/// question summary) when the outcome carries one. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +pub struct NoticeSide { + /// The side chat's full id — clients open the chat with it; the notice's + /// text names the short form. + pub chat_id: String, + /// The side chat's title as of the notice, so the transcript reads it + /// without a registry lookup (a later rename never rewrites history). + pub title: String, + pub outcome: NoticeOutcome, + /// The one-line detail for errored (reason), interrupted (restart + /// reason) and waiting-for-input (question summary) rows; absent + /// otherwise. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + /// One rendered part of an assistant message. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TS)] #[serde(tag = "kind", rename_all = "camelCase")] @@ -219,6 +270,39 @@ pub enum MessagePart { /// source does not rewrite history. source_title: String, }, + /// The seam a one-way harness switch leaves in a chat's transcript + /// (t3-orchestration-v2 ticket 24): everything above ran on another + /// harness whose provider session the switch dropped; the next send on + /// `harness` carries the budgeted history of everything above. Written + /// once per switch by the engine; renders as a "Switched to " + /// divider. Old desktop builds' unknown-kind fallback yields an empty + /// text part (invisible). There is no switch BACK: returning to the old + /// harness is just another one-way switch. + #[serde(rename_all = "camelCase")] + HarnessSwitch { + id: String, + /// The harness the chat switched TO (wire name, e.g. "codex"). + harness: HarnessId, + }, + /// The engine's completion notice (orchestration v2): an engine-authored + /// SYSTEM entry in a parent chat reporting the recorded outcomes of its + /// agent-created side chats' turns — finished, failed (one-line reason), + /// stopped, interrupted by an engine restart without auto-resume, or + /// waiting for input (one-line summary). The entry's sibling + /// text part carries the same notice as plain text, so older layouts — + /// whose unknown-kind fallback yields an empty part — stay readable. + /// Written only by the engine when it delivers a notice; never a user + /// message, never agent content. + #[serde(rename_all = "camelCase")] + CompletionNotice { + id: String, + /// One row per side chat the notice reports (a single outcome when + /// one side chat finished; several when outcomes merged — ticket 13). + /// Defaults for leniency: a partial row reads as an empty notice + /// rather than a decode failure. + #[serde(default)] + sides: Vec, + }, } impl MessagePart { @@ -230,7 +314,9 @@ impl MessagePart { | MessagePart::Tool { id, .. } | MessagePart::Input { id, .. } | MessagePart::Error { id, .. } - | MessagePart::Fork { id, .. } => id, + | MessagePart::Fork { id, .. } + | MessagePart::HarnessSwitch { id, .. } + | MessagePart::CompletionNotice { id, .. } => id, } } @@ -268,6 +354,19 @@ impl MessagePart { source_title, .. } => source_chat_id.len() + source_title.len(), + MessagePart::HarnessSwitch { harness, .. } => { + // Fixed-size structure (the wire name): never grows, so it + // never drives a continuation split. + serde_json::to_vec(harness).map_or(0, |v| v.len()) + } + MessagePart::CompletionNotice { sides, .. } => sides + .iter() + .map(|side| { + side.chat_id.len() + + side.title.len() + + side.detail.as_ref().map_or(0, String::len) + }) + .sum(), } } } @@ -424,7 +523,7 @@ pub fn fold_event_into_parts(out: &mut Vec, event: &AgentEvent) { } } } - AgentEvent::Error { message } => { + AgentEvent::Error { message, .. } => { let id = format!("e{}", out.len()); out.push(MessagePart::Error { id, @@ -496,11 +595,15 @@ pub fn fold_event_into_parts(out: &mut Vec, event: &AgentEvent) { // AvailableCommands feeds the engine's per-harness command cache, not // the transcript. UserMessage becomes its own doc ENTRY (the engine's // subagent sink writes it), never a part of the assistant message. + // TurnRef names the provider turn the streaming entry belongs to — + // engine bookkeeping (stamped when the segment finishes), never a + // rendered part. AgentEvent::AssistantMessageCompleted { .. } | AgentEvent::Usage { .. } | AgentEvent::ContextUsage { .. } | AgentEvent::AvailableCommands { .. } - | AgentEvent::UserMessage { .. } => {} + | AgentEvent::UserMessage { .. } + | AgentEvent::TurnRef { .. } => {} } } diff --git a/crates/doc/src/queue.rs b/crates/doc/src/queue.rs index f6d0387e6d..3514d5bf90 100644 --- a/crates/doc/src/queue.rs +++ b/crates/doc/src/queue.rs @@ -50,6 +50,20 @@ pub struct QueuedMessage { /// `ReviewRequired`; they never silently become sendable again. #[serde(default, skip_serializing_if = "Option::is_none")] pub delivery_gate: Option, + /// The chat this row was sent BY through the Roboco MCP server — + /// structured attribution (additive; ADR 0005). None for user-typed rows + /// and rows written before the field existed. The promoted user message + /// entry keeps it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin_chat_id: Option, + /// The loop cap's parked completion notice (orchestration v2, ticket 14, + /// additive; ADR 0005): the sides the notice reports, so the row — the + /// notice's durable carrier while it waits for the user — releases as + /// the same System completion-notice entry a direct delivery writes, + /// never as a user message. `None` for every other row. A later capped + /// delivery merges its sides into the existing row's. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub completion_notice: Option>, } /// Why a queued row is not currently eligible for automatic or explicit @@ -97,6 +111,8 @@ impl QueuedMessage { issued_at: 0, edited_at: None, delivery_gate: None, + origin_chat_id: None, + completion_notice: None, } } } @@ -360,6 +376,15 @@ fn write_queued_map(map: &loro::LoroMap, item: &QueuedMessage) -> Result<(), Doc crate::schema::loro_value_from_json(&serde_json::to_value(gate)?), )?; } + if let Some(origin_chat_id) = &item.origin_chat_id { + map.insert("originChatId", origin_chat_id.as_str())?; + } + if let Some(sides) = &item.completion_notice { + map.insert( + "completionNotice", + crate::schema::loro_value_from_json(&serde_json::to_value(sides)?), + )?; + } Ok(()) } @@ -403,6 +428,13 @@ fn queued_from_json(v: serde_json::Value) -> Option { }) }), }, + origin_chat_id: v + .get("originChatId") + .and_then(|o| o.as_str()) + .map(str::to_string), + completion_notice: v + .get("completionNotice") + .and_then(|sides| serde_json::from_value(sides.clone()).ok()), }) } @@ -424,6 +456,8 @@ mod tests { issued_at: 1_000, edited_at: None, delivery_gate: None, + origin_chat_id: None, + completion_notice: None, } } @@ -482,6 +516,52 @@ mod tests { assert!(!rows[1].hold_for_turn_end); } + #[test] + fn origin_round_trips_and_old_rows_read_as_unattributed() { + let doc = doc(); + let mut sent = item("q1", "from an agent"); + sent.origin_chat_id = Some("chat-origin-1".into()); + doc.push_queued(&sent).unwrap(); + doc.push_queued(&item("q2", "typed by the user")).unwrap(); + + let rows = doc.read_queue().unwrap(); + assert_eq!(rows[0].origin_chat_id.as_deref(), Some("chat-origin-1")); + assert_eq!(rows[1].origin_chat_id, None); + // The wire name is camelCase and additive: an old writer's row + // (simulated by dropping the field) keeps reading. + let json = serde_json::to_value(rows).unwrap(); + assert_eq!(json[0]["originChatId"], "chat-origin-1"); + assert!(json[1].get("originChatId").is_none()); + let legacy = serde_json::from_value::>(json).unwrap(); + assert_eq!(legacy[0].origin_chat_id.as_deref(), Some("chat-origin-1")); + } + + #[test] + fn completion_notice_sides_round_trip_and_old_rows_read_as_plain() { + let doc = doc(); + let mut parked = item("notice:1", "Side chat reached outcomes."); + parked.completion_notice = Some(vec![crate::NoticeSide { + chat_id: "chat-side-1234".into(), + title: "Implement login".into(), + outcome: crate::NoticeOutcome::Completed, + detail: None, + }]); + doc.push_queued(&parked).unwrap(); + doc.push_queued(&item("q1", "typed by the user")).unwrap(); + + let rows = doc.read_queue().unwrap(); + assert_eq!(rows[0].completion_notice, parked.completion_notice); + assert_eq!(rows[1].completion_notice, None); + // The wire name is camelCase and additive: an old writer's row + // (simulated by dropping the field) keeps reading as a plain row. + let json = serde_json::to_value(rows).unwrap(); + assert_eq!(json[0]["completionNotice"][0]["chatId"], "chat-side-1234"); + assert!(json[1].get("completionNotice").is_none()); + let legacy = serde_json::from_value::>(json).unwrap(); + assert_eq!(legacy[0].completion_notice, parked.completion_notice); + assert_eq!(legacy[1].completion_notice, None); + } + #[test] fn delivery_gate_round_trips_and_finish_clears_it_atomically() { let doc = doc(); diff --git a/crates/doc/src/rebuild.rs b/crates/doc/src/rebuild.rs index a2d31e11ce..52e88a01c5 100644 --- a/crates/doc/src/rebuild.rs +++ b/crates/doc/src/rebuild.rs @@ -156,6 +156,7 @@ mod tests { fn fat_entry(id: &str) -> SessionMessageEntry { SessionMessageEntry { + origin_chat_id: None, id: id.into(), role: MessageRole::Assistant, parts: vec![ @@ -190,11 +191,13 @@ mod tests { status: Some(MessageStatus::Complete), continuation_of: None, duration_ms: None, + provider_turn: None, } } fn command(id: &str, status: SessionCommandStatus) -> SessionCommandEntry { SessionCommandEntry { + origin_chat_id: None, id: id.into(), payload: SessionCommandPayload::Interrupt {}, issued_by: "dev-b".into(), diff --git a/crates/doc/src/registry.rs b/crates/doc/src/registry.rs index aa6c959cc4..0da96b29d1 100644 --- a/crates/doc/src/registry.rs +++ b/crates/doc/src/registry.rs @@ -1240,6 +1240,14 @@ impl RegistryDoc { chat.room_gen.map(|g| json!(g)).unwrap_or(Value::Null), ), ("parentChatId", opt_str(chat.parent_chat_id.as_deref())), + ( + "agentOrigin", + chat.agent_origin + .as_ref() + .map(serde_json::to_value) + .transpose()? + .unwrap_or(Value::Null), + ), ]); self.write(KIND_CHATS, &chat.id.clone(), OpKind::Upsert, set); Ok(()) @@ -1493,6 +1501,14 @@ impl RegistryDoc { /// Upsert a session-status row (writer discipline: each device writes only /// its own runs' rows). Staleness is checked client-side via `updatedAt`. pub fn upsert_session(&mut self, session: &Session) -> Result<(), DocError> { + let limit = match &session.limit { + Some(limit) => serde_json::to_value(limit)?, + None => Value::Null, + }; + let scheduled_resume = match &session.scheduled_resume { + Some(scheduled) => serde_json::to_value(scheduled)?, + None => Value::Null, + }; let set = fields([ ("chatId", json!(session.chat_id)), ("deviceId", json!(session.device_id)), @@ -1500,6 +1516,8 @@ impl RegistryDoc { ("lastCompletedTurn", json!(session.last_completed_turn)), ("startedAt", opt_ms(session.started_at)), ("updatedAt", json!(session.updated_at.timestamp_millis())), + ("limit", limit), + ("scheduledResume", scheduled_resume), ]); self.write(KIND_SESSIONS, &session.chat_id.clone(), OpKind::Upsert, set); Ok(()) @@ -1626,11 +1644,27 @@ impl RegistryDoc { ("spaceId", opt_str(chat.space_id.as_deref())), ("lastSeenAt", opt_ms(chat.last_seen_at)), ("parentChatId", opt_str(chat.parent_chat_id.as_deref())), + ( + "agentOrigin", + chat.agent_origin + .as_ref() + .map(serde_json::to_value) + .transpose()? + .unwrap_or(Value::Null), + ), ]), ); } for session in &state.sessions { let ms = newest(&[Some(session.updated_at), session.started_at]); + let limit = match &session.limit { + Some(limit) => serde_json::to_value(limit)?, + None => Value::Null, + }; + let scheduled_resume = match &session.scheduled_resume { + Some(scheduled) => serde_json::to_value(scheduled)?, + None => Value::Null, + }; seed( KIND_SESSIONS, &session.chat_id, @@ -1642,6 +1676,8 @@ impl RegistryDoc { ("lastCompletedTurn", json!(session.last_completed_turn)), ("startedAt", opt_ms(session.started_at)), ("updatedAt", json!(session.updated_at.timestamp_millis())), + ("limit", limit), + ("scheduledResume", scheduled_resume), ]), ); } diff --git a/crates/doc/src/registry/tests.rs b/crates/doc/src/registry/tests.rs index 452ebb35ca..a1b3b84338 100644 --- a/crates/doc/src/registry/tests.rs +++ b/crates/doc/src/registry/tests.rs @@ -252,6 +252,7 @@ fn device(id: &str, name: &str) -> Device { fn chat(id: &str, device_id: &str) -> Chat { Chat { + agent_origin: None, id: id.into(), device_id: device_id.into(), title: Some("First chat".into()), @@ -301,6 +302,8 @@ fn session(chat_id: &str, device_id: &str, status: SessionStatus) -> Session { status, started_at: Some(ts(3_000)), updated_at: ts(3_500), + limit: None, + scheduled_resume: None, } } @@ -375,6 +378,21 @@ fn rows_round_trip_and_upsert_refreshes() { assert_eq!(chats[0].last_message_preview.as_deref(), Some("hello")); } +/// The armed Resume-at-reset continuation rides the session row (remote +/// banners' countdown), and a cancel clears it. +#[test] +fn the_scheduled_resume_rides_the_session_row() { + let mut doc = RegistryDoc::new("dev-a"); + let mut row = session("chat-1", "dev-a", SessionStatus::Errored); + row.scheduled_resume = Some(roboco_proto::ScheduledResume { fire_at: 9_000 }); + doc.upsert_session(&row).unwrap(); + assert_eq!(doc.read_sessions().unwrap(), vec![row.clone()]); + + row.scheduled_resume = None; + doc.upsert_session(&row).unwrap(); + assert_eq!(doc.read_sessions().unwrap()[0].scheduled_resume, None); +} + #[test] fn own_push_ack_never_advances_the_cursor() { // Field incident: on the HTTPS transport, push ran before pull in one diff --git a/crates/doc/src/schema.rs b/crates/doc/src/schema.rs index 998ae67a13..ca14038f28 100644 --- a/crates/doc/src/schema.rs +++ b/crates/doc/src/schema.rs @@ -4,14 +4,14 @@ //! - `meta`: LoroMap { chatId: string, schemaVersion: number } (host-only writer) //! - `messages`: LoroList of LoroMap { //! id, role, parts: LoroList, createdAt, deviceId, status?, -//! continuationOf?, durationMs? } +//! continuationOf?, durationMs?, providerTurn? } //! - `commands`: LoroList of LoroMap { //! id, kind, payload(json), issuedBy, issuedAt, basedOn?, expiresAt?, status, resolution? } //! - `queue`: LoroMovableList of LoroMap { //! id, text, attachments?, issuedBy, issuedAt, editedAt? } (any device writes) //! -//! Part maps: { id, kind: "text"|"reasoning"|"tool"|"input"|"error"|"image", text?: LoroText, -//! reasoning?: LoroText, call?: json, isError?, questions?: json, resolved?, message? }. +//! Part maps: { id, kind: "text"|"reasoning"|"tool"|"input"|"error"|"image"|"fork"|"completionNotice", +//! text?: LoroText, reasoning?: LoroText, call?: json, isError?, questions?: json, resolved?, message? }. //! Text bodies are **LoroText** so streaming appends RLE-merge (1.03x oplog overhead vs //! 125x for whole-value rewrites). @@ -60,6 +60,20 @@ pub struct SessionMessageEntry { /// the field existed. #[serde(default, skip_serializing_if = "Option::is_none")] pub duration_ms: Option, + /// The chat this user entry was sent BY through the Roboco MCP server — + /// structured attribution (additive; ADR 0005), the durable record the + /// completion-notice rules key on. None on typed messages and entries + /// written before the field existed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin_chat_id: Option, + /// The provider turn that produced this assistant entry (additive, + /// t3-orchestration-v2 ticket 17): harness, provider session id, and the + /// harness's own turn ref. A fork continues the provider session at + /// exactly this entry using it. Absent on user/system rows and on docs + /// written before the field existed — those fall back to the budgeted + /// paste. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_turn: Option, } /// The doc-resident flat part map (`DocMessagePart` in TS). Distinct from the app-layer @@ -99,6 +113,11 @@ struct DocPartJson { source_chat_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] source_title: Option, + /// Harness-switch seam (`kind: "harnessSwitch"`, additive): the harness + /// (wire name) the chat switched to; everything above the seam ran on + /// another harness whose provider session the switch dropped. + #[serde(default, skip_serializing_if = "Option::is_none")] + harness: Option, /// Tool output summary (additive — absent on old rows and old writers; /// pre-strip writers stored up to 4KB of capped output here). #[serde(default, skip_serializing_if = "Option::is_none")] @@ -127,6 +146,11 @@ struct DocPartJson { /// One-line live tail of the subagent's output (additive). #[serde(default, skip_serializing_if = "Option::is_none")] subagent_tail: Option, + /// Completion-notice rows (kind: "completionNotice", additive): the + /// side chats and their recorded outcomes an engine-authored notice + /// reports. + #[serde(default, skip_serializing_if = "Option::is_none")] + sides: Option, } /// App parts → doc part json (mirror of `toDocParts`). @@ -225,6 +249,20 @@ fn to_doc_part(part: &MessagePart) -> Result { source_title: Some(source_title.clone()), ..Default::default() }, + MessagePart::HarnessSwitch { id, harness } => DocPartJson { + id: id.clone(), + kind: "harnessSwitch".into(), + harness: serde_json::to_value(harness) + .ok() + .and_then(|value| value.as_str().map(str::to_owned)), + ..Default::default() + }, + MessagePart::CompletionNotice { id, sides } => DocPartJson { + id: id.clone(), + kind: "completionNotice".into(), + sides: Some(serde_json::to_value(sides)?), + ..Default::default() + }, }) } @@ -280,6 +318,31 @@ fn from_doc_part(p: DocPartJson) -> MessagePart { source_chat_id: p.source_chat_id.unwrap_or_default(), source_title: p.source_title.unwrap_or_default(), }, + "harnessSwitch" => { + // A harness name this build does not know (a future wire value) + // degrades to an invisible empty part rather than failing the doc. + let harness = p.harness.and_then(|name| { + serde_json::from_value::(serde_json::Value::String(name)) + .ok() + }); + match harness { + Some(harness) => MessagePart::HarnessSwitch { + id: p.id, + harness, + }, + None => MessagePart::Text { + id: p.id, + text: String::new(), + }, + } + } + "completionNotice" => MessagePart::CompletionNotice { + id: p.id, + sides: p + .sides + .and_then(|s| serde_json::from_value(s).ok()) + .unwrap_or_default(), + }, _ => MessagePart::Text { id: p.id, text: p.text.unwrap_or_default(), @@ -395,6 +458,118 @@ impl SessionDoc { Ok(()) } + /// The chat's outstanding native-fork intent (additive meta, + /// t3-orchestration-v2 ticket 17): the source harness, provider session + /// id, and inclusive boundary recorded by the ForkSideChat RPC. The + /// fork's first dispatch consumes it — the driver then forks the source + /// provider session instead of resuming or starting fresh. + pub fn provider_fork(&self) -> Option { + let meta = self.doc.get_map("meta"); + let str_field = |key: &str| match meta.get(key) { + Some(loro::ValueOrContainer::Value(LoroValue::String(s))) => Some(s.to_string()), + _ => None, + }; + let fork = roboco_proto::ProviderFork { + session_id: str_field("forkSourceSession")?, + last_turn: str_field("forkSourceTurn")?, + fallback_history: None, + }; + (!fork.session_id.is_empty() && !fork.last_turn.is_empty()).then_some(fork) + } + + /// The recorded source harness of the native-fork intent, when present. + /// A run on any other harness must fall back to the budgeted paste. + pub fn provider_fork_harness(&self) -> Option { + match self.doc.get_map("meta").get("forkSourceHarness") { + Some(loro::ValueOrContainer::Value(LoroValue::String(s))) => { + serde_json::from_value(serde_json::Value::String(s.to_string())).ok() + } + _ => None, + } + } + + /// Record the native-fork intent: the source harness's provider session + /// and the inclusive boundary turn the fork continues at. One commit, so + /// a partial write is never observable. + pub fn set_provider_fork( + &self, + harness: roboco_proto::HarnessId, + fork: &roboco_proto::ProviderFork, + ) -> Result<(), DocError> { + let harness = serde_json::to_value(harness) + .ok() + .and_then(|v| v.as_str().map(str::to_owned)) + .ok_or_else(|| DocError::Schema("harness wire name not a string".into()))?; + let meta = self.doc.get_map("meta"); + meta.insert("forkSourceHarness", harness)?; + meta.insert("forkSourceSession", fork.session_id.as_str())?; + meta.insert("forkSourceTurn", fork.last_turn.as_str())?; + self.doc.commit(); + Ok(()) + } + + /// Consume the native-fork intent for a run on `harness`: the recorded + /// spec when the source harness matches, `None` otherwise (the intent + /// is spent either way — a changed harness falls back to the budgeted + /// paste and never comes back to the native path). + pub fn take_provider_fork( + &self, + harness: roboco_proto::HarnessId, + ) -> Option { + let fork = self.provider_fork(); + let recorded = self.provider_fork_harness(); + self.clear_provider_fork(); + (recorded == Some(harness)).then_some(fork).flatten() + } + + /// Drop the native-fork intent (consumed, mismatched, or superseded). + /// No-op when nothing is recorded — dispatch consults the intent on + /// every side-chat send, and a doc commit per dispatch would be pure + /// write amplification. + pub fn clear_provider_fork(&self) { + if self.provider_fork().is_none() && self.provider_fork_harness().is_none() { + return; + } + let meta = self.doc.get_map("meta"); + for key in ["forkSourceHarness", "forkSourceSession", "forkSourceTurn"] { + let _ = meta.delete(key); + } + self.doc.commit(); + } + + /// Stamp a provider turn reference onto a transcript entry (additive): + /// the harness, provider session id and per-harness turn ref that + /// produced it. A fork uses the boundary entry's ref to continue the + /// provider session at exactly that turn. No-op for a missing entry — + /// a segment that never streamed has nothing to stamp. + pub fn stamp_provider_turn( + &self, + entry_id: &str, + reference: &roboco_proto::ProviderTurnRef, + ) -> Result<(), DocError> { + let messages = self.doc.get_list("messages"); + for index in 0..messages.len() { + let Some(loro::ValueOrContainer::Container(loro::Container::Map(map))) = + messages.get(index) + else { + continue; + }; + let matches = matches!(map.get("id"), + Some(loro::ValueOrContainer::Value(LoroValue::String(s))) + if s.as_str() == entry_id); + if !matches { + continue; + } + map.insert( + "providerTurn", + loro_value_from_json(&serde_json::to_value(reference)?), + )?; + self.doc.commit(); + return Ok(()); + } + Ok(()) + } + pub fn chat_id(&self) -> Option { match self.doc.get_map("meta").get("chatId") { Some(loro::ValueOrContainer::Value(LoroValue::String(s))) => Some(s.to_string()), @@ -554,6 +729,9 @@ impl SessionDoc { .as_str() .ok_or_else(|| DocError::Schema("status not a string".into()))?, )?; + if let Some(origin_chat_id) = &entry.origin_chat_id { + map.insert("originChatId", origin_chat_id.as_str())?; + } self.doc.commit(); Ok(()) } @@ -815,6 +993,15 @@ fn write_entry_scalar_fields(map: &LoroMap, entry: &SessionMessageEntry) -> Resu if let Some(duration_ms) = entry.duration_ms { map.insert("durationMs", duration_ms)?; } + if let Some(origin_chat_id) = &entry.origin_chat_id { + map.insert("originChatId", origin_chat_id.as_str())?; + } + if let Some(reference) = &entry.provider_turn { + map.insert( + "providerTurn", + loro_value_from_json(&serde_json::to_value(reference)?), + )?; + } Ok(()) } @@ -848,6 +1035,7 @@ fn push_part(parts: &LoroList, part: &MessagePart) -> Result<(), DocError> { ("mimeType", &doc_part.mime_type), ("sourceChatId", &doc_part.source_chat_id), ("sourceTitle", &doc_part.source_title), + ("harness", &doc_part.harness), ] { if let Some(value) = value { map.insert(key, value.as_str())?; @@ -895,6 +1083,9 @@ fn push_part(parts: &LoroList, part: &MessagePart) -> Result<(), DocError> { if let Some(subagent_tail) = &doc_part.subagent_tail { map.insert("subagentTail", subagent_tail.as_str())?; } + if let Some(sides) = &doc_part.sides { + map.insert("sides", loro_value_from_json(sides))?; + } Ok(()) } @@ -914,6 +1105,10 @@ fn entry_from_json(v: serde_json::Value) -> Result, #[serde(default)] duration_ms: Option, + #[serde(default)] + origin_chat_id: Option, + #[serde(default)] + provider_turn: Option, } match serde_json::from_value::(v.clone()) { Ok(raw) => Ok(SessionMessageEntry { @@ -925,6 +1120,8 @@ fn entry_from_json(v: serde_json::Value) -> Result SegmentWriter<'a> { write_entry_scalar_fields( &map, &SessionMessageEntry { + origin_chat_id: None, id: entry_id.into(), role: MessageRole::Assistant, parts: vec![], @@ -1134,6 +1336,7 @@ impl<'a> SegmentWriter<'a> { status: Some(MessageStatus::Streaming), continuation_of: None, duration_ms: None, + provider_turn: None, }, )?; map.insert_container("parts", LoroList::new())?; @@ -1408,7 +1611,9 @@ mod tests { source_title: "Main conversation".into(), }; doc.push_message(&SessionMessageEntry { + origin_chat_id: None, duration_ms: None, + provider_turn: None, id: "fork:side".into(), role: MessageRole::System, parts: vec![seam.clone()], @@ -1424,6 +1629,200 @@ mod tests { assert_eq!(entries[0].parts, vec![seam]); } + /// The one-way harness switch seam (t3-orchestration-v2 ticket 24): a + /// SYSTEM entry whose marker part round-trips through the doc with the + /// harness wire name. Additive (ADR 0005): an older reader without the + /// kind degrades to an invisible empty part; a row from an older writer + /// simply never carries one. + #[test] + fn harness_switch_seam_round_trips_through_the_doc() { + let doc = SessionDoc::init("switch").unwrap(); + let seam = MessagePart::HarnessSwitch { + id: "switch:1".into(), + harness: roboco_proto::HarnessId::Codex, + }; + doc.push_message(&SessionMessageEntry { + origin_chat_id: None, + duration_ms: None, + provider_turn: None, + id: "switch:1".into(), + role: MessageRole::System, + parts: vec![seam.clone()], + created_at: 1, + device_id: "dev".into(), + status: Some(MessageStatus::Complete), + continuation_of: None, + }) + .unwrap(); + let entries = doc.read_entries().unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].role, MessageRole::System); + assert_eq!(entries[0].parts, vec![seam]); + } + + /// The engine-authored completion notice: a SYSTEM entry whose marker + /// part (sides + outcomes) round-trips through the doc beside the plain + /// text that keeps older layouts readable. Wire names stay camelCase and + /// additive (ADR 0005): a row from an older writer carries no sides and + /// degrades to an empty text part, never a decode failure. + #[test] + fn completion_notice_round_trips_through_the_doc() { + let doc = SessionDoc::init("notice").unwrap(); + let marker = MessagePart::CompletionNotice { + id: "n0".into(), + sides: vec![ + NoticeSide { + chat_id: "chat-side-1234".into(), + title: "Implement login".into(), + outcome: NoticeOutcome::Completed, + detail: None, + }, + NoticeSide { + chat_id: "chat-side-5678".into(), + title: "Flaky test".into(), + outcome: NoticeOutcome::Errored, + detail: Some("build timed out".into()), + }, + ], + }; + let text = MessagePart::Text { + id: "t0".into(), + text: "2 side chats reached outcomes:".into(), + }; + doc.push_message(&SessionMessageEntry { + origin_chat_id: None, + duration_ms: None, + provider_turn: None, + id: "notice:1".into(), + role: MessageRole::System, + parts: vec![marker.clone(), text.clone()], + created_at: 1, + device_id: "dev".into(), + status: Some(MessageStatus::Complete), + continuation_of: None, + }) + .unwrap(); + let entries = doc.read_entries().unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].role, MessageRole::System); + assert_eq!(entries[0].parts, vec![marker, text]); + + // The wire shape: camelCase, additive, tag "completionNotice". + let value = serde_json::to_value(entries[0].parts.clone()).unwrap(); + assert_eq!(value[0]["kind"], "completionNotice"); + assert_eq!(value[0]["sides"][0]["chatId"], "chat-side-1234"); + assert_eq!(value[0]["sides"][0]["outcome"], "completed"); + assert_eq!(value[0]["sides"][1]["outcome"], "errored"); + assert_eq!(value[0]["sides"][1]["detail"], "build timed out"); + assert!(value[0]["sides"][0].get("detail").is_none()); + // An older writer's marker (no sides) degrades to empty rows. + let legacy = serde_json::json!({ + "kind": "completionNotice", + "id": "n0", + }); + assert_eq!( + serde_json::from_value::(legacy).unwrap(), + MessagePart::CompletionNotice { + id: "n0".into(), + sides: Vec::new(), + } + ); + } + + #[test] + fn provider_turn_round_trips_and_stamps_only_existing_entries() { + let doc = SessionDoc::init("refs").unwrap(); + let reference = roboco_proto::ProviderTurnRef { + harness: roboco_proto::HarnessId::Codex, + session_id: "th-1".into(), + turn_id: "t-1".into(), + }; + // A stamped assistant entry keeps its ref; an untouched sibling does + // not; a missing id is a no-op, never an error. + let mut entry = user_entry("u1", "question"); + entry.role = MessageRole::Assistant; + doc.push_message(&entry).unwrap(); + doc.stamp_provider_turn("u1", &reference).unwrap(); + doc.stamp_provider_turn("gone", &reference).unwrap(); + let mut plain = user_entry("u2", "unmarked"); + plain.role = MessageRole::Assistant; + doc.push_message(&plain).unwrap(); + let entries = doc.read_entries().unwrap(); + assert_eq!(entries[0].provider_turn.as_ref(), Some(&reference)); + assert_eq!(entries[1].provider_turn, None); + // A snapshot import keeps the ref (the fork boundary survives a + // device sync). + let imported = LoroDoc::new(); + imported + .import(&doc.export_snapshot().unwrap()) + .unwrap(); + let loaded = SessionDoc::from_doc(imported); + assert_eq!( + loaded.read_entries().unwrap()[0].provider_turn.as_ref(), + Some(&reference) + ); + } + + #[test] + fn provider_turn_survives_salvage_of_a_torn_entry() { + // A strict parse that fails on an unrelated field (a missing + // deviceId) must still keep the ref — the 2026-08-10 incident rule. + let torn = serde_json::json!({ + "id": "a1", + "role": "assistant", + "parts": [], + "createdAt": 1, + "providerTurn": { + "harness": "codex", + "sessionId": "th-1", + "turnId": "t-9" + } + }); + let salvaged = + salvage_entry(torn, serde_json::from_str::("not json").unwrap_err()) + .unwrap(); + let reference = roboco_proto::ProviderTurnRef { + harness: roboco_proto::HarnessId::Codex, + session_id: "th-1".into(), + turn_id: "t-9".into(), + }; + assert_eq!(salvaged.provider_turn.as_ref(), Some(&reference)); + } + + #[test] + fn provider_fork_intent_is_recorded_and_consumed_once() { + let doc = SessionDoc::init("fork-intent").unwrap(); + assert_eq!(doc.provider_fork(), None); + let fork = roboco_proto::ProviderFork { + session_id: "th-source".into(), + last_turn: "turn-7".into(), + fallback_history: None, + }; + doc.set_provider_fork(roboco_proto::HarnessId::Codex, &fork) + .unwrap(); + assert_eq!(doc.provider_fork_harness(), Some(roboco_proto::HarnessId::Codex)); + assert_eq!(doc.provider_fork(), Some(fork)); + // Consumed by a run on the recorded harness. + assert_eq!(doc.take_provider_fork(roboco_proto::HarnessId::Codex), Some(fork)); + // Consumed once: the next dispatch — same harness or not — has + // nothing left, so it falls back to the budgeted paste. + assert_eq!(doc.take_provider_fork(roboco_proto::HarnessId::Codex), None); + assert_eq!(doc.provider_fork(), None); + // A mismatched harness spends the intent too, without ever taking + // the native path. + doc.set_provider_fork(roboco_proto::HarnessId::Codex, &fork) + .unwrap(); + assert_eq!(doc.take_provider_fork(roboco_proto::HarnessId::Mock), None); + assert_eq!(doc.provider_fork(), None); + // A partial write (session without turn) never reads as an intent. + doc.doc + .get_map("meta") + .insert("forkSourceSession", "th-source") + .unwrap(); + doc.doc.commit(); + assert_eq!(doc.provider_fork(), None); + } + #[test] fn generated_image_persists_updates_and_salvages() { let doc = SessionDoc::init("images").unwrap(); @@ -1468,6 +1867,7 @@ mod tests { fn user_entry(id: &str, text: &str) -> SessionMessageEntry { SessionMessageEntry { + origin_chat_id: None, id: id.into(), role: MessageRole::User, parts: vec![MessagePart::Text { @@ -1479,6 +1879,7 @@ mod tests { status: Some(MessageStatus::Complete), continuation_of: None, duration_ms: None, + provider_turn: None, } } @@ -1487,6 +1888,7 @@ mod tests { let doc = SessionDoc::init("whale").unwrap(); for segment in 0..4 { doc.push_message(&SessionMessageEntry { + origin_chat_id: None, id: format!("segment-{segment}"), role: MessageRole::Assistant, parts: (0..100) @@ -1500,6 +1902,7 @@ mod tests { status: Some(MessageStatus::Complete), continuation_of: (segment > 0).then(|| "segment-0".into()), duration_ms: None, + provider_turn: None, }) .unwrap(); } @@ -1732,6 +2135,7 @@ mod tests { fn resolve_input_stamps_the_part_in_place() { let doc = SessionDoc::init("chat-1").unwrap(); doc.push_message(&SessionMessageEntry { + origin_chat_id: None, id: "m1".into(), role: MessageRole::Assistant, parts: vec![MessagePart::Input { @@ -1746,6 +2150,7 @@ mod tests { status: Some(MessageStatus::Aborted), continuation_of: None, duration_ms: None, + provider_turn: None, }) .unwrap(); assert!(!doc.resolve_input("nope").unwrap()); @@ -1981,6 +2386,7 @@ mod tests { fn pre_strip_doc_parts_still_round_trip() { let doc = SessionDoc::init("chat-3").unwrap(); doc.push_message(&SessionMessageEntry { + origin_chat_id: None, id: "m1".into(), role: MessageRole::Assistant, parts: vec![MessagePart::Tool { @@ -2009,6 +2415,7 @@ mod tests { status: Some(MessageStatus::Complete), continuation_of: None, duration_ms: None, + provider_turn: None, }) .unwrap(); let entries = doc.read_entries().unwrap(); @@ -2046,6 +2453,7 @@ mod tests { use crate::commands::{SessionCommandPayload, SessionCommandStatus}; let doc = SessionDoc::init("chat-1").unwrap(); let entry = SessionCommandEntry { + origin_chat_id: None, id: "c1".into(), payload: SessionCommandPayload::Steer { prompt: "focus".into(), diff --git a/crates/doc/src/transcript_delta.rs b/crates/doc/src/transcript_delta.rs index 1e7395911f..e56077be81 100644 --- a/crates/doc/src/transcript_delta.rs +++ b/crates/doc/src/transcript_delta.rs @@ -360,6 +360,7 @@ mod tests { fn entry(id: &str, text: &str) -> SessionMessageEntry { SessionMessageEntry { + origin_chat_id: None, id: id.into(), role: MessageRole::Assistant, parts: vec![MessagePart::Text { @@ -371,6 +372,7 @@ mod tests { status: None, continuation_of: None, duration_ms: None, + provider_turn: None, } } @@ -545,6 +547,7 @@ mod context_update_tests { #[test] fn replay_cutoff_keeps_only_historical_parts_and_unicode_text_prefixes() { let mut entry = SessionMessageEntry { + origin_chat_id: None, id: "message".into(), role: crate::MessageRole::Assistant, parts: vec![MessagePart::Text { @@ -556,6 +559,7 @@ mod context_update_tests { status: Some(crate::MessageStatus::Streaming), continuation_of: None, duration_ms: None, + provider_turn: None, }; let baseline = TranscriptBaseline::capture(&[entry.clone()]); assert!(baseline.covers(&entry)); diff --git a/crates/doc/src/transcript_resume.rs b/crates/doc/src/transcript_resume.rs index e24b45d7c9..0773a8f55e 100644 --- a/crates/doc/src/transcript_resume.rs +++ b/crates/doc/src/transcript_resume.rs @@ -345,6 +345,7 @@ mod tests { fn entry(id: &str, text: &str) -> SessionMessageEntry { SessionMessageEntry { + origin_chat_id: None, id: id.into(), role: MessageRole::Assistant, parts: vec![MessagePart::Text { @@ -356,6 +357,7 @@ mod tests { status: None, continuation_of: None, duration_ms: None, + provider_turn: None, } } diff --git a/crates/doc/src/workspace.rs b/crates/doc/src/workspace.rs index e8df63adb5..f24333f5c7 100644 --- a/crates/doc/src/workspace.rs +++ b/crates/doc/src/workspace.rs @@ -286,6 +286,13 @@ impl WorkspaceDoc { )?; set_opt_str(&row, "spaceId", chat.space_id.as_deref())?; set_opt_str(&row, "parentChatId", chat.parent_chat_id.as_deref())?; + match &chat.agent_origin { + Some(origin) => row.insert( + "agentOrigin", + LoroValue::from(serde_json::to_value(origin)?), + )?, + None => row.delete("agentOrigin")?, + } set_opt_ms(&row, "lastSeenAt", chat.last_seen_at)?; self.doc.commit(); Ok(()) @@ -485,6 +492,20 @@ impl WorkspaceDoc { )?; set_opt_ms(&row, "startedAt", session.started_at)?; row.insert("updatedAt", session.updated_at.timestamp_millis())?; + match &session.limit { + Some(limit) => row.insert( + "limit", + LoroValue::from(serde_json::to_value(limit)?), + )?, + None => row.delete("limit")?, + } + match &session.scheduled_resume { + Some(scheduled) => row.insert( + "scheduledResume", + LoroValue::from(serde_json::to_value(scheduled)?), + )?, + None => row.delete("scheduledResume")?, + } self.doc.commit(); Ok(()) } @@ -720,6 +741,8 @@ pub(crate) struct RawChat { room_gen: Option, #[serde(default)] parent_chat_id: Option, + #[serde(default)] + agent_origin: Option, } /// Decode a chat row's `config` leniently: unknown enum values (a newer @@ -761,6 +784,7 @@ impl From for Chat { last_seen_at: raw.last_seen_at.map(dt), room_gen: raw.room_gen, parent_chat_id: raw.parent_chat_id, + agent_origin: raw.agent_origin, } } } @@ -777,6 +801,10 @@ pub(crate) struct RawSession { started_at: Option, #[serde(default)] updated_at: i64, + #[serde(default)] + limit: Option, + #[serde(default)] + scheduled_resume: Option, } impl From for Session { @@ -788,6 +816,8 @@ impl From for Session { status: raw.status, started_at: raw.started_at.map(dt), updated_at: dt(raw.updated_at), + limit: raw.limit, + scheduled_resume: raw.scheduled_resume, } } } @@ -846,6 +876,7 @@ mod tests { fn chat(id: &str, device_id: &str) -> Chat { Chat { + agent_origin: None, id: id.into(), device_id: device_id.into(), title: Some("First chat".into()), @@ -895,6 +926,8 @@ mod tests { status, started_at: Some(ts(3_000)), updated_at: ts(3_500), + limit: None, + scheduled_resume: None, } } @@ -971,6 +1004,37 @@ mod tests { assert_eq!(ws.read_sessions().unwrap()[0].last_completed_turn, None); } + /// The usage-limit record rides the session row additively: it + /// round-trips through the doc, a row from an older writer (no such + /// key) reads as unlimited, and clearing it on a later upsert (a new + /// turn started) drops the stale marker instead of keeping it. + #[test] + fn limit_record_round_trips_and_clears_on_the_session_row() { + let ws = WorkspaceDoc::new(); + let mut row = session("chat-1", "dev-a", SessionStatus::Errored); + row.limit = Some(roboco_proto::SessionLimit { + harness: HarnessId::ClaudeCode, + window: "5-hour".into(), + resets_at: Some(1_762_675_400_000), + account_id: Some("slot-9f2c".into()), + }); + ws.upsert_session(&row).unwrap(); + assert_eq!(ws.read_sessions().unwrap(), vec![row.clone()]); + // Old writers never wrote the field: a row without the key parses + // as unlimited. + ws.row("sessions", "chat-1") + .unwrap() + .delete("limit") + .unwrap(); + assert_eq!(ws.read_sessions().unwrap()[0].limit, None); + // A new turn clears the record: the engine upserts the row without + // one, and the stale marker must not survive that write. + let mut resumed = session("chat-1", "dev-a", SessionStatus::Working); + ws.upsert_session(&resumed).unwrap(); + resumed.limit = None; + assert_eq!(ws.read_sessions().unwrap(), vec![resumed]); + } + #[test] fn rows_round_trip() { let ws = WorkspaceDoc::new(); diff --git a/crates/doc/tests/attachments_roundtrip.rs b/crates/doc/tests/attachments_roundtrip.rs index 595acb0893..3816cd2fd7 100644 --- a/crates/doc/tests/attachments_roundtrip.rs +++ b/crates/doc/tests/attachments_roundtrip.rs @@ -19,8 +19,11 @@ fn run_request_attachments_survive_command_round_trip() { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }; doc.queue_command(&SessionCommandEntry { + origin_chat_id: None, id: "c1".into(), payload: SessionCommandPayload::Run { request, diff --git a/crates/engine-client/src/cache.rs b/crates/engine-client/src/cache.rs index 5e5bc6c59c..aaab337c65 100644 --- a/crates/engine-client/src/cache.rs +++ b/crates/engine-client/src/cache.rs @@ -176,7 +176,7 @@ mod tests { core.doc_host .open("same-chat") .unwrap() - .write_user_message("message", "cached history", 123) + .write_user_message("message", "cached history", 123, None) .unwrap(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let url = format!("ws://{}", listener.local_addr().unwrap()); diff --git a/crates/engine/examples/android_fixture.rs b/crates/engine/examples/android_fixture.rs index 299784a2d9..73f2b61705 100644 --- a/crates/engine/examples/android_fixture.rs +++ b/crates/engine/examples/android_fixture.rs @@ -578,6 +578,8 @@ fn seed_request(repo: &Path, chat: &SeedChat) -> RunRequest { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } diff --git a/crates/engine/src/account_failover.rs b/crates/engine/src/account_failover.rs new file mode 100644 index 0000000000..3318dea1bf --- /dev/null +++ b/crates/engine/src/account_failover.rs @@ -0,0 +1,630 @@ +//! Resume on another account (orchestration v2, ticket 09): the engine +//! operation behind a Limited chat's account failover. +//! +//! A Limited **Claude Code** or **Codex** chat can continue on a different +//! harness account of the same harness. The engine: +//! +//! 1. ranks the CACHED harness accounts (no network — see +//! [`AgentAccounts::failover_candidates`]): switchable, not an API key, +//! not signed out, not still limited, not the account that hit the limit +//! nor the live login; lowest usage first, accounts sharing the limited +//! one's org or workspace ranked lower; +//! 2. probes the chosen one once ([`AgentAccounts::probe_account`]) — a +//! probe that reveals a dead login or an exhausted window moves to the +//! next candidate; +//! 3. activates it (the CLI store swap every chat on that harness runs +//! under from now on); +//! 4. retires the chat's warm harness runtime, and stamps every warm +//! runtime with the login's credential fingerprint so a runtime started +//! under another login is never reused (the dispatch guard in +//! `sessions.rs`); +//! 5. resumes the chat's SAME harness session with the continue prompt — +//! through the durable command plane, so queued messages stay behind +//! the continuation exactly as a plain Resume. +//! +//! If the harness refuses the native resume, the continuation request +//! carries the budgeted history (ticket 01) as its fallback: the driver +//! pastes it in front of the prompt on the fresh session it starts. If the +//! resumed turn fails with a signed-out error, the run's own marking +//! (ticket 08) has already marked that account; the engine moves to the +//! next candidate. Any other failure restores the previously active +//! account — a failed failover leaves the harness as it found it. +//! +//! The RPC entry points (`ListResumeAccountCandidates`, +//! `ResumeOnAnotherAccount`) live in `rpc.rs`; this module is the +//! engine-side state machine, driven through [`SessionsEngine`]. + +use std::collections::HashSet; +use std::time::Duration; + +use roboco_doc::{MessagePart, MessageRole, MessageStatus, SessionCommandPayload, SessionMessageEntry}; +use roboco_proto::{ + AgentAccount, AgentSignInState, HarnessId, ResumeAccountCandidates, ResumeOnAccountOutcome, + RESUME_CONTINUE_PROMPT, +}; + +use crate::agent_accounts::AgentAccounts; +use crate::sessions::SessionsEngine; +use crate::{new_id, now_ms, EngineError}; + +/// Only these harnesses fail over: each keeps one switchable account login +/// whose sessions survive a switch (research 03). Pure. +pub(crate) fn failover_supported(harness: HarnessId) -> bool { + matches!(harness, HarnessId::ClaudeCode | HarnessId::Codex) +} + +fn harness_label(harness: HarnessId) -> &'static str { + match harness { + HarnessId::ClaudeCode => "Claude Code", + HarnessId::Codex => "Codex", + _ => "This harness", + } +} + +/// How long the watcher waits for the continuation turn to START. The +/// command dispatches within milliseconds on an idle chat; a wide bound +/// only covers an agent update holding the queue. +const TURN_START_WINDOW: Duration = Duration::from_secs(30); + +/// Cap on how many candidates one failover chain tries: the ranked list is +/// short, but a store that keeps returning the same row must never spin. +const MAX_ATTEMPTS: usize = 8; + +/// How long the watcher waits for the continuation turn to SETTLE after it +/// started: the wait is driven by the status watch (no polling), so a long +/// turn costs nothing — the bound only stops a watcher leaking forever on +/// a run that never settles (a chat parked on a question for days counts as +/// never here; the switch itself stands, only the chain ends). +const SETTLE_WINDOW: Duration = Duration::from_secs(24 * 60 * 60); + +/// How long the watcher re-checks the signed-out mark for after a settle +/// (bounded): the run's own marking is spawned, not awaited (it queues +/// behind the accounts ops lock a usage probe can hold), so the mark can +/// land a beat after the turn errored. +const MARK_GRACE: Duration = Duration::from_secs(2); + +/// The ranked switch targets for `chat_id` (ticket 09): the cached rows of +/// the chat's harness, best first, plus the account the chat's limit hit. +/// Empty for a harness without failover, or when every other account is +/// limited, signed out, an API key, or unswitchable — then only Resume at +/// reset is offered. [`SessionsEngine::resume_account_candidates`]. +pub(crate) async fn candidates( + engine: &SessionsEngine, + chat_id: &str, +) -> Result { + let session = engine.session_status(chat_id); + // The harness whose accounts to switch: the limit record's when the + // chat is Limited (the common path), else the chat's last run's. + let harness = session + .as_ref() + .and_then(|row| row.limit.as_ref().map(|limit| limit.harness)) + .or_else(|| engine.last_request(chat_id).and_then(|last| last.harness)); + let Some(harness) = harness else { + return Ok(ResumeAccountCandidates { + chat_id: chat_id.to_string(), + harness: HarnessId::ClaudeCode, + limited_account_id: None, + candidates: Vec::new(), + signed_out: Vec::new(), + }); + }; + let (limited_account_id, candidates, signed_out) = match engine.agent_accounts() { + Some(accounts) => { + let limited = session + .as_ref() + .and_then(|row| row.limit.as_ref()) + .and_then(|limit| limit.account_id.clone()); + if failover_supported(harness) { + let candidates = accounts.failover_candidates(harness, limited.as_deref()).await; + ( + limited, + candidates, + // The dead logins the picker offers "Sign in again" + // for (stories 47/50) — listed, never switched to. + accounts.failover_signed_out(harness).await, + ) + } else { + (limited, Vec::new(), Vec::new()) + } + } + None => (None, Vec::new(), Vec::new()), + }; + Ok(ResumeAccountCandidates { + chat_id: chat_id.to_string(), + harness, + limited_account_id, + candidates, + signed_out, + }) +} + +/// **Resume on another account** (ticket 09): rank, probe, activate, +/// retire the warm runtime, dispatch the continuation. Returns the +/// synchronous outcome; the turn's own result arrives through the chat's +/// status, and a signed-out failure is handled by the engine (next +/// candidate, else the previous account restored) before the chat +/// settles. [`SessionsEngine::resume_on_another_account`]. +pub(crate) async fn resume_on_another_account( + engine: &SessionsEngine, + chat_id: &str, + chosen: Option<&str>, +) -> Result { + let Some(accounts) = engine.agent_accounts() else { + return Ok(ResumeOnAccountOutcome::Failed { + reason: "This engine has no harness accounts.".into(), + }); + }; + let Some(limit) = engine.session_status(chat_id).and_then(|row| row.limit) else { + return Ok(ResumeOnAccountOutcome::Failed { + reason: "Only a Limited chat can resume on another account.".into(), + }); + }; + let harness = limit.harness; + if !failover_supported(harness) { + return Ok(ResumeOnAccountOutcome::Failed { + reason: format!("{} has no switchable accounts.", harness_label(harness)), + }); + } + // The account to restore if the failover fails: the login the harness + // runs under right now (the account the limit hit, unless the user + // switched since the chat went Limited). + let previous = accounts.active_account(harness).await.map(|row| row.id); + let attempt = attempt( + engine, + &accounts, + chat_id, + harness, + limit.account_id.as_deref(), + previous.clone(), + HashSet::new(), + chosen, + ) + .await; + match attempt { + Attempt::Switched(row) => Ok(ResumeOnAccountOutcome::Switched { + account_id: row.id.clone(), + account_email: row.email.clone().or_else(|| row.display_name.clone()), + }), + Attempt::NoCandidates => Ok(ResumeOnAccountOutcome::NoCandidates), + Attempt::Failed(reason) => Ok(ResumeOnAccountOutcome::Failed { reason }), + } +} + +/// One attempt's result. +enum Attempt { + /// The account was probed, activated and the continuation dispatched; + /// the watcher owns the rest. + Switched(AgentAccount), + /// No healthy candidate remains. + NoCandidates, + /// The attempt failed; nothing was switched. + Failed(String), +} + +/// Probe, activate, retire and dispatch one candidate (skipping it when +/// the probe reveals it is dead or limited — then the next). `pick` +/// names the user's chosen account — used for the FIRST candidate only, +/// later iterations fall back to the best-ranked. `tried` guards against +/// a store that keeps offering the same row. +#[allow(clippy::too_many_arguments)] +// Boxed with an explicit `Send` bound: `attempt` and `watch` recurse +// (a signed-out continuation tries the next candidate), and a recursive +// async fn's future cannot be proven `Send` for `tokio::spawn` otherwise. +fn attempt<'a>( + engine: &'a SessionsEngine, + accounts: &'a AgentAccounts, + chat_id: &'a str, + harness: HarnessId, + limited_account_id: Option<&'a str>, + previous: Option, + mut tried: HashSet, + mut pick: Option<&'a str>, +) -> futures::future::BoxFuture<'a, Attempt> { + Box::pin(async move { + for _ in 0..MAX_ATTEMPTS { + if engine.shutting_down() { + return Attempt::Failed("the engine is shutting down".into()); + } + let mut ranked = accounts + .failover_candidates(harness, limited_account_id) + .await; + // Rows this chain already tried are out — a probe that failed + // transiently (the row never changed) must move ON, not re-pick the + // same account and die on the second attempt. Skipped rows (signed + // out, still limited) drop out of the ranking on their own: their + // probes marked them. + ranked.retain(|row| !tried.contains(&row.id)); + // The user's pick must be a candidate — anything else (the limited + // account, a signed-out row, a foreign id) is refused, not silently + // replaced by the recommendation. + let candidate = match pick.take() { + Some(id) => { + let Some(at) = ranked.iter().position(|row| row.id == id) else { + return Attempt::Failed( + "That account is not a candidate — it is limited, signed out, or in use." + .into(), + ); + }; + Some(ranked.remove(at)) + } + None => ranked.first().cloned(), + }; + let Some(candidate) = candidate else { + return Attempt::NoCandidates; + }; + tried.insert(candidate.id.clone()); + // The chosen account is probed ONCE before switching: fresh meters, + // a hard rejection caught before anything is touched. + let row = match accounts.probe_account(harness, &candidate.id).await { + Ok(row) => row, + Err(err) => { + tracing::warn!( + chat = %chat_id, + account = %candidate.id, + error = %err, + "failover probe failed; trying the next candidate" + ); + continue; + } + }; + if row.sign_in_state == AgentSignInState::SignedOut { + tracing::info!( + chat = %chat_id, + account = %candidate.id, + "failover candidate is signed out; trying the next" + ); + continue; + } + if row.limited_until.is_some_and(|until| until > chrono::Utc::now()) { + tracing::info!( + chat = %chat_id, + account = %candidate.id, + "failover candidate is still limited; trying the next" + ); + continue; + } + // Activate — the swap every chat on this harness now runs under. + if let Err(err) = accounts.activate(harness, &row.id).await { + tracing::warn!( + chat = %chat_id, + account = %row.id, + error = %err, + "failover activation failed; trying the next candidate" + ); + continue; + } + // Retire the chat's warm runtime so the continuation starts under + // the new login (the fingerprint guard covers every other chat's + // warm runtime at its next dispatch). + let _ = engine.terminate(chat_id).await; + if let Err(reason) = dispatch_continuation(engine, chat_id, harness) { + // The switch landed but the continuation could not be queued: + // undo it, or the harness is left on an account nothing uses. + restore(engine, accounts, chat_id, harness, previous.as_deref()).await; + return Attempt::Failed(reason); + } + // The watcher owns the turn's outcome: signed-out → the next + // candidate; other failures → the previous account back. + let watcher_engine = engine.clone(); + let watcher_accounts = accounts.clone(); + let watcher_chat = chat_id.to_string(); + let watcher_limited = limited_account_id.map(str::to_owned); + let watcher_previous = previous.clone(); + tokio::spawn(async move { + watch( + &watcher_engine, + &watcher_accounts, + &watcher_chat, + harness, + watcher_limited.as_deref(), + watcher_previous, + tried, + ) + .await; + }); + return Attempt::Switched(row); + } + Attempt::Failed("too many failover attempts".into()) + }) +} + +/// Queue the continuation: the same harness session, resumed with the +/// continue prompt, through the durable command plane (the same seam the +/// composer's Resume uses — queued messages wait behind it, and the turn +/// survives an engine restart between the switch and the dispatch). The +/// budgeted transcript (ticket 01) rides as the request's fallback +/// history, delivered by the driver only when the harness refuses the +/// native resume and starts a fresh session. +fn dispatch_continuation( + engine: &SessionsEngine, + chat_id: &str, + harness: HarnessId, +) -> Result<(), String> { + let last = engine + .last_request(chat_id) + .ok_or_else(|| "the chat has no run configuration to continue".to_string())?; + let resume = engine.harness_session_to_resume(chat_id, &last.cwd, harness); + let fallback = fallback_history(engine, chat_id); + let request = roboco_proto::RunRequest { + prompt: RESUME_CONTINUE_PROMPT.into(), + harness: Some(harness), + model: last.model.clone(), + reasoning: last.reasoning.clone(), + model_options: last.model_options.clone(), + cwd: last.cwd.clone(), + sandbox: last.sandbox, + auto_approve: last.auto_approve, + // The continuation is text only: attachments and worktrees belong + // to the turn that asked for them; the checkout is already where + // the chat ran. Dispatch stamps its own MCP server. + attachments: Vec::new(), + worktree: None, + mcp: None, + // The stored session, resumed EXPLICITLY (bypassing dispatch's + // re-derivation) so the fallback history pairs with it; a failover + // never forks. + resume, + resume_fallback_history: fallback, + provider_fork: None, + }; + let host = engine + .host() + .ok_or_else(|| "the engine's doc host is not wired".to_string())?; + host.queue_command( + chat_id, + SessionCommandPayload::Run { + request, + message_id: new_id(), + }, + ) + .map(drop) + .map_err(|err| err.to_string()) +} + +/// The chat's own budgeted transcript — the fallback a fresh session gets +/// when the harness refuses the native resume. The chat itself is the +/// readable source (`read_chat`); the selection rules and budget live in +/// [`crate::transcript_budget`]. +fn fallback_history(engine: &SessionsEngine, chat_id: &str) -> Option { + let handle = engine.host()?.open(chat_id).ok()?; + let entries = handle.doc().read_entries().unwrap_or_default(); + crate::transcript_budget::budgeted_transcript_prefix(&entries, Some(chat_id)) +} + +/// Watch the continuation turn (fire-and-forget): once it settles, a +/// signed-out failure moves to the next candidate (the run's error has +/// already marked the account — ticket 08); any other failure restores the +/// previously active account and REPORTS it (ticket 09) — the chat's +/// transcript gains a System entry saying the switch failed and the +/// previous account is active again, so the user (and the web client, which +/// renders System entries) sees it. A turn the user stopped, or a chat that +/// vanished, ends the chain and leaves the switch standing — the user's +/// stop is not a failed failover. +/// +/// Driven by the session status watch, not polling (the old 100ms/250ms +/// loops could miss a fast Working→Errored flip entirely and spun forever +/// while a question waited): the watcher sleeps until a row it cares about +/// changes, wakes on the engine's own transitions, and bounds itself by +/// [`TURN_START_WINDOW`] (the start) and [`SETTLE_WINDOW`] (the settle). +async fn watch( + engine: &SessionsEngine, + accounts: &AgentAccounts, + chat_id: &str, + harness: HarnessId, + limited_account_id: Option<&str>, + previous: Option, + tried: HashSet, +) { + // The row the pre-switch chat settled on (Limited: Errored with a + // written-at). A status row is only "the continuation's outcome" when + // it is newer than this moment — the chat sat Errored before the turn + // ever started, so the old row must not read as the turn's failure. + let began = chrono::Utc::now(); + // Phase 1: the continuation starts. The queued command dispatches + // within milliseconds; the bound only covers a held queue. The watch + // wakes on every transition — including a turn that errors before it + // ever reads Working, which the old poller could sleep straight through. + let mut rows = engine.watch_sessions(); + let start_by = tokio::time::Instant::now() + TURN_START_WINDOW; + let started = loop { + if engine.shutting_down() { + return; + } + let row = engine.session_status(chat_id); + match row.map(|row| (row.status, row.updated_at)) { + Some((roboco_proto::SessionStatus::Working | roboco_proto::SessionStatus::AwaitingInput, _)) => { + break true; + } + // Errored again (or idle) on a row WRITTEN after we began: the + // turn ran and settled within the window — watch it as settled. + Some((roboco_proto::SessionStatus::Idle | roboco_proto::SessionStatus::Errored, at)) + if at > began => + { + break false; + } + // No row yet (or the chat's pre-switch row: Errored with a + // written-at before we began) — the command is still on its way. + _ => {} + } + tokio::select! { + changed = rows.changed() => { + if changed.is_err() { + return; + } + } + _ = tokio::time::sleep_until(start_by) => { + tracing::info!( + chat = %chat_id, + "failover continuation never started; leaving the queued command to drain" + ); + return; + } + } + }; + // Phase 2: settle. AwaitingInput is still the user's turn in progress — + // the watcher stays (for free) until the turn ends one way or another. + let settled = if started { + let settle_by = tokio::time::Instant::now() + SETTLE_WINDOW; + let mut settled = None; + loop { + if engine.shutting_down() { + return; + } + let status = engine.session_status(chat_id).map(|row| row.status); + match status { + Some(roboco_proto::SessionStatus::Idle) + | Some(roboco_proto::SessionStatus::Errored) + | None => { + settled = status; + break; + } + // Working | AwaitingInput: keep waiting on the watch. + _ => {} + } + tokio::select! { + changed = rows.changed() => { + if changed.is_err() { + return; + } + } + _ = tokio::time::sleep_until(settle_by) => { + tracing::info!( + chat = %chat_id, + "failover continuation never settled; leaving the switch standing" + ); + return; + } + } + } + settled + } else { + // The turn already settled during phase 1: read its final state. + engine.session_status(chat_id).map(|row| row.status) + }; + match settled { + Some(roboco_proto::SessionStatus::Errored) => { + // Why did it fail? A signed-out error has already marked the + // switched-to account (the run's own wiring, ticket 08): the + // ACTIVE row reading SignedOut is that mark. The mark is + // spawned (it queues behind the accounts ops lock), so it can + // land a beat after the settle — re-check it for a bounded + // grace before concluding the failure was something else. + let mut signed_out = false; + let deadline = tokio::time::Instant::now() + MARK_GRACE; + loop { + let active = accounts.active_account(harness).await; + signed_out = active + .as_ref() + .is_some_and(|row| row.sign_in_state == AgentSignInState::SignedOut); + if signed_out || tokio::time::Instant::now() >= deadline { + break; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + if signed_out { + tracing::info!( + chat = %chat_id, + "failover resume failed signed out; trying the next candidate" + ); + let next = attempt( + engine, + accounts, + chat_id, + harness, + limited_account_id, + previous.clone(), + tried, + None, + ) + .await; + // The chain ran out of candidates (or the store broke): the + // failover failed as a whole — put the previous account + // back so the harness is not left on a login whose turn + // just failed and nothing uses, and say so on the chat. + if matches!(next, Attempt::NoCandidates | Attempt::Failed(_)) { + let reason = match next { + Attempt::NoCandidates => { + "every other account is limited, signed out, or in use".to_string() + } + Attempt::Failed(reason) => reason, + Attempt::Switched(_) => unreachable!("matched above"), + }; + restore(engine, accounts, chat_id, harness, previous.as_deref()).await; + note_failed_failover(engine, chat_id, previous.as_deref(), &reason); + } + } else { + restore(engine, accounts, chat_id, harness, previous.as_deref()).await; + note_failed_failover( + engine, + chat_id, + previous.as_deref(), + "the resumed turn failed", + ); + } + } + // Completed, interrupted (the user's stop — not a failed + // failover), or the chat went away: the chain ends here. + _ => {} + } +} + +/// Write the failed-failover report into the chat's transcript (ticket 09: +/// "restore the previously active account and report the failure"): a +/// System entry, rendered by every client, saying what failed and which +/// account the harness is back on. Best-effort — a failed write costs the +/// note, never the switch. +fn note_failed_failover(engine: &SessionsEngine, chat_id: &str, previous: Option<&str>, reason: &str) { + let Some(host) = engine.host() else { + return; + }; + let Some(handle) = host.open(chat_id).ok() else { + return; + }; + let account = previous + .map(|id| format!("account {id}")) + .unwrap_or_else(|| "its previous login".into()); + let text = format!( + "Resume on another account failed: {reason}. The harness is back on {account}." + ); + let entry = SessionMessageEntry { + id: format!("failover:{}", new_id()), + role: MessageRole::System, + parts: vec![MessagePart::Text { + id: "t0".into(), + text, + }], + created_at: now_ms(), + device_id: host.device_id().to_owned(), + status: Some(MessageStatus::Complete), + continuation_of: None, + duration_ms: None, + origin_chat_id: None, + provider_turn: None, + }; + if let Err(err) = handle.doc().push_message(&entry) { + tracing::warn!(chat = %chat_id, error = %err, "failed-failover note write failed"); + } +} + +/// Put the previously active account back (best-effort): a failover that +/// failed left the harness on an account whose turn errored — the world +/// returns to the login the chats were running under. +async fn restore( + engine: &SessionsEngine, + accounts: &AgentAccounts, + chat_id: &str, + harness: HarnessId, + previous: Option<&str>, +) { + let Some(previous) = previous else { + return; + }; + if let Err(err) = accounts.activate(harness, previous).await { + tracing::warn!( + chat = %chat_id, + account = %previous, + error = %err, + "could not restore the previous harness account after a failed failover" + ); + } +} diff --git a/crates/engine/src/agent_accounts.rs b/crates/engine/src/agent_accounts.rs index 512bf8c0b0..430b5b8fb2 100644 --- a/crates/engine/src/agent_accounts.rs +++ b/crates/engine/src/agent_accounts.rs @@ -77,6 +77,19 @@ //! (shown instead of "Usage unavailable"), and backs off — honouring //! `Retry-After` — before that account is probed again. Only 401/403 counts //! as a rejected token (and only then is a saved Claude slot refreshed). +//! +//! Sign-in state (ticket 08): every account row reports whether its login is +//! known to work — `signed-in` after the last successful probe, `signed-out` +//! after a HARD rejection, `unknown` otherwise. A rejection is hard only when +//! it proves the login is dead: the slot's own refresh was refused (Claude, +//! Grok: 400 invalid_grant) or the just-refreshed token was rejected again, or +//! a provider whose keys never lapse refused one (Devin). An expired access +//! token the CLI refreshes itself stays `unknown` — ambiguous, never +//! signed-out. The engine also marks an account signed out when a run fails +//! with ticket 05's structured `signed-out` error +//! ([`AgentAccounts::mark_account_signed_out`]); a later successful probe or +//! sign-in clears the mark. Each row also carries the latest reset among its +//! exhausted windows (`limited_until`), the moment a limit is known to lift. use std::collections::HashMap; use std::path::{Path, PathBuf}; @@ -92,7 +105,8 @@ use sha2::{Digest, Sha256}; use roboco_proto::{ AgentAccount, AgentAccountWarning, AgentAccountsSnapshot, AgentAuthKind, AgentLoginMode, - AgentLoginPoll, AgentLoginStart, AgentLoginStatus, AgentUsageWindow, HarnessId, + AgentLoginPoll, AgentLoginStart, AgentLoginStatus, AgentSignInState, AgentUsageWindow, + HarnessId, }; use crate::repos::home_dir; @@ -495,6 +509,16 @@ enum ProbeError { /// 401/403 — the token was rejected; the ONLY status worth a refresh. #[serde(rename_all = "camelCase")] Unauthorized { status: u16 }, + /// A rejection that PROVES the login is dead, not merely stale: the + /// slot's own refresh was refused (Claude, Grok: 400 invalid_grant) or + /// the just-refreshed token was rejected too, or a provider whose keys + /// never lapse on their own refused one (Devin: a rejection means the + /// key was revoked). Distinct from [`ProbeError::Unauthorized`], which an + /// expired access token also produces and only the CLI's own refresh can + /// settle — this is the hard signal that marks the account signed out + /// until a sign-in or a successful probe clears it. + #[serde(rename_all = "camelCase")] + SignedOut { status: u16 }, /// 429 — `retry_after_secs` from the `Retry-After` header when sent. #[serde(rename_all = "camelCase")] RateLimited { retry_after_secs: Option }, @@ -535,6 +559,7 @@ impl ProbeError { fn class(&self) -> &'static str { match self { ProbeError::Unauthorized { .. } => "unauthorized", + ProbeError::SignedOut { .. } => "signed-out", ProbeError::RateLimited { .. } => "rate-limited", ProbeError::Http { status, .. } if *status >= 500 => "server-error", ProbeError::Http { .. } => "http-error", @@ -548,7 +573,9 @@ impl ProbeError { fn status(&self) -> Option { match self { - ProbeError::Unauthorized { status } | ProbeError::Http { status, .. } => Some(*status), + ProbeError::Unauthorized { status } + | ProbeError::SignedOut { status } + | ProbeError::Http { status, .. } => Some(*status), ProbeError::RateLimited { .. } => Some(429), _ => None, } @@ -575,6 +602,7 @@ impl ProbeError { // Lifted early when the slot's credentials change (re-login, the // CLI refreshing its token) — see `UsageEntry::probe_due`. ProbeError::Unauthorized { .. } + | ProbeError::SignedOut { .. } | ProbeError::NoCredentials { .. } | ProbeError::UntrustedEndpoint => Duration::from_secs(10 * 60), } @@ -596,6 +624,13 @@ struct UsageEntry { /// The last probe's failure; cleared by the next success. #[serde(default, skip_serializing_if = "Option::is_none")] error: Option, + /// Hard sign-in evidence, surviving ambiguous failures: `SignedIn` after + /// the last successful probe, `SignedOut` after a hard rejection (see + /// [`ProbeError::SignedOut`]) or a run's signed-out error (see + /// [`AgentAccounts::mark_account_signed_out`]). `None` = unknown — never + /// probed, or only ambiguous outcomes. + #[serde(default, skip_serializing_if = "Option::is_none")] + sign_in: Option, /// Last probe attempt, success or failure. #[serde(default)] checked_at: i64, @@ -610,7 +645,13 @@ struct UsageEntry { impl UsageEntry { /// Whether a forced list should probe this account now. fn probe_due(&self, credentials: &str, now: i64) -> bool { - if now - self.checked_at < FORCED_MIN_INTERVAL.as_millis() as i64 { + // The last failure was about OTHER credentials (a sign-in landed, the + // CLI rotated its token): the entry knows nothing about the current + // ones, so even the freshness gate lifts — the reload right after a + // sign-in must re-establish the account's state, not replay the old + // rejection. Same principle the backoff arm below encodes. + let credentials_changed = self.credentials.as_deref().is_some_and(|c| c != credentials); + if !credentials_changed && now - self.checked_at < FORCED_MIN_INTERVAL.as_millis() as i64 { return false; } match (self.retry_at, &self.error) { @@ -621,9 +662,10 @@ impl UsageEntry { matches!( error, ProbeError::Unauthorized { .. } + | ProbeError::SignedOut { .. } | ProbeError::NoCredentials { .. } | ProbeError::UntrustedEndpoint - ) && self.credentials.as_deref() != Some(credentials) + ) && credentials_changed } _ => true, } @@ -638,14 +680,41 @@ impl UsageEntry { self.error = None; self.retry_at = None; self.credentials = None; + // A probe that reached the provider and came back green is + // proof the login works — it clears any earlier signed-out + // mark (the display fix's "later successful probe"). + self.sign_in = Some(AgentSignInState::SignedIn); } Err(error) => { self.retry_at = Some(now + error.backoff().as_millis() as i64); + // Only a HARD rejection flips the state to signed out; every + // other failure (expired-but-CLI-refreshes, rate limit, + // network) says nothing about the sign-in and leaves the last + // hard evidence standing. + if matches!(error, ProbeError::SignedOut { .. }) { + self.sign_in = Some(AgentSignInState::SignedOut); + } self.error = Some(error); self.credentials = Some(credentials); } } } + + /// The account's sign-in state as of this list. A signed-out mark was + /// recorded against the credentials the failure saw; when the slot holds + /// different ones now (a sign-in landed, the CLI refreshed the token), + /// the mark is stale and reads unknown until the next probe settles it. + fn sign_in_state(&self, credentials: &str) -> AgentSignInState { + match self.sign_in { + Some(AgentSignInState::SignedOut) + if self.credentials.as_deref().is_some_and(|c| c != credentials) => + { + AgentSignInState::Unknown + } + Some(state) => state, + None => AgentSignInState::Unknown, + } + } } #[derive(Debug, Default, Serialize, Deserialize)] @@ -705,6 +774,27 @@ impl Default for ProbeEndpoints { } } +/// Every endpoint on one mock server, with slot refresh and loopback http +/// admitted — the set [`AgentAccounts::with_mocked_endpoints`] and the +/// in-crate provider tests build their mocks on. +fn mocked_endpoints(base: &str) -> ProbeEndpoints { + ProbeEndpoints { + claude_usage: format!("{base}/api/oauth/usage"), + claude_token: format!("{base}/v1/oauth/token"), + claude_loopback_token: format!("{base}/v1/oauth/token"), + claude_profile: format!("{base}/api/oauth/profile"), + codex_usage: format!("{base}/backend-api/wham/usage"), + grok_usage: format!("{base}/v1/billing?format=credits"), + github_api: base.to_string(), + github_login: base.to_string(), + openai_auth: base.to_string(), + openai_port: 0, + nous_portal: base.to_string(), + allow_slot_refresh: true, + allow_loopback_http: true, + } +} + /// The live Claude credential read, cached briefly per identity (see /// [`AgentAccounts::read_claude_credentials_cached`]). struct CachedClaudeCredentials { @@ -860,6 +950,15 @@ impl AgentAccounts { lock(&self.inner.cli_overrides).insert(harness, path.into()); } + /// Test seam: [`Self::new`] with every provider endpoint pointed at + /// `base` (a local mock server) instead of the real providers — the same + /// endpoint set the in-crate provider tests use. Integration tests drive + /// usage probes and token flows against it; never use in production. + #[doc(hidden)] + pub fn with_mocked_endpoints(config: AgentAccountsConfig, base: &str) -> Self { + Self::with_endpoints(config, mocked_endpoints(base)) + } + /// The ACP harness for `harness`'s CLI, honouring [`Self::override_cli`]. fn acp_harness(&self, harness: HarnessId) -> Option { let acp = match harness { @@ -889,6 +988,214 @@ impl AgentAccounts { self.list_locked(force_usage).await } + /// The ACTIVE account row for `harness` — the login the CLI is running + /// under right now, with its cached usage windows (the last good probe; + /// no network). The sessions engine reads this to complete a usage-limit + /// record: which account the limit hit, and the reset time when the + /// harness's own error did not carry one. + pub async fn active_account(&self, harness: HarnessId) -> Option { + self.list(false) + .await + .ok()? + .accounts + .into_iter() + .find(|account| account.harness == harness && account.active) + } + + /// The harness's OTHER accounts that are signed out (ticket 09's + /// picker): never switch targets — the ranking excludes them — but the + /// picker lists them with a "Sign in again" action (the existing login + /// flow ticket 08 drives) so a dead login can be revived in place + /// (spec stories 47 and 50). Cached rows, no network; the live login is + /// never listed (it is the account the picker says it switches FROM). + pub async fn failover_signed_out(&self, harness: HarnessId) -> Vec { + self.list(false) + .await + .map(|snapshot| { + snapshot + .accounts + .into_iter() + .filter(|row| { + row.harness == harness + && !row.active + && row.sign_in_state == AgentSignInState::SignedOut + }) + .collect() + }) + .unwrap_or_default() + } + + /// The ranked failover candidates for `harness` (orchestration v2, + /// ticket 09) — the CACHED rows, no network: the forced probe is the + /// engine's call on the CHOSEN candidate only ([`Self::probe_account`]). + /// `limited_account_id` names the account the chat's limit hit (the + /// limit record's account, else the harness's active login when the + /// record carried none). See [`rank_failover_candidates`] for the + /// exclusion and ranking rules. + pub async fn failover_candidates( + &self, + harness: HarnessId, + limited_account_id: Option<&str>, + ) -> Vec { + let Ok(snapshot) = self.list(false).await else { + return Vec::new(); + }; + // The org/workspace group of every row the ranker demotes by. The + // Codex row carries no account key on the wire, so the groups come + // from the slots (id → key); Claude's group IS the row's + // organization field. + let slot_groups: HashMap> = self + .read_slots(harness) + .into_iter() + .map(|slot| { + let group = failover_group( + harness, + slot.profile.organization.as_deref(), + &slot.account_key, + ); + (slot.id, group) + }) + .collect(); + let rows: Vec = snapshot + .accounts + .into_iter() + .filter(|row| row.harness == harness) + .map(|row| { + let group = match harness { + HarnessId::ClaudeCode => { + failover_group(harness, row.organization.as_deref(), "") + } + _ => slot_groups.get(&row.id).cloned().flatten(), + }; + FailoverCandidate { row, group } + }) + .collect(); + // The account the limit hit: by id when the record names it, else + // the live login. Its group demotes its same-pool peers. + let limited = rows.iter().find(|candidate| { + limited_account_id.is_some_and(|id| id == candidate.row.id) + || (limited_account_id.is_none() && candidate.row.active) + }); + let (limited_id, limited_group) = match limited { + Some(candidate) => (Some(candidate.row.id.clone()), candidate.group.clone()), + None => (None, None), + }; + rank_failover_candidates(rows, limited_id.as_deref(), limited_group.as_deref()) + } + + /// One candidate's usage probe (ticket 09): refresh ONLY this account's + /// cached meters — honoring the cache's own backoff and freshness gates, + /// so a rate-limited usage endpoint is never re-probed to death — and + /// return the fresh row. The engine probes the chosen candidate once + /// before switching: a hard rejection (`SignedOut`) or a newly-exhausted + /// window tells it to move to the next candidate instead. Never touches + /// the other accounts' cache entries (the sweep that forgets removed + /// slots belongs to [`Self::refresh_usage`], which sees every slot). + pub async fn probe_account( + &self, + harness: HarnessId, + account_id: &str, + ) -> Result { + let _ops = self.inner.ops.lock().await; + let slot = self + .read_slots(harness) + .into_iter() + .find(|s| s.id == account_id) + .ok_or_else(|| { + EngineError::Other( + "That saved login no longer exists — refresh and try again.".into(), + ) + })?; + let active = self + .live_account_key(harness, slot.store_key.as_deref()) + .await + .as_deref() + == Some(slot.account_key.as_str()); + let key = usage_key(harness, &slot.account_key); + let credentials = credentials_fingerprint(&slot.credentials); + let now = now_ms(); + let due = lock(&self.inner.usage) + .get(&key) + .is_none_or(|entry| entry.probe_due(&credentials, now)); + if due && lock(&self.inner.inflight_probes).insert(key.clone()) { + let result = self.probe_usage(harness, &slot, active).await; + lock(&self.inner.inflight_probes).remove(&key); + let mut usage = lock(&self.inner.usage); + usage + .entry(key) + .or_default() + .record(result, credentials, now); + self.persist_usage(&usage); + } + self.list_locked(false) + .await? + .accounts + .into_iter() + .find(|row| row.harness == harness && row.id == account_id) + .ok_or_else(|| { + EngineError::Other("The account vanished while probing it.".into()) + }) + } + + /// Fingerprint of the CLI store's live login for `harness` — a digest of + /// the login's IDENTITY, never its token bytes (ticket 09): a CLI token + /// refresh rewrites the credential file on every expiry, and a + /// bytes-keyed fingerprint would retire every warm runtime on the SAME + /// login — a mid-turn steer became an interrupt-and-redispatch. Claude's + /// identity (account uuid, email, organization) lives in the config + /// file's `oauthAccount`; Codex's rides the auth file's `id_token` + /// claims (or its API key). The sessions engine stamps the digest on + /// every warm runtime it starts, so a runtime launched under another + /// login is never reused after a switch. `None` for a harness whose live + /// store roboco does not read — and for a Claude Keychain login, which + /// has no file: no fingerprint, no forced retirement. When the identity + /// cannot be read, the raw credential bytes stand in (the old shape): + /// identity unknown is the one case where the tokens themselves are the + /// best available proxy for "same login". + pub fn live_credentials_fingerprint(&self, harness: HarnessId) -> Option { + let identity_fingerprint = |parts: &[Option]| { + let joined: Vec<&str> = parts.iter().flatten().map(String::as_str).collect(); + (!joined.is_empty()).then(|| { + crate::repos::hex(&Sha256::digest(joined.join("\u{0}").as_bytes()))[..16] + .to_string() + }) + }; + match harness { + HarnessId::ClaudeCode => { + // The credentials file's tokens churn on every refresh; the + // config's `oauthAccount` is the durable identity of the + // login every run on this harness uses. + let cfg = read_json(&self.inner.config.claude_config_file); + let identity = cfg.as_ref().and_then(|c| c.get("oauthAccount")).map(|oauth| { + [ + str_field(oauth, "accountUuid"), + str_field(oauth, "emailAddress"), + str_field(oauth, "organizationUuid"), + ] + }); + if let Some(digest) = identity.and_then(|parts| identity_fingerprint(&parts)) { + return Some(digest); + } + let bytes = std::fs::read(self.inner.config.claude_creds_file()).ok()?; + Some(crate::repos::hex(&Sha256::digest(&bytes))[..16].to_string()) + } + HarnessId::Codex => { + let auth = read_json(&self.inner.config.codex_auth_file())?; + // The id_token rotates; its CLAIMS (the user and workspace + // ids) are the login's identity. An API-key login is its + // own stable identity. + if let Some(parts) = codex_login_identity(&auth) + && let Some(digest) = identity_fingerprint(&parts) + { + return Some(digest); + } + let bytes = serde_json::to_string(&auth).ok()?; + Some(crate::repos::hex(&Sha256::digest(bytes.as_bytes()))[..16].to_string()) + } + _ => None, + } + } + /// Caller holds [`Inner::ops`]. async fn list_locked(&self, force_usage: bool) -> Result { let mut warnings: Vec = Vec::new(); @@ -1013,6 +1320,7 @@ impl AgentAccounts { let active = active_set.is_some_and(|keys| keys.contains(&slot.account_key)); let entry = usage.get(&usage_key(harness, &slot.account_key)); let snapshot = entry.and_then(|entry| entry.usage.as_ref()); + let credentials = credentials_fingerprint(&slot.credentials); accounts.push(AgentAccount { id: slot.id.clone(), harness, @@ -1038,6 +1346,10 @@ impl AgentAccounts { now, ) }), + sign_in_state: entry + .map(|entry| entry.sign_in_state(&credentials)) + .unwrap_or(AgentSignInState::Unknown), + limited_until: snapshot.and_then(|usage| limited_until(&usage.windows)), display_name: slot.profile.display_name.clone(), organization: slot.profile.organization.clone(), auth_kind: Some(slot.profile.auth_kind), @@ -1062,6 +1374,9 @@ impl AgentAccounts { usage_windows: Vec::new(), usage_fetched_at: None, usage_error: None, + // The secret couldn't be read, so nothing was ever probed. + sign_in_state: AgentSignInState::Unknown, + limited_until: None, display_name: u.profile.display_name.clone(), organization: u.profile.organization.clone(), auth_kind: Some(u.profile.auth_kind), @@ -1408,6 +1723,84 @@ impl AgentAccounts { self.list_locked(false).await } + // ── sign-in state ────────────────────────────────────────────────────── + + /// Mark one saved account signed out from OUTSIDE the usage probe — a + /// run that failed with a signed-out error (ticket 05's structured + /// `AgentErrorKind::SignedOut`, on the harness's active account), or an + /// account-failover resume whose sign-in failed (ticket 09). The mark is + /// durable (the usage cache is rewritten, so it survives restarts) and + /// keyed to the slot's current credentials: a sign-in that lands on the + /// account, or a later successful probe, clears it. + /// + /// No network: this only records evidence. The wiring that watches runs + /// for signed-out errors resolves the active account and calls this. + pub fn mark_account_signed_out( + &self, + harness: HarnessId, + account_id: &str, + ) -> Result<(), EngineError> { + let slot = self + .read_slots(harness) + .into_iter() + .find(|s| s.id == account_id) + .ok_or_else(|| EngineError::Other("Unknown account.".into()))?; + let key = usage_key(harness, &slot.account_key); + let credentials = credentials_fingerprint(&slot.credentials); + { + let mut usage = lock(&self.inner.usage); + let entry = usage.entry(key).or_default(); + entry.sign_in = Some(AgentSignInState::SignedOut); + // Anchor the mark to these credentials so a re-login (or the + // CLI rotating its token) reads back as unknown — see + // [`UsageEntry::sign_in_state`]. + entry.credentials = Some(credentials); + let file = UsageCacheFile { + entries: usage.clone(), + }; + let persisted = serde_json::to_vec_pretty(&file) + .map_err(|e| EngineError::Other(e.to_string())) + .and_then(|json| { + self.inner.config.private_root()?; + write_file_atomic(&self.inner.config.usage_cache_file(), &json, true) + }); + if let Err(err) = persisted { + tracing::warn!(error = %err, "agent usage cache write failed"); + } + } + tracing::info!( + provider = harness_slug(harness), + slot = %account_id, + "harness account marked signed out" + ); + Ok(()) + } + + /// [`Self::mark_account_signed_out`] for a RUN's signed-out error + /// (ticket 05's structured kind, wired by the sessions engine): resolve + /// the CLI's LIVE login for `harness` and mark ITS account — the account + /// every new run on this harness would use. The list first + /// auto-snapshots the live login into its slot, so the row exists even + /// when no accounts view ever ran. Single-login agents resolve + /// unambiguously; a per-provider agent's login names its provider, which + /// the caller must resolve before this can mark anything (no driver + /// classifies those signed-out yet). + pub async fn mark_live_signed_out(&self, harness: HarnessId) -> Result<(), EngineError> { + let snapshot = self.list(false).await?; + let row = snapshot + .accounts + .iter() + .find(|account| account.harness == harness && account.active) + .ok_or_else(|| { + EngineError::Other(format!( + "no live {} login to mark signed out", + harness_slug(harness) + )) + })?; + let account_id = row.id.clone(); + self.mark_account_signed_out(harness, &account_id) + } + // ── add-account OAuth flows ───────────────────────────────────────────── pub async fn start_login(&self, harness: HarnessId) -> Result { @@ -2645,8 +3038,13 @@ impl AgentAccounts { .map(|(harness, slot, _)| usage_key(*harness, &slot.account_key)) .collect(); usage.retain(|key, _| live.contains(key)); - // Persist under the lock: overlapping lists must not interleave - // writes of the same file. + self.persist_usage(&usage); + } + + /// Write the usage cache to disk under its lock: overlapping probes and + /// lists must not interleave writes of the same file. Best-effort — a + /// failed write keeps the in-memory state (the next change retries). + fn persist_usage(&self, usage: &HashMap) { let file = UsageCacheFile { entries: usage.clone(), }; @@ -2722,11 +3120,26 @@ impl AgentAccounts { Err(ProbeError::Unauthorized { status }) if !is_active && self.inner.endpoints.allow_slot_refresh => { - match self.refresh_claude_slot(slot).await? { - Some(fresh) => self.claude_usage_request(&fresh).await, + match self.refresh_claude_slot(slot).await { + Ok(Some(fresh)) => match self.claude_usage_request(&fresh).await { + // The token this slot JUST rotated to was rejected + // too: the login is dead, not stale. + Err(ProbeError::Unauthorized { status }) => { + Err(ProbeError::SignedOut { status }) + } + result => result, + }, // Refresh in flight elsewhere (single-flight) — report // the original rejection; the next probe sees the result. - None => Err(ProbeError::Unauthorized { status }), + Ok(None) => Err(ProbeError::Unauthorized { status }), + // A refused refresh (400 invalid_grant: revoked/used + // token) means this login is dead — the hard signed-out + // signal, not an ambiguous expired token. Every other + // refresh failure (network, 5xx) stays what it was. + Err(ProbeError::Unauthorized { status }) => { + Err(ProbeError::SignedOut { status }) + } + Err(other) => Err(other), } } result => result, @@ -3049,6 +3462,10 @@ impl AntigravityLogin { usage_windows: Vec::new(), usage_fetched_at: None, usage_error: None, + // Roboco never reads Antigravity's token: no probe ever ran, so + // its sign-in state stays unknown. + sign_in_state: AgentSignInState::Unknown, + limited_until: None, display_name: Some(label.to_string()), organization: None, auth_kind: Some(kind), @@ -3275,6 +3692,30 @@ fn with_claude_ai_oauth( } } +/// The Codex auth file's login identity (for the credential fingerprint — +/// [`AgentAccounts::live_credentials_fingerprint`]): the id_token's user +/// and workspace claims, or the API key. `None` when the file names no +/// identity — the caller then falls back to the raw bytes. +fn codex_login_identity(auth: &serde_json::Value) -> Option>> { + if let Some(id_token) = auth + .get("tokens") + .and_then(|t| t.get("id_token")) + .and_then(|v| v.as_str()) + { + let claims = jwt_claims(id_token).unwrap_or_else(|| serde_json::json!({})); + let oa = claims + .get("https://api.openai.com/auth") + .cloned() + .unwrap_or_default(); + return Some(vec![ + str_field(&claims, "email"), + str_field(&oa, "chatgpt_account_id"), + str_field(&oa, "chatgpt_user_id"), + ]); + } + str_field(auth, "OPENAI_API_KEY").map(|key| vec![Some(key)]) +} + /// Parse a codex `auth.json` (the live one or a fresh login's). fn parse_codex_auth(auth: serde_json::Value) -> Option { if let Some(id_token) = auth @@ -3404,6 +3845,110 @@ fn usage_key(harness: HarnessId, account_key: &str) -> String { format!("{}:{account_key}", harness_slug(harness)) } +/// One row in the failover ranking (ticket 09): the wire account plus the +/// engine-local org/workspace group the ranking demotes shared-limit pools +/// by (see [`failover_group`]). +struct FailoverCandidate { + row: AgentAccount, + group: Option, +} + +/// The org or workspace an account may share a limit pool with (ticket 09): +/// Claude's organization name, Codex's ChatGPT workspace (the account key's +/// `user::workspace` tail — team seats share the workspace id; personal +/// logins keep their own key and share nothing). `None` when the account +/// names no group. Lower-cased: it is an identity, not a display value. +fn failover_group( + harness: HarnessId, + organization: Option<&str>, + account_key: &str, +) -> Option { + match harness { + HarnessId::ClaudeCode => organization + .map(str::trim) + .filter(|org| !org.is_empty()) + .map(|org| org.to_ascii_lowercase()), + HarnessId::Codex => Some( + account_key + .split_once("::") + .map(|(_, workspace)| workspace) + .unwrap_or(account_key) + .to_ascii_lowercase(), + ), + _ => None, + } +} + +/// Rank the failover candidates for a Limited chat's "Resume on another +/// account" (ticket 09). Pure. +/// +/// **Excluded:** +/// - the account that hit the limit (`limited_id`, the row named by the +/// chat's limit record) — and every ACTIVE row (the login the next run +/// would use anyway; switching to it is a no-op); +/// - non-switchable rows (Hermes' own pool, credentials roboco could not +/// read, single-login agents); +/// - API-key logins (no plan windows — a switch has nothing to fail over +/// to); +/// - accounts marked signed out (ticket 08 — they offer "Sign in again" +/// instead); +/// - accounts still limited (`limited_until` in the future). +/// +/// **Ranked:** accounts OUTSIDE the limited account's org/workspace first +/// (a shared pool may hit the same wall again) — the org penalty is the +/// PRIMARY key, outranking usage; then lowest usage first — the account's +/// most-used window (`0.0` when the meters are unknown). The pre-rank order +/// (slot creation order) breaks the remaining ties so the ranking is +/// deterministic. +fn rank_failover_candidates( + candidates: Vec, + limited_id: Option<&str>, + limited_group: Option<&str>, +) -> Vec { + let now = Utc::now(); + let mut ranked: Vec<(bool, f32, usize, AgentAccount)> = Vec::new(); + for (index, candidate) in candidates.into_iter().enumerate() { + let row = candidate.row; + if row.active || limited_id.is_some_and(|id| id == row.id) { + continue; // the account the limit hit / the live login + } + if !row.switchable { + continue; // Hermes' pool, unreadable credentials, single logins + } + if row.auth_kind == Some(AgentAuthKind::ApiKey) { + continue; // API keys have no plan windows to fail over to + } + if row.sign_in_state == AgentSignInState::SignedOut { + continue; // dead login — "Sign in again", not a candidate + } + if row + .limited_until + .is_some_and(|until| until > now) + { + continue; // still limited + } + // Lowest usage first: the account's most-used window (an account + // without meters has no cached evidence of exhaustion). + let usage = row + .usage_windows + .iter() + .map(|window| window.used_fraction.clamp(0.0, 1.0)) + .fold(0.0_f32, f32::max); + let shares_pool = candidate + .group + .as_deref() + .zip(limited_group) + .is_some_and(|(mine, limited)| mine == limited); + ranked.push((shares_pool, usage, index, row)); + } + ranked.sort_by(|a, b| { + a.0.cmp(&b.0) + .then_with(|| a.1.total_cmp(&b.1)) + .then_with(|| a.2.cmp(&b.2)) + }); + ranked.into_iter().map(|(_, _, _, row)| row).collect() +} + /// A short digest of a slot's credential blob — tells "same token that was /// rejected" from "the CLI refreshed / the user re-logged in". Never the /// secret itself: this lands in the usage cache file. @@ -3411,6 +3956,19 @@ fn credentials_fingerprint(credentials: &serde_json::Value) -> String { crate::repos::hex(&Sha256::digest(credentials.to_string().as_bytes()))[..16].to_string() } +/// The latest reset among the account's EXHAUSTED windows (100% used): the +/// moment every blocked window has lifted — while it is in the future, the +/// account is still limited. Windows that aren't full, or that don't know +/// their reset, contribute nothing; `None` = not limited by any cached +/// window. Pure (the row's `limited_until`). +fn limited_until(windows: &[AgentUsageWindow]) -> Option> { + windows + .iter() + .filter(|window| window.used_fraction >= 1.0) + .filter_map(|window| window.resets_at) + .max() +} + /// Send one probe request and decode its JSON body, classifying every /// failure. Logs provider/step/status/class/Retry-After — never the request /// (bearer tokens) or the response body (it can echo account details). @@ -3532,6 +4090,9 @@ fn usage_error_message( .map(|at| format!(" — retrying in {}", short_duration(at - now))) .unwrap_or_default(); Some(match error { + // A hard rejection — the account is signed out regardless of which + // login it is (this is what flipped the sign-in state). + ProbeError::SignedOut { .. } => "Signed out — sign in again".to_string(), ProbeError::RateLimited { .. } => format!("Rate limited by {provider}{retry}"), // Devin's key never expires on its own: a rejection means it was // revoked, whichever login it is. @@ -4203,6 +4764,7 @@ fn stage_file_atomic( #[cfg(test)] mod tests { use super::*; + use chrono::{TimeDelta, TimeZone}; #[test] fn plan_labels() { @@ -4685,6 +5247,21 @@ mod probe_tests { .as_deref(), Some("Signed out — sign in again") ); + // A hard rejection reads as signed out on every login, even the + // active one (a run error or a refused refresh marked it). + assert_eq!( + message(HarnessId::ClaudeCode, true, &ProbeError::SignedOut { status: 400 }).as_deref(), + Some("Signed out — sign in again") + ); + assert_eq!( + message(HarnessId::Devin, false, &ProbeError::SignedOut { status: 403 }).as_deref(), + Some("Signed out — sign in again") + ); + // The ambiguous arms keep their text. + assert_eq!( + message(HarnessId::ClaudeCode, true, &ProbeError::Unauthorized { status: 401 }).as_deref(), + Some("Session expired — it refreshes the next time claude runs") + ); assert_eq!( message( HarnessId::Codex, @@ -4709,8 +5286,130 @@ mod probe_tests { let entry = &back.entries["claude-code:acct"]; assert_eq!(entry.usage, Some(snapshot())); assert_eq!(entry.fetched_at, Some(5)); + assert_eq!(entry.sign_in, Some(AgentSignInState::SignedIn)); // The fingerprint is a digest, never the credential blob. assert!(!json.contains("accessToken")); + // A cache from before the sign-in state existed still parses, as + // unknown — and a signed-out mark survives the round trip. + let legacy: UsageCacheFile = + serde_json::from_str(r#"{"entries":{"codex:acct":{"checkedAt":9}}}"#).unwrap(); + assert_eq!(legacy.entries["codex:acct"].sign_in, None); + let mut marked = UsageEntry::default(); + marked.sign_in = Some(AgentSignInState::SignedOut); + marked.credentials = Some("f".into()); + let file = UsageCacheFile { + entries: HashMap::from([("grok:acct".to_string(), marked)]), + }; + let back: UsageCacheFile = + serde_json::from_str(&serde_json::to_string(&file).unwrap()).unwrap(); + assert_eq!(back.entries["grok:acct"].sign_in, Some(AgentSignInState::SignedOut)); + } + + /// The sign-in state moves only on hard evidence: a hard rejection + /// marks the account signed out, a success marks it signed in, and + /// every other failure (an expired token, a rate limit, a network + /// blip) leaves the last hard evidence standing. + #[test] + fn only_hard_rejections_and_successes_move_the_sign_in_state() { + let mut entry = UsageEntry::default(); + assert_eq!(entry.sign_in, None, "never probed is unknown"); + // An ambiguous rejection (the CLI refreshes such tokens itself). + entry.record( + Err(ProbeError::Unauthorized { status: 401 }), + "c".into(), + 1_000, + ); + assert_eq!(entry.sign_in, None, "an expired token is not a sign-out"); + // A rate limit says nothing about the login. + entry.record( + Err(ProbeError::RateLimited { retry_after_secs: None }), + "c".into(), + 2_000, + ); + assert_eq!(entry.sign_in, None); + // A hard rejection — the slot's refresh was refused. + entry.record( + Err(ProbeError::SignedOut { status: 400 }), + "c".into(), + 3_000, + ); + assert_eq!(entry.sign_in, Some(AgentSignInState::SignedOut)); + // Later ambiguous failures must not resurrect it as healthy. + entry.record( + Err(ProbeError::RateLimited { retry_after_secs: None }), + "c".into(), + 4_000, + ); + assert_eq!(entry.sign_in, Some(AgentSignInState::SignedOut)); + // A successful probe clears it. + entry.record(Ok(snapshot()), "c".into(), 5_000); + assert_eq!(entry.sign_in, Some(AgentSignInState::SignedIn)); + } + + /// A signed-out mark is about the credentials it was recorded against: + /// once the slot holds different ones (a sign-in landed, the CLI rotated + /// its token), the mark reads back as unknown — and the next forced list + /// probes right away instead of replaying the old rejection. + #[test] + fn a_signed_out_mark_for_old_credentials_reads_unknown_and_reprobes() { + let mut entry = UsageEntry::default(); + entry.record( + Err(ProbeError::SignedOut { status: 400 }), + "old".into(), + 1_000, + ); + assert_eq!(entry.sign_in_state("old"), AgentSignInState::SignedOut); + assert_eq!(entry.sign_in_state("new"), AgentSignInState::Unknown); + // Fresh credentials lift even the 30s freshness gate … + let soon = 1_000 + FORCED_MIN_INTERVAL.as_millis() as i64 - 1; + assert!(entry.probe_due("new", soon)); + // … while the same credentials keep it (the backoff arm also holds). + assert!(!entry.probe_due("old", soon)); + let past_freshness = 1_000 + FORCED_MIN_INTERVAL.as_millis() as i64 + 1; + assert!(!entry.probe_due("old", past_freshness)); + let backoff = ProbeError::SignedOut { status: 400 }.backoff().as_millis() as i64; + assert!(entry.probe_due("old", 1_000 + backoff)); + // A mark with no recorded credentials (never failed a probe) reads + // as recorded for any credentials. + let mut bare = UsageEntry::default(); + bare.sign_in = Some(AgentSignInState::SignedOut); + assert_eq!(bare.sign_in_state("anything"), AgentSignInState::SignedOut); + } + + /// `limited_until` is the LATEST reset among the exhausted windows — + /// when it is in the future the account is still limited. Not-full + /// windows and resets already past count for nothing. + #[test] + fn limited_until_is_the_latest_reset_of_exhausted_windows() { + use chrono::{TimeDelta, TimeZone}; + let at =|hours: i64| Utc.with_ymd_and_hms(2026, 10, 17, 9, 0, 0).unwrap() + TimeDelta::hours(hours); + let window = |label: &str, used: f32, reset: Option>| AgentUsageWindow { + label: label.into(), + used_fraction: used, + resets_at: reset, + }; + // No windows, or none exhausted. + assert_eq!(limited_until(&[]), None); + assert_eq!( + limited_until(&[window("Session", 0.99, Some(at(1))), window("Week", 0.5, Some(at(3)))]), + None + ); + // One exhausted window with no reset: nothing knowable. + assert_eq!(limited_until(&[window("Session", 1.0, None)]), None); + // The latest reset of the exhausted ones wins — not the latest of all. + assert_eq!( + limited_until(&[ + window("Session", 1.0, Some(at(1))), + window("Week", 1.0, Some(at(3))), + window("Day", 0.1, Some(at(8))), + ]), + Some(at(3)) + ); + // Exhausted at 100% exactly — 99% is not exhausted. + assert_eq!( + limited_until(&[window("Session", 1.0, Some(at(2))), window("Backup", 0.99, Some(at(5)))]), + Some(at(2)) + ); } /// A transient DNS failure during the usage probe must surface as a @@ -4762,6 +5461,124 @@ mod probe_tests { "the transport-failed probe must not be followed by a refresh POST" ); } + + // ── failover ranking (orchestration v2, ticket 09) ──────────────────── + + fn candidate( + id: &str, + used: Option, + group: Option<&str>, + mut row: AgentAccount, + ) -> FailoverCandidate { + row.id = id.into(); + row.usage_windows = used + .map(|fraction| { + vec![AgentUsageWindow { + label: "Session".into(), + used_fraction: fraction, + resets_at: None, + }] + }) + .unwrap_or_default(); + FailoverCandidate { + row, + group: group.map(str::to_owned), + } + } + + fn plain_row() -> AgentAccount { + serde_json::from_value(serde_json::json!({ + "id": "row", + "harness": "claude-code", + "email": null, + "planLabel": null, + "active": false, + "switchable": true, + "authKind": "oauth", + })) + .unwrap() + } + + fn ids(rows: &[AgentAccount]) -> Vec<&str> { + rows.iter().map(|row| row.id.as_str()).collect() + } + + /// The exclusions, one per row: the limit's own account and the active + /// login, non-switchable rows, API keys, signed-out accounts, and + /// accounts still limited never surface as candidates. + #[test] + fn failover_ranking_excludes_dead_and_current_rows() { + let mut active = plain_row(); + active.active = true; + let mut unswitchable = plain_row(); + unswitchable.switchable = false; + let mut api_key = plain_row(); + api_key.auth_kind = Some(AgentAuthKind::ApiKey); + let mut signed_out = plain_row(); + signed_out.sign_in_state = AgentSignInState::SignedOut; + let mut still_limited = plain_row(); + still_limited.limited_until = Some(Utc::now() + chrono::TimeDelta::hours(3)); + let expired_limit = { + // A limited_until in the PAST: the window lifted. + let mut row = plain_row(); + row.limited_until = Some(Utc::now() - chrono::TimeDelta::hours(1)); + row + }; + let rows = vec![ + candidate("a-active", Some(0.1), None, active), + candidate("b-limited-id", Some(0.2), None, plain_row()), + candidate("c-unswitchable", Some(0.3), None, unswitchable), + candidate("d-api-key", Some(0.4), None, api_key), + candidate("e-signed-out", Some(0.5), None, signed_out), + candidate("f-still-limited", Some(0.6), None, still_limited), + candidate("g-lifted", Some(0.7), None, expired_limit), + candidate("h-healthy", Some(0.8), None, plain_row()), + ]; + let ranked = rank_failover_candidates(rows, Some("b-limited-id"), None); + assert_eq!(ids(&ranked), vec!["g-lifted", "h-healthy"]); + } + + /// Lowest usage first (unknown meters read as nothing used), and among + /// equals the accounts OUTSIDE the limited account's org or workspace + /// rank first. + #[test] + fn failover_ranking_prefers_low_usage_then_other_pools() { + let rows = vec![ + candidate("high", Some(0.9), Some("acme"), plain_row()), + candidate("low", Some(0.1), Some("acme"), plain_row()), + candidate("other-pool", Some(0.1), Some("globex"), plain_row()), + candidate("unknown", None, Some("acme"), plain_row()), + ]; + let ranked = rank_failover_candidates(rows, Some("limited"), Some("acme")); + assert_eq!( + ids(&ranked), + vec!["other-pool", "unknown", "low", "high"], + "lowest usage first (no meters read as nothing used); a different pool wins ties" + ); + } + + /// The org/workspace group: Claude's organization (case-insensitive), + /// Codex's workspace tail of the account key, absent elsewhere. + #[test] + fn failover_groups_map_claude_orgs_and_codex_workspaces() { + assert_eq!( + failover_group(HarnessId::ClaudeCode, Some("Acme Corp "), ""), + Some("acme corp".into()) + ); + assert_eq!(failover_group(HarnessId::ClaudeCode, Some(""), ""), None); + assert_eq!(failover_group(HarnessId::ClaudeCode, None, ""), None); + assert_eq!( + failover_group(HarnessId::Codex, None, "user-1::ws-9"), + Some("ws-9".into()) + ); + // A personal login (no workspace tail) shares nothing: its whole key + // IS its identity. + assert_eq!( + failover_group(HarnessId::Codex, None, "dev@example.com"), + Some("dev@example.com".into()) + ); + assert_eq!(failover_group(HarnessId::Pi, Some("Acme"), "k"), None); + } } #[cfg(test)] diff --git a/crates/engine/src/agent_accounts/provider_tests.rs b/crates/engine/src/agent_accounts/provider_tests.rs index 1069698b46..66c617c785 100644 --- a/crates/engine/src/agent_accounts/provider_tests.rs +++ b/crates/engine/src/agent_accounts/provider_tests.rs @@ -1,6 +1,9 @@ //! Grok / Devin / OpenCode / Pi / Hermes accounts against fake credential //! stores in temp dirs, fake CLIs and a local mock of every provider -//! endpoint — never a real login, token or network call. +//! endpoint — never a real login, token or network call. The Claude +//! sign-in-state tests live here too: they need the same mocked usage / +//! token endpoints to drive a hard (refused-refresh) rejection against a +//! saved slot. use std::sync::atomic::{AtomicUsize, Ordering}; @@ -33,10 +36,12 @@ fn chatgpt_access(email: &str, account_id: &str, plan: &str) -> String { })) } -type Handler = dyn Fn(&str, &str, &str) -> (u16, String) + Send + Sync; +type Handler = dyn Fn(&str, &str, &str, &str) -> (u16, String) + Send + Sync; /// A local stand-in for provider endpoints: `handler(method, path+query, -/// body)` answers every request; `hits` records `"METHOD path"`. +/// body, bearer)` answers every request (the bearer is the Authorization +/// header's credential, `""` when none was sent); `hits` records +/// `"METHOD path"`. struct MockServer { base: String, hits: Arc>>, @@ -45,7 +50,7 @@ struct MockServer { impl MockServer { async fn start( - handler: impl Fn(&str, &str, &str) -> (u16, String) + Send + Sync + 'static, + handler: impl Fn(&str, &str, &str, &str) -> (u16, String) + Send + Sync + 'static, ) -> Self { let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) .await @@ -89,8 +94,19 @@ impl MockServer { let mut first = head.lines().next().unwrap_or("").split_whitespace(); let method = first.next().unwrap_or("").to_string(); let path = first.next().unwrap_or("").to_string(); + // The bearer's credential, when the request carried one — + // handlers branch on WHICH account probed (hyper sends + // lowercase header names). + let bearer = head + .lines() + .find(|l| l.to_ascii_lowercase().starts_with("authorization:")) + .and_then(|l| l.split_once(':')) + .map(|(_, v)| v.trim()) + .and_then(|v| v.strip_prefix("Bearer ")) + .map(str::to_string) + .unwrap_or_default(); lock(&hits).push(format!("{method} {path}")); - let (status, reply) = handler(&method, &path, &body); + let (status, reply) = handler(&method, &path, &body, &bearer); let response = http_response( &format!("{status} X"), &[("Content-Type", "application/json")], @@ -124,23 +140,11 @@ fn accounts_with(root: &Path, endpoints: ProbeEndpoints) -> (AgentAccounts, Agen ) } -/// Endpoints that all point at `base` (a mock), with slot refresh allowed. +/// Endpoints that all point at `base` (a mock), with slot refresh allowed — +/// [`mocked_endpoints`] in the parent, kept as the local spelling the flows +/// read best in. fn mocked(base: &str) -> ProbeEndpoints { - ProbeEndpoints { - claude_usage: format!("{base}/api/oauth/usage"), - claude_token: format!("{base}/v1/oauth/token"), - claude_loopback_token: format!("{base}/v1/oauth/token"), - claude_profile: format!("{base}/api/oauth/profile"), - codex_usage: format!("{base}/backend-api/wham/usage"), - grok_usage: format!("{base}/v1/billing?format=credits"), - github_api: base.to_string(), - github_login: base.to_string(), - openai_auth: base.to_string(), - openai_port: 0, - nous_portal: base.to_string(), - allow_slot_refresh: true, - allow_loopback_http: true, - } + mocked_endpoints(base) } fn rows(snapshot: &AgentAccountsSnapshot, harness: HarnessId) -> Vec { @@ -415,7 +419,7 @@ async fn forgetting_a_live_login_signs_out_only_that_login() { async fn a_rejected_saved_grok_token_refreshes_once_against_its_issuer() { let issuer_hits = Arc::new(AtomicUsize::new(0)); let hits = issuer_hits.clone(); - let server = MockServer::start(move |method, path, body| match (method, path) { + let server = MockServer::start(move |method, path, body, _| match (method, path) { ("GET", p) if p.starts_with("/v1/billing") => { // Only the refreshed token is accepted. (200, r#"{"config":{"creditUsagePercent":30}}"#.to_string()) @@ -432,7 +436,7 @@ async fn a_rejected_saved_grok_token_refreshes_once_against_its_issuer() { }) .await; // Billing rejects every token but "fresh". - let billing = MockServer::start(|_, _, _| (401, String::new())).await; + let billing = MockServer::start(|_, _, _, _| (401, String::new())).await; let tmp = tempfile::tempdir().unwrap(); let mut endpoints = mocked(&server.base); endpoints.grok_usage = format!("{}/v1/billing?format=credits", billing.base); @@ -459,9 +463,11 @@ async fn a_rejected_saved_grok_token_refreshes_once_against_its_issuer() { // A saved one is — once — and the rotated pair lands in its slot. let saved = accounts.grok_usage(&slot, false).await; assert_eq!(issuer_hits.load(Ordering::SeqCst), 1); + // The issuer minted a fresh token and billing STILL rejects it: the + // login is dead — the hard signed-out signal, not an ambiguous expiry. assert!( - matches!(saved, Err(ProbeError::Unauthorized { .. })), - "billing mock rejects all" + matches!(saved, Err(ProbeError::SignedOut { .. })), + "billing mock rejects all: {saved:?}" ); let stored = accounts.read_slot(HarnessId::Grok, &slot.id).unwrap(); assert_eq!(stored.credentials["key"], "fresh"); @@ -576,7 +582,7 @@ fn devin_usage_inverts_remaining_quota_and_names_the_account() { #[tokio::test] async fn devin_swaps_and_learns_its_identity_from_the_usage_probe() { - let server = MockServer::start(|method, path, body| { + let server = MockServer::start(|method, path, body, _| { assert_eq!(method, "POST"); assert!( path.ends_with("SeatManagementService/GetUserStatus"), @@ -785,7 +791,7 @@ async fn pi_swaps_under_its_lockfile_and_groups_rows_per_provider() { #[tokio::test] async fn an_opaque_pi_claude_token_is_identified_once_then_matched_by_token() { - let server = MockServer::start(|_, path, _| match path { + let server = MockServer::start(|_, path, _, _| match path { "/api/oauth/profile" => ( 200, serde_json::json!({ @@ -826,7 +832,7 @@ async fn an_opaque_pi_claude_token_is_identified_once_then_matched_by_token() { #[tokio::test] async fn an_unidentifiable_live_token_is_listed_but_not_switchable() { - let server = MockServer::start(|_, _, _| (401, String::new())).await; + let server = MockServer::start(|_, _, _, _| (401, String::new())).await; let tmp = tempfile::tempdir().unwrap(); let (accounts, config) = accounts_with(tmp.path(), mocked(&server.base)); write( @@ -850,7 +856,7 @@ async fn an_unidentifiable_live_token_is_listed_but_not_switchable() { #[tokio::test] async fn chatgpt_sign_in_for_pi_lands_on_the_loopback_and_connects_the_first_login() { - let server = MockServer::start(|method, path, body| match (method, path) { + let server = MockServer::start(|method, path, body, _| match (method, path) { ("POST", "/oauth/token") => { assert!(body.contains("grant_type=authorization_code"), "{body}"); assert!(body.contains("code=good-code"), "{body}"); @@ -936,7 +942,7 @@ async fn chatgpt_sign_in_for_pi_lands_on_the_loopback_and_connects_the_first_log /// stores too). #[tokio::test] async fn re_signing_in_the_live_chatgpt_account_replaces_its_dead_tokens() { - let server = MockServer::start(|method, path, _| match (method, path) { + let server = MockServer::start(|method, path, _, _| match (method, path) { ("POST", "/oauth/token") => ( 200, serde_json::json!({ @@ -991,7 +997,7 @@ async fn re_signing_in_the_live_chatgpt_account_replaces_its_dead_tokens() { async fn a_new_login_never_replaces_an_unidentified_live_one() { let user_calls = Arc::new(AtomicUsize::new(0)); let calls = user_calls.clone(); - let server = MockServer::start(move |method, path, _| match (method, path) { + let server = MockServer::start(move |method, path, _, _| match (method, path) { ("POST", "/login/device/code") => ( 200, r#"{"device_code":"dc","user_code":"ABCD-1234","verification_uri":"https://github.com/login/device","interval":1}"#.into(), @@ -1063,7 +1069,7 @@ async fn a_new_chatgpt_sign_in_supersedes_one_holding_the_same_port() { async fn copilot_device_sign_in_for_opencode_shows_the_code_and_connects() { let polls = Arc::new(AtomicUsize::new(0)); let polled = polls.clone(); - let server = MockServer::start(move |method, path, body| match (method, path) { + let server = MockServer::start(move |method, path, body, _| match (method, path) { ("POST", "/login/device/code") => { assert!(body.contains("client_id=Ov23li8tweQw6odWQebz"), "{body}"); ( @@ -1148,7 +1154,7 @@ fn copilot_usage_reads_metered_quotas_and_the_plan() { #[tokio::test] async fn keyed_logins_probe_the_vendor_behind_them() { - let server = MockServer::start(|method, path, _| match (method, path) { + let server = MockServer::start(|method, path, _, _| match (method, path) { ("GET", "/backend-api/wham/usage") => ( 200, r#"{"plan_type":"plus","rate_limit":{"primary_window":{"used_percent":40,"limit_window_seconds":18000}}}"#.into(), @@ -1222,7 +1228,7 @@ fn hermes_auth() -> serde_json::Value { #[tokio::test] async fn hermes_lists_its_own_pool_read_only() { - let server = MockServer::start(|_, path, _| match path { + let server = MockServer::start(|_, path, _, _| match path { "/api/oauth/account" => ( 200, r#"{"user":{"email":"nous@example.com"},"subscription":{"plan":"plus","monthly_credits":100,"credits_remaining":25,"current_period_end":"2030-02-01T00:00:00Z"}}"#.into(), @@ -1610,7 +1616,7 @@ fn devin_sign_in_urls_must_be_devins_own_pages() { #[tokio::test] async fn untrusted_endpoints_in_credentials_are_never_sent_a_secret() { - let server = MockServer::start(|_, _, _| (200, "{}".into())).await; + let server = MockServer::start(|_, _, _, _| (200, "{}".into())).await; let tmp = tempfile::tempdir().unwrap(); // Production endpoints: a loopback url in a credential is NOT trusted. let (accounts, config) = accounts_with(tmp.path(), ProbeEndpoints::default()); @@ -1721,7 +1727,7 @@ fn the_accounts_root_and_sign_in_homes_are_owner_only() { /// any network call, and restored byte-for-byte after switching away. #[tokio::test] async fn a_self_hosted_ghes_login_switches_away_and_back_without_the_network() { - let server = MockServer::start(|_, path, _| match path { + let server = MockServer::start(|_, path, _, _| match path { "/user" => ( 200, r#"{"id":42,"login":"octo","email":"octo@example.com"}"#.into(), @@ -1898,3 +1904,327 @@ async fn a_failed_sign_ins_output_is_redacted_before_the_ui_sees_it() { "failed at https://auth.x.ai/device?… with code [code] token=[redacted]" ); } + +// ── Claude: the harness account sign-in state (ticket 08) ─────────────────── + +/// Write a Claude login (config + credentials) for `uuid` with `token`. +fn write_claude_login(config: &AgentAccountsConfig, uuid: &str, email: &str, token: &str) { + write( + &config.claude_config_file, + &serde_json::json!({ + "oauthAccount": { + "accountUuid": uuid, + "emailAddress": email, + "displayName": "Test User", + }, + }) + .to_string(), + ); + write( + &config.claude_creds_file(), + &serde_json::json!({ + "claudeAiOauth": { + "accessToken": token, + "refreshToken": format!("refresh-{token}"), + // Far-future expiry: probes must never try to rotate it. + "expiresAt": 4_102_444_800_000i64, + } + }) + .to_string(), + ); +} + +/// Claude's `/api/oauth/usage` reply for two windows at the given percents. +fn claude_usage_body(session: f64, week: f64) -> String { + serde_json::json!({ + "five_hour": { "utilization": session, "resets_at": "2026-10-18T08:00:00Z" }, + "seven_day": { "utilization": week, "resets_at": "2026-10-20T16:00:00Z" }, + }) + .to_string() +} + +/// The live credential fingerprint keys on the login's IDENTITY, not its +/// token bytes (ticket 09): a CLI token refresh rewrites the credential +/// file on every expiry, and a bytes-keyed fingerprint would retire every +/// warm runtime on the SAME login — a mid-turn steer became an +/// interrupt-and-redispatch. Only a real login switch may change it. +#[test] +fn the_live_credential_fingerprint_keys_on_identity_not_tokens() { + let tmp = tempfile::tempdir().unwrap(); + let config = AgentAccountsConfig::isolated(tmp.path()); + let accounts = AgentAccounts::new(config.clone()); + write_claude_login(&config, "uuid-alice", "alice@example.com", "alice-token"); + let alice = accounts + .live_credentials_fingerprint(HarnessId::ClaudeCode) + .expect("a fingerprint for the live login"); + + // The CLI refreshes the access token: the credential file's bytes all + // change, the login does not. + write_claude_login(&config, "uuid-alice", "alice@example.com", "alice-token-2"); + assert_eq!( + accounts.live_credentials_fingerprint(HarnessId::ClaudeCode), + Some(alice.clone()), + "a token refresh must not churn the fingerprint" + ); + + // A real switch to another account: a different identity, a different + // fingerprint — the warm runtimes retire. + write_claude_login(&config, "uuid-bob", "bob@example.com", "bob-token"); + let bob = accounts + .live_credentials_fingerprint(HarnessId::ClaudeCode) + .expect("a fingerprint for the switched login"); + assert_ne!(bob, alice, "a login switch changes the fingerprint"); +} + +/// A live login that a run refused: an expired access token the CLI refreshes +/// itself is AMBIGUOUS — unknown, never signed out — and no refresh is ever +/// attempted for the live login. +#[tokio::test] +async fn an_expired_active_claude_login_is_unknown_not_signed_out() { + let server = MockServer::start(|_method, path, _body, _bearer| match path { + "/api/oauth/usage" => (401, String::new()), + _ => (404, String::new()), + }) + .await; + let tmp = tempfile::tempdir().unwrap(); + let (accounts, config) = accounts_with(tmp.path(), mocked(&server.base)); + write_claude_login(&config, "uuid-alice", "alice@example.com", "alice-token"); + + let rows = rows(&accounts.list(true).await.unwrap(), HarnessId::ClaudeCode); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].sign_in_state, AgentSignInState::Unknown); + assert_eq!( + rows[0].usage_error.as_deref(), + Some("Session expired — it refreshes the next time claude runs") + ); + // The active login is the CLI's to rotate: no refresh was attempted. + assert_eq!(server.hits("POST /v1/oauth/token"), 0); +} + +/// A hard rejection — the saved slot's refresh was refused (invalid_grant) — +/// marks the account signed out, and the reason shows BESIDE the cached +/// meters instead of being hidden by them (the display bug: stale windows +/// made a signed-out account look healthy). +#[tokio::test] +async fn a_refused_claude_refresh_marks_the_slot_signed_out_beside_cached_windows() { + let server = MockServer::start(|_method, path, _body, bearer| match path { + "/api/oauth/usage" => { + if bearer == "alice-token" { + (200, claude_usage_body(6.0, 35.0)) + } else { + (401, String::new()) + } + } + // The refused refresh: 400 invalid_grant means the login is dead. + "/v1/oauth/token" => (400, r#"{"error":"invalid_grant"}"#.into()), + _ => (404, String::new()), + }) + .await; + let tmp = tempfile::tempdir().unwrap(); + let (accounts, config) = accounts_with(tmp.path(), mocked(&server.base)); + + // Alice is live; Bob is a saved, inactive slot. + write_claude_login(&config, "uuid-alice", "alice@example.com", "alice-token"); + accounts.list(false).await.unwrap(); + write_claude_login(&config, "uuid-bob", "bob@example.com", "bob-token"); + accounts.list(false).await.unwrap(); + write_claude_login(&config, "uuid-alice", "alice@example.com", "alice-token"); + accounts.list(false).await.unwrap(); + + // Bob's meters were right a while ago: the cache serves them until a + // probe lands. Age the entry so the forced list below re-probes him. + lock(&accounts.inner.usage).insert( + usage_key(HarnessId::ClaudeCode, "uuid-bob"), + UsageEntry { + usage: Some(UsageSnapshot { + windows: vec![AgentUsageWindow { + label: "Session".into(), + used_fraction: 0.4, + resets_at: None, + }], + plan_label: None, + }), + fetched_at: Some(1), + checked_at: 1, + ..Default::default() + }, + ); + + let rows = rows(&accounts.list(true).await.unwrap(), HarnessId::ClaudeCode); + let bob = rows + .iter() + .find(|a| a.email.as_deref() == Some("bob@example.com")) + .unwrap(); + assert_eq!(bob.sign_in_state, AgentSignInState::SignedOut); + // The stale meters survive the failure — and no longer hide the reason. + assert!(!bob.usage_windows.is_empty()); + assert_eq!(bob.usage_error.as_deref(), Some("Signed out — sign in again")); + // The live account's own probe is green. + let alice = rows + .iter() + .find(|a| a.email.as_deref() == Some("alice@example.com")) + .unwrap(); + assert_eq!(alice.sign_in_state, AgentSignInState::SignedIn); + assert!(alice.usage_error.is_none()); +} + +/// "Sign in again": completing the login replaces the dead slot's +/// credentials, the next forced list re-probes them right away (fresh +/// credentials lift the freshness gate), and the account reads signed in — +/// with its exhausted window's reset as `limited_until`. +#[tokio::test] +async fn signing_in_again_marks_the_account_signed_in() { + let server = MockServer::start(|_method, path, body, bearer| match path { + "/api/oauth/usage" => match bearer { + "alice-token" => (200, claude_usage_body(6.0, 35.0)), + "bob-fresh" => (200, claude_usage_body(100.0, 35.0)), + _ => (401, String::new()), + }, + "/v1/oauth/token" => { + // The slot-refresh POST (JSON: grant_type refresh_token) is + // refused; the sign-in's code exchange succeeds. + if body.contains("refresh_token") { + (400, r#"{"error":"invalid_grant"}"#.into()) + } else { + ( + 200, + serde_json::json!({ + "access_token": "bob-fresh", + "refresh_token": "refresh-bob-fresh", + "expires_in": 3600, + "account": { + "email_address": "bob@example.com", + "uuid": "uuid-bob", + }, + }) + .to_string(), + ) + } + } + _ => (404, String::new()), + }) + .await; + let tmp = tempfile::tempdir().unwrap(); + let (accounts, config) = accounts_with(tmp.path(), mocked(&server.base)); + + // Alice live, Bob a saved slot whose probe is hard-rejected. + write_claude_login(&config, "uuid-alice", "alice@example.com", "alice-token"); + accounts.list(false).await.unwrap(); + write_claude_login(&config, "uuid-bob", "bob@example.com", "bob-token"); + accounts.list(false).await.unwrap(); + write_claude_login(&config, "uuid-alice", "alice@example.com", "alice-token"); + let rows = rows(&accounts.list(true).await.unwrap(), HarnessId::ClaudeCode); + let bob = rows + .iter() + .find(|a| a.email.as_deref() == Some("bob@example.com")) + .unwrap(); + assert_eq!(bob.sign_in_state, AgentSignInState::SignedOut); + + // "Sign in again" — the existing login flow, finishing in the browser. + let start = accounts.start_login(HarnessId::ClaudeCode).await.unwrap(); + let state = query_param(&start.url, "state"); + let port = start.callback_port.unwrap(); + let page = browser_get(port, &format!("/callback?code=good-code&state={state}")).await; + assert!(page.starts_with("HTTP/1.1 302"), "{page}"); + let polls = settle(&accounts, &start.login_id).await; + assert_eq!(polls.last().unwrap().status, AgentLoginStatus::Done); + + // The post-login reload re-probes Bob immediately: his slot now holds + // fresh credentials, so the old rejection no longer applies. + let rows = self::rows(&accounts.list(true).await.unwrap(), HarnessId::ClaudeCode); + let bob = rows + .iter() + .find(|a| a.email.as_deref() == Some("bob@example.com")) + .unwrap(); + assert_eq!(bob.sign_in_state, AgentSignInState::SignedIn); + assert!(bob.usage_error.is_none()); + assert!(!bob.usage_windows.is_empty()); + // His session window is exhausted (100%): the account is limited until + // that window's reset — the latest reset among exhausted windows. + assert_eq!( + bob.limited_until, + Some( + DateTime::parse_from_rfc3339("2026-10-18T08:00:00Z") + .unwrap() + .with_timezone(&Utc) + ) + ); + // Alice stays signed in too. + let alice = rows + .iter() + .find(|a| a.email.as_deref() == Some("alice@example.com")) + .unwrap(); + assert_eq!(alice.sign_in_state, AgentSignInState::SignedIn); + assert_eq!(alice.limited_until, None); +} + +/// The run-error path's recording seam: `mark_account_signed_out` writes a +/// durable mark for a slot (ticket 05's wiring calls it with the structured +/// signed-out error's harness), and a later successful probe clears it. +/// An unknown account is refused. +#[tokio::test] +async fn a_run_error_sign_out_mark_lands_and_a_later_probe_clears_it() { + let server = MockServer::start(|_method, path, _body, _bearer| match path { + "/api/oauth/usage" => (200, claude_usage_body(6.0, 35.0)), + _ => (404, String::new()), + }) + .await; + let tmp = tempfile::tempdir().unwrap(); + let (accounts, config) = accounts_with(tmp.path(), mocked(&server.base)); + write_claude_login(&config, "uuid-alice", "alice@example.com", "alice-token"); + let alice_id = rows(&accounts.list(false).await.unwrap(), HarnessId::ClaudeCode) + .remove(0) + .id; + + // A run failed with a signed-out error on the active account. + accounts + .mark_account_signed_out(HarnessId::ClaudeCode, &alice_id) + .unwrap(); + let rows = rows(&accounts.list(false).await.unwrap(), HarnessId::ClaudeCode); + assert_eq!(rows[0].sign_in_state, AgentSignInState::SignedOut); + assert!(rows[0].usage_error.is_none(), "no probe ran to fail"); + assert_eq!(server.hits("GET /api/oauth/usage"), 0, "the mark is offline"); + + // The next successful probe re-establishes the account. + let rows = self::rows(&accounts.list(true).await.unwrap(), HarnessId::ClaudeCode); + assert_eq!(rows[0].sign_in_state, AgentSignInState::SignedIn); + + // An unknown slot id is refused, not silently ignored. + let error = accounts + .mark_account_signed_out(HarnessId::ClaudeCode, "0123456789abcdef") + .unwrap_err(); + assert!(error.to_string().contains("Unknown account")); +} + +/// Devin's keys never lapse: a rejected status call is a revocation — the +/// hard signed-out signal — and reads as such on the row. +#[tokio::test] +async fn a_rejected_devin_key_marks_the_account_signed_out() { + let server = MockServer::start(|method, path, _body, _bearer| { + assert_eq!(method, "POST"); + assert!( + path.ends_with("SeatManagementService/GetUserStatus"), + "{path}" + ); + (401, String::new()) + }) + .await; + let tmp = tempfile::tempdir().unwrap(); + let (accounts, config) = accounts_with(tmp.path(), mocked(&server.base)); + write( + &config.devin_credentials_file, + &format!( + "windsurf_api_key = \"key-a\" +api_server_url = \"{}\" +", + server.base + ), + ); + + let rows = rows(&accounts.list(true).await.unwrap(), HarnessId::Devin); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].sign_in_state, AgentSignInState::SignedOut); + assert_eq!(rows[0].usage_error.as_deref(), Some("Signed out — sign in again")); +} diff --git a/crates/engine/src/agent_accounts/usage.rs b/crates/engine/src/agent_accounts/usage.rs index 8f708ea8b2..60badc01b5 100644 --- a/crates/engine/src/agent_accounts/usage.rs +++ b/crates/engine/src/agent_accounts/usage.rs @@ -9,7 +9,9 @@ //! the CLI's to rotate. //! - **Devin**: Connect-JSON `SeatManagementService/GetUserStatus` (what //! `devin auth status` calls) — daily/weekly quota, plus the identity the -//! key file lacks, written back into the slot. Keys don't refresh. +//! key file lacks, written back into the slot. Keys don't refresh and never +//! lapse on their own, so a rejection (401/403) means the key was revoked: +//! the hard signed-out signal. //! - **ChatGPT logins** (OpenCode `openai`, Pi `openai-codex`, Hermes //! `openai-codex`): the Codex usage endpoint — the same token kind. //! - **Claude logins** (Pi `anthropic`): Claude's `/api/oauth/usage`. @@ -51,9 +53,24 @@ impl AgentAccounts { Err(ProbeError::Unauthorized { status }) if !is_active && self.inner.endpoints.allow_slot_refresh => { - match self.refresh_grok_slot(slot).await? { - Some(fresh) => self.grok_usage_request(&fresh).await, - None => Err(ProbeError::Unauthorized { status }), + match self.refresh_grok_slot(slot).await { + Ok(Some(fresh)) => match self.grok_usage_request(&fresh).await { + // The issuer refused the token it just minted: the + // login is dead, not stale. + Err(ProbeError::Unauthorized { status }) => { + Err(ProbeError::SignedOut { status }) + } + result => result, + }, + // Refresh in flight elsewhere (single-flight) — the + // original rejection; the next probe sees the result. + Ok(None) => Err(ProbeError::Unauthorized { status }), + // A refused refresh (invalid_grant: revoked/used token) + // is the hard signed-out signal. + Err(ProbeError::Unauthorized { status }) => { + Err(ProbeError::SignedOut { status }) + } + Err(other) => Err(other), } } result => result, @@ -162,7 +179,7 @@ impl AgentAccounts { ) .ok_or(ProbeError::UntrustedEndpoint)?; let server = server.as_str(); - let body = probe_json( + let body = match probe_json( "devin", "usage", self.inner @@ -186,7 +203,15 @@ impl AgentAccounts { } })), ) - .await?; + .await + { + // Devin keys never lapse on their own: a rejection means the key + // was revoked — the hard signed-out signal, on any login. + Err(ProbeError::Unauthorized { status }) => { + return Err(ProbeError::SignedOut { status }) + } + other => other?, + }; let (snapshot, identity) = devin_usage_snapshot(&body).ok_or_else(|| schema_error("devin", &body))?; // Adopt the probed identity (best-effort; a failed write re-probes). diff --git a/crates/engine/src/completion_notices.rs b/crates/engine/src/completion_notices.rs new file mode 100644 index 0000000000..512dd60670 --- /dev/null +++ b/crates/engine/src/completion_notices.rs @@ -0,0 +1,1543 @@ +//! Completion notices (orchestration v2): when a turn of an agent-created, +//! notifying side chat comes to rest, the engine records an outcome for +//! the chat that created it — based on its own recorded turn-end state, +//! never on anything the agent calls — and wakes the parent with one +//! engine-authored notice message that starts a parent turn. +//! +//! Ticket 12 was the tracer bullet (outcome records, idle-parent +//! delivery). This module carries ticket 13's rules on top: +//! +//! - **Busy parent:** a notice is never steered into a working parent. The +//! outcome stays pending until the parent's turn ends, and the notice +//! then starts its NEXT turn (the run loop calls [`deliver`] from its +//! turn-end sites — see `spawn_notice_delivery` in `sessions.rs`). +//! - **Merging:** outcomes that arrive before a delivery — several side +//! chats finishing while the parent works, or one chat's later turns — +//! pile up in the same pending set and are delivered as ONE notice. +//! - **Durability:** the pending set is mirrored to an engine-local ledger +//! (ADR 0004) on every change and loaded at assembly, so undelivered +//! outcomes survive an engine restart and reach their parent after boot. +//! - **Runs lost to a restart:** boot recovery records an [`Outcome::Interrupted`] +//! outcome for a notifiable run the restart killed and did not +//! auto-resume. An auto-resumed run owes nothing at boot — the +//! re-dispatch keeps the starting message's origin, so the notice +//! follows the resumed turn's own end. +//! - **Archived and deleted chats are quiet both ways:** an archived side +//! chat records nothing for its parent, an archived or deleted parent is +//! owed nothing (the delete paths drop the parent's notices; a deletion +//! that raced a record finds the row gone and stays quiet). +//! +//! Ticket 14's rules on top: +//! +//! - **Acknowledgement:** an outcome is dropped when the parent observed it +//! after it happened — through `wait_for_turn`, `send_message` with wait, +//! `read_chat` or `get_chat`. The MCP server reports such observations to +//! the engine ([`CompletionNotices::ack`]) with the moment the +//! observation started, so the watermark covers the settles THAT +//! observation saw — never one that landed between the read and the +//! report, which is the parent's unseen outcome. +//! - **Stop and interrupt:** stopping or interrupting the parent drops the +//! notices owed to it and suppresses outcomes from side chats whose +//! notifiable turns were running at that moment ([`stop_parent`] — T3's +//! interrupt behaviour). Side chats keep running; Stop does not cascade. +//! A later message from the parent to a side chat re-arms its next turn +//! (`mark_notifiable` in `sessions.rs`). +//! - **Loop cap:** at most [`NOTICE_LOOP_CAP`] consecutive parent turns may +//! be started by notices without a user message in between. Beyond it, +//! the notice parks as a visible, PAUSED queue row on the parent — +//! carrying the notice's own sides, so its release writes the same +//! System completion-notice entry, never a user message — and merges with +//! any row already parked. It waits for the user: a user-authored message +//! resets the count and thaws the queue, and the row then runs as the +//! parent's next turn. +//! - **Delivery is single-flight per parent:** two settling children cannot +//! both deliver — the second would steer into the notice turn the first +//! is starting. The loser's outcomes stay pending for that turn's end. +//! +//! The notice message is a SYSTEM entry — never a user message — carrying a +//! [`roboco_doc::MessagePart::CompletionNotice`] marker so clients can render +//! it distinctly (ticket 15), with the same text as a plain part so older +//! layouts stay readable. Its text names each side chat by title and short +//! id with its outcome, and points the parent agent at `read_chat`. + +use std::collections::HashMap; +use std::collections::VecDeque; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use serde::{Deserialize, Serialize}; + +use roboco_doc::{ + MessagePart, MessageRole, MessageStatus, NoticeOutcome, NoticeSide, QueuedMessage, + SessionMessageEntry, +}; + +use crate::doc_host::DocHost; +use crate::sessions::SessionsEngine; +use crate::{new_id, now_ms}; + +/// How many recent settles per chat the observation watermark can look +/// back through (ticket 14's acknowledgement): the map keeps one timestamp +/// per settled turn, bounded so a long-lived chat cannot grow it without +/// end. An observation older than every kept settle reads as watermark 0 +/// (nothing dropped) — conservative, never a lost report. +const SETTLE_HISTORY: usize = 1024; + +/// Cap for a notice's one-line detail — an error's reason or a question's +/// summary. Matches the transcript's tool-output summary budget: the notice +/// is a pointer to `read_chat`, never a report. +pub(crate) const NOTICE_DETAIL_MAX: usize = 160; + +/// The reason a restart-lost outcome carries: one line, from the engine's +/// own interruption note ("Run interrupted by engine restart"). +pub(crate) const INTERRUPTED_REASON: &str = "engine restart"; + +/// At most this many consecutive parent turns may be started by notices +/// without a user message in between (ticket 14; the spec's default 25). +/// Beyond it a notice parks in the parent's queue, paused, until the user +/// acts — a runaway orchestration loop stops and waits for them. +pub(crate) const NOTICE_LOOP_CAP: u32 = 25; + +/// The id prefix the engine stamps on its own notice artifacts: the system +/// notice entry a delivery writes, and the paused queue row the loop cap +/// parks (ticket 14). The turn-start bookkeeping keys on it — a turn whose +/// starting message id carries it is notice-started, not user-started +/// (see [`CompletionNotices::notice_turn_started`]). +pub(crate) const NOTICE_ID_PREFIX: &str = "notice:"; + +/// The engine's recorded turn-end state for one notifiable side-chat turn. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum Outcome { + /// The turn completed. + Completed, + /// The turn errored, with its one-line reason. + Errored { reason: Option }, + /// The turn was stopped or interrupted by the user. + Stopped, + /// The run was lost to an engine restart and not auto-resumed (ticket + /// 13): the parent is told the work never finished. An auto-resumed + /// run owes nothing here — its notice follows the resumed turn's end. + Interrupted { reason: Option }, + /// The turn is waiting for input, with the one-line summary of the + /// question or approval it is waiting on. + WaitingForInput { summary: String }, +} + +impl Outcome { + /// The wire row this outcome records as: the outcome kind plus the + /// one-line detail it carries. One shape for the notice row, the + /// durable ledger and the notice text. + fn as_row(&self) -> (NoticeOutcome, Option) { + match self { + Outcome::Completed => (NoticeOutcome::Completed, None), + Outcome::Errored { reason } => (NoticeOutcome::Errored, reason.clone()), + Outcome::Stopped => (NoticeOutcome::Stopped, None), + Outcome::Interrupted { reason } => (NoticeOutcome::Interrupted, reason.clone()), + Outcome::WaitingForInput { summary } => { + (NoticeOutcome::WaitingForInput, Some(summary.clone())) + } + } + } + + /// Rebuild an outcome from a stored row ([`Outcome::as_row`]'s inverse). + fn from_row(outcome: NoticeOutcome, detail: Option) -> Outcome { + match outcome { + NoticeOutcome::Completed => Outcome::Completed, + NoticeOutcome::Errored => Outcome::Errored { reason: detail }, + NoticeOutcome::Stopped => Outcome::Stopped, + NoticeOutcome::Interrupted => Outcome::Interrupted { reason: detail }, + NoticeOutcome::WaitingForInput => Outcome::WaitingForInput { + summary: detail.unwrap_or_else(|| "input".into()), + }, + } + } +} + +/// An outcome recorded for a parent, awaiting delivery. +#[derive(Debug, Clone)] +pub(crate) struct PendingOutcome { + /// The side chat whose turn ended. + pub(crate) side_chat_id: String, + pub(crate) outcome: Outcome, + /// The side chat's settle serial this outcome belongs to (ticket 14): + /// the monotonic turn counter stamped by [`CompletionNotices::settle`] + /// at the turn's settle. An observation of the side chat covers + /// outcomes at or below its watermark. A reloaded (restarted) outcome + /// reads as 0, which any later observation covers — that outcome is for + /// a turn that settled before the restart, so a read after it has seen + /// it. + pub(crate) settle: u64, +} + +/// One pending outcome as it rests on disk. The wire row's own shape +/// (camelCase, additive — ADR 0005), so the ledger needs no schema of its +/// own and a row written by an older engine still reads. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct StoredOutcome { + side_chat_id: String, + outcome: NoticeOutcome, + #[serde(default, skip_serializing_if = "Option::is_none")] + detail: Option, +} + +impl From<&PendingOutcome> for StoredOutcome { + fn from(pending: &PendingOutcome) -> Self { + let (outcome, detail) = pending.outcome.as_row(); + Self { + side_chat_id: pending.side_chat_id.clone(), + outcome, + detail, + } + } +} + +impl From for PendingOutcome { + fn from(stored: StoredOutcome) -> Self { + Self { + side_chat_id: stored.side_chat_id, + outcome: Outcome::from_row(stored.outcome, stored.detail), + // A restarted outcome's serial is unknown — 0 reads as "settled + // before the restart", which any later observation of the side + // chat covers (see `PendingOutcome::settle`). + settle: 0, + } + } +} + +/// The ledger on disk (ticket 14): the pending outcomes plus the loop-cap +/// counters, both of which must survive a restart. Written as +/// `{ pending, loops }`; a ticket-13 ledger (a bare map of parent → +/// outcome rows) still reads — its pending set loads, its loop counts are +/// empty. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct StoredLedger { + #[serde(default)] + pending: HashMap>, + /// parent → consecutive notice-started turns at the last write. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + loops: HashMap, +} + +/// The notice bookkeeping under one lock: pending outcomes (keyed by +/// parent), the observation watermarks (keyed by parent + side chat), and +/// the loop-cap counters (keyed by parent). One mutex makes every mutation +/// below atomic against a concurrent record, take or acknowledgement — +/// and each persists the whole ledger under the lock. +#[derive(Debug, Default)] +struct NoticeState { + pending: HashMap>, + /// (parent, side chat) → the side chat's settle serial when the parent + /// last observed it (ticket 14): outcomes at or below the watermark owe + /// no notice, including a record that lands right after the observation. + acks: HashMap<(String, String), u64>, + /// parent → consecutive notice-started turns since the last + /// user-authored message (ticket 14's loop cap). + loops: HashMap, + /// parent → when its last Stop happened (epoch ms), for the re-arm + /// rule: a message the parent sent BEFORE that stop must not re-arm a + /// side chat's next turn (story 24 counts only later messages). Transient + /// by design — never persisted; a restart loses it and the gate opens, + /// which costs at most one notice the stop meant to suppress. + stopped_at: HashMap, + /// parent → a delivery of its notice turn is being built right now + /// (the single-flight guard): a second outcome racing it must not + /// steer into the turn the first is starting — it re-arms at that + /// turn's end instead. + delivering: std::collections::HashSet, +} + +/// One settle's mark: the serial it stamped and the moment it happened. +#[derive(Debug, Clone)] +struct SettleMark { + /// The serial this settle stamped. + serial: u64, + /// When the settle happened (epoch ms): the clock an observation's + /// "what did I see" timestamp compares against. + at: i64, +} + +/// Outcome records owed to parents, keyed by parent chat id. +#[derive(Debug, Default)] +pub(crate) struct CompletionNotices { + state: Mutex, + /// side chat → the turns settled so far (ticket 14): the monotonic + /// serial outcomes and observations compare on, with the moment each + /// settle happened so an acknowledgement can say WHICH settles its + /// observation covered (the serial it read, not the serial the engine + /// happens to be at when the report arrives — a settle racing between + /// the read and the report is the parent's UNSEEN outcome and must + /// survive the ack). In-memory by design — a restart's boot recovery + /// re-stamps serials for the outcomes it records, and a lost watermark + /// costs at most one redundant notice. Never taken together with + /// [`Self::state`] except by `ack`, which takes this one first (the + /// one ordering rule to keep deadlock-free). + settles: Mutex>>, + /// Where the ledger is mirrored (ticket 13): an outcome is a result the + /// parent has not received yet, and losing one means a child's work + /// silently never reaches its creator. `None` until a store path is + /// wired — bare `SessionsEngine` tests run in-memory, like the + /// journal's. + store: Mutex>, +} + +impl CompletionNotices { + pub(crate) fn record(&self, parent: &str, outcome: PendingOutcome) { + let mut state = lock(&self.state); + // The parent observed this settle already (ticket 14): the + // acknowledgement raced the record — most tightly through + // `wait_for_turn`, which resolves off the same status flip that + // triggered the record. Dropped, not delivered. + if state + .acks + .get(&(parent.to_string(), outcome.side_chat_id.clone())) + .is_some_and(|watermark| outcome.settle <= *watermark) + { + return; + } + state + .pending + .entry(parent.to_string()) + .or_default() + .push(outcome); + self.persist(&state); + } + + /// Take everything owed to `parent` for a delivery. `None` when the + /// parent is owed nothing. + pub(crate) fn take_for_delivery(&self, parent: &str) -> Option> { + let mut state = lock(&self.state); + let outcomes = state + .pending + .remove(parent) + .filter(|outcomes| !outcomes.is_empty()); + if outcomes.is_some() { + self.persist(&state); + } + outcomes + } + + /// A delivery that could not start a parent turn (busy again, no run + /// configuration, no doc): the outcomes stay owed, at the front of the + /// queue in their recorded order. + pub(crate) fn requeue(&self, parent: &str, outcomes: Vec) { + if outcomes.is_empty() { + return; + } + let mut state = lock(&self.state); + let owed = state.pending.entry(parent.to_string()).or_default(); + for outcome in outcomes.into_iter().rev() { + owed.insert(0, outcome); + } + self.persist(&state); + } + + /// Archived and deleted parents are owed nothing (ticket 13): the + /// pending records go, and the ledger forgets them. + pub(crate) fn drop_for(&self, parent: &str) { + let mut state = lock(&self.state); + if state.pending.remove(parent).is_some() { + self.persist(&state); + } + } + + /// The parents owed outcomes right now — the boot delivery's iteration. + pub(crate) fn parents(&self) -> Vec { + lock(&self.state).pending.keys().cloned().collect() + } + + /// Advance and return the chat's settle serial: one per turn that came + /// to rest (completed, errored, stopped, or parked awaiting input). + /// Called from the run loop at each settle, BEFORE the settled status + /// publishes, so anything that observes the chat through that status + /// sees a serial at least as high as everything it observed. + pub(crate) fn settle(&self, chat_id: &str) -> u64 { + let mut settles = lock(&self.settles); + let marks = settles.entry(chat_id.to_string()).or_default(); + let serial = marks.back().map(|mark| mark.serial).unwrap_or(0) + 1; + marks.push_back(SettleMark { + serial, + at: now_ms(), + }); + if marks.len() > SETTLE_HISTORY { + marks.pop_front(); + } + serial + } + + /// The chat's current settle serial — zero for a chat that never + /// settled in this engine run. + pub(crate) fn settle_serial(&self, chat_id: &str) -> u64 { + lock(&self.settles) + .get(chat_id) + .and_then(|marks| marks.back()) + .map(|mark| mark.serial) + .unwrap_or(0) + } + + /// The settle serial an observation made at `observed_at` covered: the + /// newest settle that happened at or before the observation. Settles + /// are monotonic, so an outcome for any settle past this watermark + /// happened after what the observer saw and owes its notice. Zero when + /// the observation predates every kept settle (a restarted engine, a + /// very old report): conservative — nothing is dropped. + fn serial_observed_at(&self, chat_id: &str, observed_at: i64) -> u64 { + lock(&self.settles) + .get(chat_id) + .and_then(|marks| { + marks + .iter() + .rev() + .find(|mark| mark.at <= observed_at) + .map(|mark| mark.serial) + }) + .unwrap_or(0) + } + + /// The parent observed the side chat after its last outcome happened + /// (ticket 14) — through `wait_for_turn`, `send_message` with wait, + /// `read_chat` or `get_chat`; the MCP server reports it, with the + /// moment the observation started (`observed_at`) when the caller + /// knows it. The watermark is the serial THAT observation covered, not + /// the serial the engine happens to be at when the report arrives — a + /// settle that landed between the read and this report is the parent's + /// UNSEEN outcome and survives the ack. Outcomes for settles at or + /// below the watermark are dropped (they were seen), and the watermark + /// stands so a record that lands right after this — the settle raced + /// the report — is dropped too. Outcomes for LATER turns owe their + /// notice. Returns the watermark for the log. + pub(crate) fn ack(&self, parent: &str, side_chat: &str, observed_at: Option) -> u64 { + let watermark = match observed_at { + Some(at) => self.serial_observed_at(side_chat, at), + None => self.settle_serial(side_chat), + }; + let mut state = lock(&self.state); + if let Some(owed) = state.pending.get_mut(parent) { + owed.retain(|outcome| { + !(outcome.side_chat_id == side_chat && outcome.settle <= watermark) + }); + if owed.is_empty() { + state.pending.remove(parent); + } + } + state + .acks + .insert((parent.to_string(), side_chat.to_string()), watermark); + self.persist(&state); + watermark + } + + /// Try to begin a delivery for `parent` (the single-flight guard): a + /// delivery already being built owns the parent's notice turn, so a + /// second one must not race it — the outcomes stay pending and the + /// parent's turn-end delivery picks them up instead. `false` when a + /// delivery is already in flight. + pub(crate) fn begin_delivery(&self, parent: &str) -> bool { + lock(&self.state).delivering.insert(parent.to_string()) + } + + /// Release the single-flight guard `begin_delivery` armed. + pub(crate) fn end_delivery(&self, parent: &str) { + lock(&self.state).delivering.remove(parent); + } + + /// A notice turn started for `parent` (ticket 14's loop cap): the + /// consecutive count of notice-started turns grows. Called from the + /// dispatch path whenever the starting message id carries the engine's + /// notice prefix — a delivered notice entry or a released parked row. + pub(crate) fn notice_turn_started(&self, parent: &str) { + let mut state = lock(&self.state); + *state.loops.entry(parent.to_string()).or_insert(0) += 1; + self.persist(&state); + } + + /// Whether `parent` hit the loop cap: the next notice must not start a + /// turn — it parks in the queue instead, until a user message arrives. + pub(crate) fn loop_cap_reached(&self, parent: &str) -> bool { + lock(&self.state) + .loops + .get(parent) + .copied() + .unwrap_or(0) + .ge(&NOTICE_LOOP_CAP) + } + + /// The parent's current consecutive notice-turn count — the cap's own + /// number, for logs and tests. + pub(crate) fn loop_turns(&self, parent: &str) -> u32 { + lock(&self.state).loops.get(parent).copied().unwrap_or(0) + } + + /// When `parent` was last stopped, if it was in this engine run. The + /// stopped-parent re-arm rule: only a message sent AFTER this moment + /// makes a side chat's next turn notifiable again (ticket 14). + pub(crate) fn parent_stopped_at(&self, parent: &str) -> Option { + lock(&self.state).stopped_at.get(parent).copied() + } + + /// Record `parent`'s stop moment (the stopped-parent rule's re-arm + /// gate): `stop_parent` calls this on every Stop. + pub(crate) fn note_stop(&self, parent: &str, at: i64) { + let mut state = lock(&self.state); + state.stopped_at.insert(parent.to_string(), at); + } + + /// A user-authored message reached `parent`: the streak of consecutive + /// notice-started turns is broken, so the cap resets (ticket 14). A + /// Stop counts as user intervention for the same reason — see + /// [`stop_parent`]. + pub(crate) fn note_user_message(&self, parent: &str) { + let mut state = lock(&self.state); + if state.loops.remove(parent).is_some() { + self.persist(&state); + } + } + + /// Wire the durable ledger and load what a previous engine run left: + /// outcomes recorded but never delivered survive the restart. A + /// missing file is a first boot (empty ledger, not a failure); an + /// unreadable or malformed one starts empty rather than failing boot — + /// the worst outcome is one undelivered notice, never a dead engine. + pub(crate) fn load(&self, path: &Path) { + { + let mut store = lock(&self.store); + *store = Some(path.to_path_buf()); + } + let bytes = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return, + Err(err) => { + tracing::warn!( + path = %path.display(), + error = %err, + "completion-notice ledger unreadable; starting empty" + ); + return; + } + }; + let malformed = |err: serde_json::Error| { + tracing::warn!( + path = %path.display(), + error = %err, + "completion-notice ledger malformed; starting empty" + ); + }; + let value: serde_json::Value = match serde_json::from_slice(&bytes) { + Ok(value) => value, + Err(err) => { + malformed(err); + return; + } + }; + // Ticket 14 wrapped the ledger as `{ pending, loops }`; a ticket-13 + // ledger was a bare map of parent → outcome rows. Both load — the + // legacy shape's loop counts are simply empty (a fresh cap). + let stored: StoredLedger = if value.get("pending").is_some() { + match serde_json::from_value(value) { + Ok(stored) => stored, + Err(err) => { + malformed(err); + return; + } + } + } else { + match serde_json::from_value::>>(value) { + Ok(pending) => StoredLedger { + pending, + loops: HashMap::new(), + }, + Err(err) => { + malformed(err); + return; + } + } + }; + let mut state = lock(&self.state); + for (parent, rows) in stored.pending { + state + .pending + .entry(parent) + .or_default() + .extend(rows.into_iter().map(PendingOutcome::from)); + } + // A fresh engine's counters are the ledger's: the loop cap must not + // reset just because the engine restarted (outcomes survive, so a + // runaway loop would otherwise get 25 more turns per boot). + state.loops.extend(stored.loops); + tracing::info!( + parents = state.pending.len(), + "completion-notice ledger loaded" + ); + } + + /// Mirror the notice state into the ledger. Called under the state + /// lock, so a mutation and its write are atomic against other + /// mutations — two outcomes racing cannot write snapshots that miss + /// each other. Synchronous and best-effort: the map is small (one row + /// per undelivered outcome plus one counter per parent), writes are + /// rare (a record, a take, a requeue, an acknowledgement, a counter + /// tick), and a lost write costs at most one notice. + fn persist(&self, state: &NoticeState) { + let Some(path) = lock(&self.store).clone() else { + return; + }; + let stored = StoredLedger { + pending: state + .pending + .iter() + .map(|(parent, outcomes)| { + ( + parent.clone(), + outcomes.iter().map(StoredOutcome::from).collect(), + ) + }) + .collect(), + loops: state.loops.clone(), + }; + let result = (|| -> Result<(), std::io::Error> { + let bytes = serde_json::to_vec(&stored) + .map_err(|err| std::io::Error::other(err.to_string()))?; + let temporary = path.with_extension("json.tmp"); + std::fs::write(&temporary, &bytes)?; + std::fs::rename(&temporary, &path)?; + Ok(()) + })(); + if let Err(err) = result { + tracing::warn!(error = %err, "completion-notice ledger write failed"); + } + } +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// First non-empty line, trimmed and capped with an ellipsis — the one-line +/// form a notice's detail owes the parent. `None` for blank text. +pub(crate) fn one_line(text: Option<&str>) -> Option { + let text = text?.trim(); + if text.is_empty() { + return None; + } + let line = text + .lines() + .find(|line| !line.trim().is_empty()) + .unwrap_or(text) + .trim_end(); + if line.chars().count() <= NOTICE_DETAIL_MAX { + return Some(line.to_owned()); + } + // Cap by characters, not bytes: multibyte text keeps the full budget. + let cut: String = line.chars().take(NOTICE_DETAIL_MAX).collect(); + Some(format!("{cut}…")) +} + +/// The one-line summary of the question or approval a side chat is waiting +/// on: the question's own text, its header when the text is blank. +pub(crate) fn question_summary(questions: &[roboco_proto::UserInputQuestion]) -> String { + let text = questions + .first() + .map(|question| { + if question.question.trim().is_empty() { + question.header.as_str() + } else { + question.question.as_str() + } + }) + .unwrap_or("input"); + one_line(Some(text)).unwrap_or_else(|| "input".into()) +} + +/// Is this chat quiet (ticket 13)? A row that says archived, and a row +/// that is GONE (deleted: the delete paths drop the parent's notices, but +/// a deletion that raced a record must not deliver to a chat that no +/// longer exists) — both are quiet. An UNREADABLE row reads as active: a +/// row the engine cannot read must not silently eat a report. +pub(crate) fn chat_archived(engine: &SessionsEngine, chat_id: &str) -> bool { + // Bind the host first: `workspace()` borrows it, and a + // `host().and_then(|host| host.workspace())` chain would borrow a + // closure-local (the same temporary-borrow trap the ticket-12 merge + // left behind in `notifiable_parent_for`). + let Some(host) = engine.host() else { + return false; + }; + match host.workspace().map(|ws| ws.chat(chat_id)) { + Some(Ok(Some(chat))) => chat.archived, + Some(Ok(None)) => true, // the chat is gone + _ => false, // the row is unreadable: keep talking, never eat a report + } +} + +/// Record an outcome for a side-chat turn and deliver what its parent is +/// owed. Called from the run loop, fire-and-forget: the side chat's own +/// event stream never waits on the parent's turn. +pub(crate) async fn record_and_deliver( + engine: &SessionsEngine, + side_chat: &str, + parent: &str, + settle: u64, + outcome: Outcome, +) { + // Archived chats are quiet both ways (ticket 13): an archived side chat + // sends nothing, an archived parent is owed nothing. Both read the row + // as it is now — archiving is a deliberate state change, not a flap. + if chat_archived(engine, side_chat) || chat_archived(engine, parent) { + return; + } + engine.notices().record( + parent, + PendingOutcome { + side_chat_id: side_chat.to_string(), + outcome, + settle, + }, + ); + deliver(engine, parent).await; +} + +/// The stopped-parent rule (ticket 14, T3's interrupt behaviour): the +/// pending notices `parent` is owed go — a parent the user stopped must +/// not restart on its own — and side chats whose notifiable turns are +/// running right now stop owing outcomes for them: the claim is cleared, +/// so the record sites (which read it at settle time) record nothing when +/// those turns end. The side chats themselves keep running — Stop does +/// not cascade — and a later message from the parent to one of them +/// re-arms it for that chat's next turn (`mark_notifiable` in +/// `sessions.rs`). A Stop is user intervention for the loop cap too, so +/// the notice-turn streak resets and the cap's parked rows go with the +/// rest of the pending notices. +pub(crate) fn stop_parent(engine: &SessionsEngine, parent: &str) { + engine.notices().note_stop(parent, now_ms()); + engine.notices().drop_for(parent); + engine.notices().note_user_message(parent); + engine.suppress_notifiable_runs(parent); + drop_parked_notices(engine, parent); +} + +/// Remove the loop cap's parked notice rows from the parent's queue (the +/// stopped-parent rule, ticket 14): a parked row IS a pending notice — the +/// visible form the cap left it waiting in — so a Stop drops it like any +/// other. The Stop's own queue pause keeps the drain away while the rows +/// go; a user's next message thaws an empty queue. +fn drop_parked_notices(engine: &SessionsEngine, parent: &str) { + let Some(host) = engine.host() else { + return; + }; + let Ok(handle) = host.open(parent) else { + return; + }; + let Ok(rows) = handle.doc().read_queue() else { + return; + }; + let parked: Vec = rows + .into_iter() + .filter(|row| row.id.starts_with(NOTICE_ID_PREFIX)) + .map(|row| row.id) + .collect(); + for id in parked { + let _ = handle.doc().remove_queued(&id); + } +} + +/// Deliver the parent's pending outcomes as one engine-authored notice that +/// starts a parent turn. Called when an outcome lands — an idle parent is +/// woken immediately — and whenever a parent's turn ends (ticket 13's +/// busy-parent rule: the notice was never steered mid-turn; it starts the +/// parent's next turn now). A parent that turns busy again while the +/// delivery is being built requeues its outcomes untouched, for the next +/// turn end. +pub(crate) async fn deliver(engine: &SessionsEngine, parent: &str) { + // Teardown never starts a parent's notice turn (the same rule that + // freezes the message queues): the outcomes stay pending — durably — + // and the next boot delivers them. + if engine.shutting_down() { + return; + } + // Single-flight (the delivery race): two settling children can both + // pass the idle check before either dispatches, and the second would + // steer into the notice turn the first is starting. One delivery owns + // the parent; the loser's outcomes stay pending — the turn-end + // delivery sites pick them up when the winner's turn ends. + if !engine.notices().begin_delivery(parent) { + return; + } + let _guard = DeliveryGuard(engine, parent); + if engine.turn_in_flight(parent) { + return; + } + // An archived or deleted parent receives nothing (ticket 13): drop what + // piled up — and the loop cap's parked rows with it, which are just the + // notice's waiting form — rather than holding it: an archived chat owes + // no wakeups, even after it is unarchived. + if chat_archived(engine, parent) { + engine.notices().drop_for(parent); + drop_parked_notices(engine, parent); + return; + } + let Some(outcomes) = engine.notices().take_for_delivery(parent) else { + return; + }; + let Some(host) = engine.host() else { + tracing::warn!(parent, "completion notice held: no doc host"); + engine.notices().requeue(parent, outcomes); + return; + }; + let sides: Vec = outcomes.iter().map(|o| side_row(&host, o)).collect(); + let text = notice_text(&sides); + // The loop cap (ticket 14): the 26th consecutive notice-started turn + // does not start. The notice parks as a visible, PAUSED queue row on + // the parent and waits for the user — a user-authored message resets + // the count and thaws the queue, and the row then runs as the parent's + // next turn (counting as notice-started itself, via its id prefix). + if engine.notices().loop_cap_reached(parent) { + park_notice(engine, parent, &host, &text, sides, outcomes); + return; + } + // The notice turn runs on the parent's own configuration: its chat row + // when there is one, else the last live request. `dispatch` re-derives + // the harness session and stamps the Roboco MCP server (so the parent + // can `read_chat` immediately). + let request = host.request_from_chat_row(parent, &text).or_else(|| { + engine.last_request(parent).map(|mut request| { + request.prompt = text.clone(); + request.resume = None; + request.attachments = Vec::new(); + request + }) + }); + let Some(request) = request else { + tracing::warn!( + parent, + "completion notice held: no run configuration for the chat" + ); + engine.notices().requeue(parent, outcomes); + return; + }; + let Ok(handle) = host.open(parent) else { + tracing::warn!(parent, "completion notice held: doc unavailable"); + engine.notices().requeue(parent, outcomes); + return; + }; + // A parent turn that started while this delivery was being built + // receives the notice after that turn instead (ticket 13): it is never + // steered mid-turn. The entry has not been written yet, so the outcomes + // go back to the front of the pending queue untouched — the run loop's + // turn-end delivery picks them up when that turn ends. + if engine.turn_in_flight(parent) { + engine.notices().requeue(parent, outcomes); + return; + } + let entry = notice_entry(host.device_id(), &sides, &text); + if let Err(err) = handle.doc().push_message(&entry) { + tracing::warn!(parent, error = %err, "completion notice entry write failed"); + engine.notices().requeue(parent, outcomes); + return; + } + let harness = host.harness_for_request(parent, &request); + if let Err(err) = engine + .dispatch(parent, harness, request, Some(entry.id.clone()), None, None) + .await + { + // The entry is already in the transcript (readable on the parent's + // next turn); only the wake failed. Do not requeue: a dispatch that + // fails once fails the same way twice. + tracing::warn!(parent, error = %err, "completion notice turn failed to start"); + } +} + +/// Ends the single-flight guard [`deliver`] armed, even on an early return +/// (a guard holding a mutex across the awaits below would block every +/// mutation instead of serialising the deliveries). +struct DeliveryGuard<'a>(&'a SessionsEngine, &'a str); + +impl Drop for DeliveryGuard<'_> { + fn drop(&mut self) { + self.0.notices().end_delivery(self.1); + } +} + +/// Park a capped notice as a visible, paused queue row on the parent (the +/// loop cap's waiting state, ticket 14). The queue pauses BEFORE the row +/// lands, so the doc-change drain finds it frozen and cannot send the row +/// straight out; a recovered queue starts paused too, so the row keeps +/// waiting across a restart. The row's id carries the notice prefix — the +/// turn it eventually starts still counts as notice-started — its text is +/// the notice itself, so the queue shows exactly what waits, and it carries +/// the notice's sides so its release writes the SAME System completion- +/// notice entry a direct delivery writes, never a user message. The +/// outcomes are consumed into the row (a durable carrier of its own: +/// queue rows live on the chat doc); only a row that cannot be written +/// puts them back into the pending set. Outcomes from a LATER capped +/// delivery MERGE into the parked row (ticket 13: outcomes arriving before +/// delivery merge into the same pending notice) instead of parking beside +/// it. +fn park_notice( + engine: &SessionsEngine, + parent: &str, + host: &DocHost, + text: &str, + sides: Vec, + outcomes: Vec, +) { + let parked_id = format!("{NOTICE_ID_PREFIX}{}", new_id()); + let row = QueuedMessage { + id: parked_id, + text: text.to_owned(), + attachments: Vec::new(), + hold_for_turn_end: false, + issued_by: host.device_id().to_owned(), + issued_at: now_ms(), + edited_at: None, + delivery_gate: None, + // Engine-authored, not user-typed: the id prefix (not the origin) + // carries that for the loop-cap bookkeeping. + origin_chat_id: None, + // The sides make the row the notice's own carrier: on release it + // writes the System completion-notice entry, not a user message. + completion_notice: Some(sides), + }; + let handle = host.open(parent).ok(); + let parked = handle.and_then(|handle| { + // Pause first: the row's own write wakes the drain, which must + // already see a frozen queue. + handle.pause_queue(); + // A row the cap already parked waits here: merge into it — one + // pending notice, not a row per capped delivery. Its id stays (the + // streak bookkeeping keys on the prefix, and the user's queue view + // keeps one row), its text and sides grow to cover every outcome. + let existing = handle + .doc() + .read_queue() + .ok() + .and_then(|rows| { + rows.into_iter().enumerate().find_map(|(ix, row)| { + (row.id.starts_with(NOTICE_ID_PREFIX)).then_some((ix, row)) + }) + }); + match existing { + Some((ix, mut old)) => { + let mut merged = old.completion_notice.clone().unwrap_or_default(); + merged.extend(row.completion_notice.clone().unwrap_or_default()); + old.text = notice_text(&merged); + old.completion_notice = Some(merged); + handle + .doc() + .remove_queued(&old.id) + .ok() + .and_then(|_| handle.doc().insert_queued(ix, &old).ok()) + .map(|_| old.id) + } + None => handle.doc().push_queued(&row).ok().map(|_| row.id), + } + }); + if parked.is_none() { + tracing::warn!( + parent, + "completion notice held: the capped notice row could not be parked" + ); + engine.notices().requeue(parent, outcomes); + return; + } + tracing::info!( + parent, + turns = engine.notices().loop_turns(parent), + "completion notice parked: notice loop cap reached" + ); +} + +fn side_row(host: &DocHost, outcome: &PendingOutcome) -> NoticeSide { + let title = host + .workspace() + .and_then(|ws| ws.chat(&outcome.side_chat_id).ok().flatten()) + .and_then(|chat| chat.title) + .map(|title| title.trim().to_owned()) + .filter(|title| !title.is_empty()) + .unwrap_or_else(|| "Side chat".into()); + let (recorded, detail) = outcome.outcome.as_row(); + NoticeSide { + chat_id: outcome.side_chat_id.clone(), + title, + outcome: recorded, + detail, + } +} + +/// The notice's readable text — also the parent turn's prompt. Names each +/// side chat by title and short id with its outcome, and tells the agent to +/// use `read_chat` for the full result. +pub(crate) fn notice_text(sides: &[NoticeSide]) -> String { + let mut lines: Vec = Vec::with_capacity(sides.len() + 2); + match sides { + [] => lines.push("Side chats reached outcomes.".into()), + [only] => lines.push(format!("Side chat {}.", side_line(only))), + many => { + lines.push(format!("{} side chats reached outcomes:", many.len())); + lines.extend(many.iter().map(|side| format!("- {}.", side_line(side)))); + } + } + lines.push(read_line(sides)); + lines.join("\n") +} + +/// The `read_chat` pointer — the notice is a pointer, never a report. +fn read_line(sides: &[NoticeSide]) -> String { + match sides { + [only] => format!( + "Read the full result with the Roboco `read_chat` tool, chat {}.", + short_id(&only.chat_id) + ), + _ => "Read each chat's full result with the Roboco `read_chat` tool.".into(), + } +} + +/// `"title" (short id) outcome` — the one-line form for one side chat. +fn side_line(side: &NoticeSide) -> String { + let verb = side.outcome.verb(); + let outcome = match side.outcome { + NoticeOutcome::Completed | NoticeOutcome::Stopped => verb.into(), + NoticeOutcome::Errored + | NoticeOutcome::Interrupted + | NoticeOutcome::WaitingForInput => with_detail(side, verb), + }; + format!("\"{}\" ({}) {}", side.title, short_id(&side.chat_id), outcome) +} + +/// `": "` when the row carries a one-liner, else `""` — +/// the shape every detail-bearing outcome shares. +fn with_detail(side: &NoticeSide, verb: &str) -> String { + match &side.detail { + Some(text) => format!("{verb}: {text}"), + None => verb.into(), + } +} + +/// Eight-char id prefix — the sidebar's convention. Ids are engine-minted +/// (ASCII) in practice, but a boundary-crossing multibyte id must not +/// panic: cut at the nearest char boundary at or before the eighth byte. +fn short_id(id: &str) -> &str { + let mut cut = id.len().min(8); + while cut > 0 && !id.is_char_boundary(cut) { + cut -= 1; + } + &id[..cut] +} + +/// The engine-authored notice message: a SYSTEM entry (never a user +/// message) carrying the structured marker clients render distinctly +/// (ticket 15), with the notice's text beside it as a plain part so older +/// layouts stay readable. Also the entry a released parked row writes +/// (its id prefixes `notice:`), so the two delivery paths land as the same +/// transcript shape. +pub(crate) fn notice_entry(device_id: &str, sides: &[NoticeSide], text: &str) -> SessionMessageEntry { + SessionMessageEntry { + id: format!("{NOTICE_ID_PREFIX}{}", new_id()), + role: MessageRole::System, + parts: vec![ + MessagePart::CompletionNotice { + id: "n0".into(), + sides: sides.to_vec(), + }, + MessagePart::Text { + id: "t0".into(), + text: text.to_owned(), + }, + ], + created_at: now_ms(), + device_id: device_id.to_owned(), + status: Some(MessageStatus::Complete), + continuation_of: None, + duration_ms: None, + origin_chat_id: None, + // A notice is an engine-authored message: no provider turn backs + // it (a fork of the parent falls back to the budgeted paste for + // it, exactly like the fork seam — ticket 17). + provider_turn: None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn side(chat_id: &str, title: &str, outcome: NoticeOutcome) -> NoticeSide { + NoticeSide { + chat_id: chat_id.into(), + title: title.into(), + outcome, + detail: None, + } + } + + /// A pending outcome for a turn settled before the engine booted + /// (the reloaded shape, settle 0). + fn pending(side_chat: &str, outcome: Outcome) -> PendingOutcome { + PendingOutcome { + side_chat_id: side_chat.into(), + outcome, + settle: 0, + } + } + + fn pending_at(side_chat: &str, settle: u64, outcome: Outcome) -> PendingOutcome { + PendingOutcome { + side_chat_id: side_chat.into(), + outcome, + settle, + } + } + + #[test] + fn one_line_takes_the_first_line_trimmed_and_capped() { + assert_eq!(one_line(None), None); + assert_eq!(one_line(Some(" \n\t")), None); + assert_eq!(one_line(Some(" provider 502 ")), Some("provider 502".into())); + // Multi-line errors keep only their first non-empty line. + assert_eq!( + one_line(Some("\nprovider 502: bad gateway\nretry later\n")), + Some("provider 502: bad gateway".into()) + ); + let long = "x".repeat(NOTICE_DETAIL_MAX + 40); + let capped = one_line(Some(&long)).unwrap(); + assert_eq!(capped.chars().count(), NOTICE_DETAIL_MAX + 1); + assert!(capped.ends_with('…')); + // Char-boundary safety on multibyte input. + let wide = "é".repeat(NOTICE_DETAIL_MAX + 5); + let capped = one_line(Some(&wide)).unwrap(); + assert_eq!(capped.chars().count(), NOTICE_DETAIL_MAX + 1); + } + + #[test] + fn short_id_cuts_at_a_char_boundary() { + // ASCII ids keep the full eight chars. + assert_eq!(short_id("chat-side-1234"), "chat-sid"); + assert_eq!(short_id("short"), "short"); + // A multibyte id never panics: the cut backs up to the nearest + // boundary (the review's byte-slice crash). + assert_eq!(short_id("é"), "é"); + let wide = "ééééééééé"; + let cut = short_id(wide); + assert!(cut.chars().count() <= 4, "{cut}"); + assert!(wide.starts_with(cut), "{cut}"); + assert_eq!(short_id(""), ""); + } + + #[test] + fn question_summary_prefers_the_question_then_the_header() { + use roboco_proto::UserInputQuestion; + let questions = vec![ + UserInputQuestion { + id: "q1".into(), + header: "Approval".into(), + question: " Should we bump the major version? ".into(), + options: vec!["Yes".into(), "No".into()], + multi_select: false, + }, + ]; + assert_eq!( + question_summary(&questions), + "Should we bump the major version?" + ); + let header_only = vec![UserInputQuestion { + id: "q1".into(), + header: "Approve shell access".into(), + question: String::new(), + options: vec![], + multi_select: false, + }]; + assert_eq!(question_summary(&header_only), "Approve shell access"); + assert_eq!(question_summary(&[]), "input"); + } + + #[test] + fn notice_text_names_title_short_id_outcome_and_read_chat() { + let mut completed = side("chat-side-1234", "Implement login", NoticeOutcome::Completed); + completed.detail = None; + let single = notice_text(&[completed.clone()]); + assert_eq!( + single, + "Side chat \"Implement login\" (chat-sid) finished.\n\ + Read the full result with the Roboco `read_chat` tool, chat chat-sid." + ); + + let mut errored = side("chat-side-5678", "Flaky test", NoticeOutcome::Errored); + errored.detail = Some("the build command timed out".into()); + let mut waiting = side("chat-side-90ab", "Review the API", NoticeOutcome::WaitingForInput); + waiting.detail = Some("Should we bump the major version?".into()); + let merged = notice_text(&[completed, errored, waiting]); + assert!(merged.starts_with("3 side chats reached outcomes:\n")); + assert!(merged.contains("- \"Implement login\" (chat-sid) finished.")); + assert!(merged.contains("- \"Flaky test\" (chat-sid) failed: the build command timed out.")); + assert!( + merged + .contains("- \"Review the API\" (chat-sid) is waiting for input: Should we bump the major version?") + ); + assert!(merged.contains("`read_chat`")); + // A stopped row carries no detail. + let stopped = side("chat-side-cdef", "Migration", NoticeOutcome::Stopped); + assert!(notice_text(&[stopped]).contains("\"Migration\" (chat-sid) was stopped.")); + // A restart-lost row says interrupted, with its one-line reason. + let mut interrupted = side("chat-side-cdef", "Migration", NoticeOutcome::Interrupted); + interrupted.detail = Some(INTERRUPTED_REASON.into()); + let text = notice_text(&[interrupted]); + assert!(text.contains("\"Migration\" (chat-sid) was interrupted: engine restart.")); + // Degenerate empty input still renders a pointer, never a panic. + assert!(notice_text(&[]).contains("read_chat")); + } + + #[test] + fn pending_outcomes_record_take_and_requeue() { + let notices = CompletionNotices::default(); + assert!(notices.take_for_delivery("parent").is_none()); + notices.record("parent", pending("chat-side-1", Outcome::Completed)); + notices.record("parent", pending("chat-side-2", Outcome::Stopped)); + let taken = notices.take_for_delivery("parent").unwrap(); + assert_eq!(taken.len(), 2); + assert_eq!(taken[0].side_chat_id, "chat-side-1"); + // Taken is taken: a second delivery owes nothing. + assert!(notices.take_for_delivery("parent").is_none()); + // A failed delivery puts them back at the front, in order. + notices.requeue("parent", taken); + let requeued = notices.take_for_delivery("parent").unwrap(); + assert_eq!(requeued[0].side_chat_id, "chat-side-1"); + assert_eq!(requeued[1].side_chat_id, "chat-side-2"); + } + + #[test] + fn drop_for_forgets_a_parent_and_parents_lists_the_owed() { + let notices = CompletionNotices::default(); + assert!(notices.parents().is_empty()); + notices.record("parent-a", pending("chat-side-1", Outcome::Completed)); + notices.record( + "parent-b", + pending( + "chat-side-2", + Outcome::Interrupted { + reason: Some(INTERRUPTED_REASON.into()), + }, + ), + ); + let mut owed = notices.parents(); + owed.sort(); + assert_eq!(owed, vec!["parent-a".to_string(), "parent-b".to_string()]); + notices.drop_for("parent-a"); + assert_eq!(notices.parents(), vec!["parent-b".to_string()]); + // Dropping a parent that is owed nothing stays a no-op. + notices.drop_for("parent-a"); + assert_eq!(notices.parents(), vec!["parent-b".to_string()]); + } + + #[test] + fn pending_outcomes_survive_a_store_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("completion-notices.json"); + + let first = CompletionNotices::default(); + // Before a store is wired, records are in-memory only. + first.record("parent", pending("chat-side-ephemeral", Outcome::Completed)); + // Wiring the ledger on a fresh engine: no file, no outcomes. + let second = CompletionNotices::default(); + second.load(&path); + assert!(second.take_for_delivery("parent").is_none()); + + second.record("parent", pending("chat-side-1234", Outcome::Completed)); + second.record( + "parent", + pending( + "chat-side-5678", + Outcome::Errored { + reason: Some("provider 502: bad gateway".into()), + }, + ), + ); + second.record( + "other-parent", + pending( + "chat-side-90ab", + Outcome::Interrupted { + reason: Some(INTERRUPTED_REASON.into()), + }, + ), + ); + // A requeue (a delivery that could not start) is as durable as a + // record: the next engine still owes the outcomes. + second.requeue( + "parent", + vec![pending( + "chat-side-cdef", + Outcome::WaitingForInput { + summary: "Should we bump the major version?".into(), + }, + )], + ); + + let reloaded = CompletionNotices::default(); + reloaded.load(&path); + let mut owed = reloaded.parents(); + owed.sort(); + assert_eq!( + owed, + vec!["other-parent".to_string(), "parent".to_string()] + ); + let mut parent = reloaded.take_for_delivery("parent").unwrap(); + assert_eq!(parent.len(), 3); + // The requeued outcome keeps its place at the front. + assert_eq!(parent.remove(0).side_chat_id, "chat-side-cdef"); + assert_eq!(parent[0].side_chat_id, "chat-side-1234"); + assert_eq!(parent[0].outcome, Outcome::Completed); + assert_eq!( + parent[1].outcome, + Outcome::Errored { + reason: Some("provider 502: bad gateway".into()) + } + ); + assert_eq!( + reloaded.take_for_delivery("other-parent").unwrap()[0].outcome, + Outcome::Interrupted { + reason: Some(INTERRUPTED_REASON.into()) + } + ); + // A ledger taken is a ledger written: re-delivery owes nothing. + let third = CompletionNotices::default(); + third.load(&path); + assert!(third.take_for_delivery("parent").is_none()); + assert!(third.take_for_delivery("other-parent").is_none()); + } + + #[test] + fn a_malformed_ledger_starts_empty_without_panicking() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("completion-notices.json"); + std::fs::write(&path, b"{not json").unwrap(); + let notices = CompletionNotices::default(); + notices.load(&path); + assert!(notices.parents().is_empty()); + // The wired ledger still accepts new records. + notices.record("parent", pending("chat-side-1", Outcome::Completed)); + assert_eq!(notices.parents(), vec!["parent".to_string()]); + } + + #[test] + fn outcome_rows_round_trip_through_every_kind() { + for (outcome, recorded, detail) in [ + (Outcome::Completed, NoticeOutcome::Completed, None), + ( + Outcome::Errored { + reason: Some("boom".into()), + }, + NoticeOutcome::Errored, + Some("boom".to_string()), + ), + (Outcome::Stopped, NoticeOutcome::Stopped, None), + ( + Outcome::Interrupted { + reason: Some(INTERRUPTED_REASON.into()), + }, + NoticeOutcome::Interrupted, + Some(INTERRUPTED_REASON.to_string()), + ), + ( + Outcome::WaitingForInput { + summary: "Proceed?".into(), + }, + NoticeOutcome::WaitingForInput, + Some("Proceed?".to_string()), + ), + ] { + let (row, row_detail) = outcome.as_row(); + assert_eq!(row, recorded); + assert_eq!(row_detail, detail); + assert_eq!(Outcome::from_row(row, row_detail), outcome); + } + // A stored row with no detail still rebuilds: errored keeps `None`, + // waiting falls back to the generic question summary. + assert_eq!( + Outcome::from_row(NoticeOutcome::Errored, None), + Outcome::Errored { reason: None } + ); + assert_eq!( + Outcome::from_row(NoticeOutcome::WaitingForInput, None), + Outcome::WaitingForInput { + summary: "input".into() + } + ); + } + + // ---- ticket 14: acknowledgement, stop, loop cap ----------------------- + + #[test] + fn an_acknowledgement_drops_observed_outcomes_and_covers_a_racing_record() { + let notices = CompletionNotices::default(); + // Turn 1 settles: the parent is busy, so the outcome waits. + let turn_one = notices.settle("chat-side-1"); + notices.record("parent", pending_at("chat-side-1", turn_one, Outcome::Completed)); + assert_eq!(notices.parents(), vec!["parent".to_string()]); + + // The parent observed the chat after the turn ended (wait_for_turn, + // read_chat, get_chat — all arrive as this call): the outcome goes. + let watermark = notices.ack("parent", "chat-side-1", None); + assert_eq!(watermark, turn_one); + assert!(notices.parents().is_empty()); + + // The report raced the record — the settle and the status flip it + // observed happened before the observation reached the engine, so a + // record for THAT settle lands after the ack: dropped too. + notices.record("parent", pending_at("chat-side-1", turn_one, Outcome::Completed)); + assert!(notices.parents().is_empty()); + + // A later turn is a different settle: it owes its notice. + let turn_two = notices.settle("chat-side-1"); + assert!(turn_two > turn_one); + notices.record("parent", pending_at("chat-side-1", turn_two, Outcome::Completed)); + let taken = notices.take_for_delivery("parent").unwrap(); + assert_eq!(taken.len(), 1); + assert_eq!(taken[0].settle, turn_two); + } + + #[test] + fn an_ack_covers_only_the_side_chat_it_observed() { + let notices = CompletionNotices::default(); + let settle = notices.settle("chat-side-a"); + notices.settle("chat-side-b"); + notices.record("parent", pending_at("chat-side-a", settle, Outcome::Completed)); + notices.record("parent", pending_at("chat-side-b", settle, Outcome::Completed)); + + // Observing one chat never silences another. + notices.ack("parent", "chat-side-a", None); + let owed = notices.take_for_delivery("parent").unwrap(); + assert_eq!(owed.len(), 1); + assert_eq!(owed[0].side_chat_id, "chat-side-b"); + // And only for the parent that observed it. + notices.record("other-parent", pending_at("chat-side-a", settle, Outcome::Completed)); + notices.ack("parent", "chat-side-a", None); + assert_eq!(notices.parents(), vec!["other-parent".to_string()]); + } + + #[test] + fn an_ack_of_an_unsettled_chat_reads_watermark_zero() { + let notices = CompletionNotices::default(); + // A side chat that never settled in this engine run: an outcome + // reloaded from a ledger (settle 0) is still covered — its turn + // settled before the restart, so a read after it has seen it. + assert_eq!(notices.ack("parent", "chat-side-1", None), 0); + notices.record("parent", pending("chat-side-1", Outcome::Completed)); + assert!(notices.parents().is_empty()); + // But a fresh settle outruns that watermark. + let turn = notices.settle("chat-side-1"); + notices.record("parent", pending_at("chat-side-1", turn, Outcome::Completed)); + assert_eq!(notices.parents(), vec!["parent".to_string()]); + } + + #[test] + fn an_ack_from_an_observation_covers_only_the_settles_it_saw() { + // The MCP server reports the moment its read started. A settle that + // lands between the read and the report is the parent's UNSEEN + // outcome: the watermark must not cover it, or an outcome the parent + // never saw is dropped silently (ticket 14). + let notices = CompletionNotices::default(); + let before = now_ms(); + let seen = notices.settle("chat-side-1"); + while now_ms() <= before { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + let read_at = now_ms(); + // The settle stamped after the read: the parent's observation never + // saw this turn. + while now_ms() <= read_at { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + let unseen = notices.settle("chat-side-1"); + notices.record("parent", pending_at("chat-side-1", seen, Outcome::Completed)); + notices.record("parent", pending_at("chat-side-1", unseen, Outcome::Completed)); + + // The report arrives last (the read happened at `read_at`): only the + // settle at or before the read is covered. + let watermark = notices.ack("parent", "chat-side-1", Some(read_at)); + assert_eq!(watermark, seen); + let owed = notices.take_for_delivery("parent").unwrap(); + assert_eq!(owed.len(), 1); + assert_eq!(owed[0].settle, unseen); + + // A report with no observation time keeps the old reading: the + // engine's current serial (the caller could not say what it saw). + let current = notices.settle_serial("chat-side-1"); + assert_eq!(notices.ack("other-parent", "chat-side-1", None), current); + } + + #[test] + fn settle_serials_advance_and_survive_many_settles() { + let notices = CompletionNotices::default(); + assert_eq!(notices.settle_serial("chat"), 0); + assert_eq!(notices.settle("chat"), 1); + assert_eq!(notices.settle("chat"), 2); + assert_eq!(notices.settle_serial("chat"), 2); + assert_eq!(notices.settle_serial("other"), 0); + } + + #[test] + fn the_loop_cap_counts_notice_turns_until_a_user_message() { + let notices = CompletionNotices::default(); + assert!(!notices.loop_cap_reached("parent")); + for turn in 1..=NOTICE_LOOP_CAP { + notices.notice_turn_started("parent"); + assert_eq!(notices.loop_turns("parent"), turn); + } + assert!(notices.loop_cap_reached("parent")); + // One more does not panic and stays capped. + notices.notice_turn_started("parent"); + assert_eq!(notices.loop_turns("parent"), NOTICE_LOOP_CAP + 1); + + // A user message resets the streak; the cap lifts. + notices.note_user_message("parent"); + assert_eq!(notices.loop_turns("parent"), 0); + assert!(!notices.loop_cap_reached("parent")); + // Resetting a chat with no streak is a no-op. + notices.note_user_message("other-parent"); + assert_eq!(notices.loop_turns("other-parent"), 0); + + // Another chat's counter is independent. + for _ in 0..NOTICE_LOOP_CAP { + notices.notice_turn_started("other-parent"); + } + assert!(notices.loop_cap_reached("other-parent")); + assert!(!notices.loop_cap_reached("parent")); + } + + #[test] + fn the_loop_counts_survive_a_restart_and_a_legacy_ledger_still_loads() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("completion-notices.json"); + + // Ticket 13's ledger: a bare parent → rows map, no counts. It loads, + // with an empty cap. + std::fs::write( + &path, + serde_json::json!({ + "chat-parent": [ + { "sideChatId": "chat-side-1", "outcome": "completed" } + ] + }) + .to_string(), + ) + .unwrap(); + let legacy = CompletionNotices::default(); + legacy.load(&path); + assert_eq!(legacy.parents(), vec!["chat-parent".to_string()]); + assert_eq!(legacy.loop_turns("chat-parent"), 0); + assert!(legacy.take_for_delivery("chat-parent").is_some()); + + // Ticket 14's ledger carries the counts alongside the pending set. + for _ in 0..NOTICE_LOOP_CAP { + legacy.notice_turn_started("chat-parent"); + } + let reloaded = CompletionNotices::default(); + reloaded.load(&path); + assert!(reloaded.loop_cap_reached("chat-parent")); + assert_eq!(reloaded.loop_turns("chat-parent"), NOTICE_LOOP_CAP); + // A user message still resets it after the reload. + reloaded.note_user_message("chat-parent"); + assert!(!reloaded.loop_cap_reached("chat-parent")); + // A count with no pending outcomes is not a parent. + assert!(reloaded.parents().is_empty()); + } +} diff --git a/crates/engine/src/doc_host.rs b/crates/engine/src/doc_host.rs index 9d18e6538b..cfd2d00573 100644 --- a/crates/engine/src/doc_host.rs +++ b/crates/engine/src/doc_host.rs @@ -393,6 +393,46 @@ impl ChatDocHandle { rx } + /// Hold queued rows until an explicit prompt thaws them (the same state + /// a Stop leaves): the sessions engine arms this when a turn ends on a + /// usage limit, so a Limited chat's queued messages wait behind the + /// continuation instead of dispatching into the same limit. A queued row + /// inserted afterwards also waits — nothing dispatches while paused. + pub fn pause_queue(&self) { + self.queue_paused.store(true, Ordering::Release); + } + + /// Release the hold [`Self::pause_queue`] armed: the explicit prompt — + /// the manual Resume's run, the user's own message, or the engine's + /// Resume-at-reset continuation — dispatched successfully, so the queued + /// rows may follow it (the drain still waits out the turn in flight). + pub fn thaw_queue(&self) { + self.queue_paused.store(false, Ordering::Release); + } + + /// Rewrite the full run requests held for this chat's turn end onto the + /// harness switch's config (ticket 24: "queued messages after a switch + /// go to the new harness"). A held send stores the request it was sent + /// with — model, reasoning, options and all — and the drain prefers it + /// over the chat row, so without this a message held before the switch + /// would run on the OLD harness (its model ids in the new harness's + /// vocabulary) after the switch. The switch's own resolution rules + /// apply: the new harness and its picks, the sandbox untouched. + pub fn rewrite_held_sends_for_switch(&self, config: &roboco_proto::ChatConfig) { + let mut sends = lock(&self.held_sends); + for request in sends.values_mut() { + request.harness = Some(config.harness); + request.model = config.model.clone(); + request.reasoning = config.reasoning; + request.model_options = config.model_options.clone(); + // The old harness's provider session and fork intent never ride + // the new harness's first run (the seam already cleared the + // chat-level ones; these are the held copies). + request.resume = None; + request.provider_fork = None; + } + } + /// Where a newly steered row goes: after the rows already steered, ahead /// of every ordinary row. Records `id` as steered. fn steer_slot(&self, id: &str) -> Result { @@ -435,11 +475,14 @@ impl ChatDocHandle { /// Write a complete user message entry, idempotent by id (the client-minted message /// id — a re-executed command or optimistic echo never duplicates the entry). + /// `origin_chat_id` is the structured attribution for messages sent by a + /// chat through the Roboco MCP server; `None` for typed messages. pub fn write_user_message( &self, message_id: &str, text: &str, created_at: i64, + origin_chat_id: Option<&str>, ) -> Result<(), DocError> { if self.doc.read_entries()?.iter().any(|e| e.id == message_id) { return Ok(()); @@ -455,7 +498,9 @@ impl ChatDocHandle { device_id: self.device_id.clone(), status: Some(MessageStatus::Complete), continuation_of: None, + provider_turn: None, duration_ms: None, + origin_chat_id: origin_chat_id.map(str::to_string), }) } @@ -1032,7 +1077,18 @@ impl DocHost { /// Drop a chat's doc unconditionally and delete its local snapshot — the /// chat is gone (DeleteChat / DeleteSpace cascade). Watchers see the /// stream end; a racing writer keeps its orphaned doc until the run ends. + /// Cancels the chat's pending Resume-at-reset continuation (ticket 07): + /// deleting the chat is the last word — locally, without re-mirroring a + /// sessions row that is going away with the chat. pub fn purge_chat(&self, chat_id: &str) { + if let Some(sessions) = self.sessions() { + sessions.cancel_scheduled_resume_with_chat(chat_id); + // A deleted parent is owed nothing (ticket 13): its pending + // completion notices go with the chat — the durable ledger + // forgets them, and a record that raced the deletion finds the + // row gone and stays quiet on delivery. + sessions.notices().drop_for(chat_id); + } let removed = lock(&self.inner.handles).remove(chat_id); drop(removed); if let Err(err) = self.inner.store.delete_snapshot(chat_id) { @@ -1048,7 +1104,7 @@ impl DocHost { chat_id: &str, payload: SessionCommandPayload, ) -> Result { - self.queue_command_with_transfers(chat_id, payload, Vec::new()) + self.queue_command_with_transfers(chat_id, payload, Vec::new(), None) } /// [`Self::queue_command`] plus queued-attachment transfers: the command's @@ -1056,12 +1112,16 @@ impl DocHost { /// dir; when another device hosts the chat, a background task pushes them /// over the peer link (retry until they land) while the command is /// already durably queued. The send is a local write — attachment bytes - /// chase it, never gate it (2026-08-19 incident). + /// chase it, never gate it (2026-08-19 incident). `origin_chat_id` is the + /// structured attribution when a chat sent this through the Roboco MCP + /// server (Run/Steer commands only); it survives the relay and retry + /// paths and is stamped on the user message entry the command writes. pub fn queue_command_with_transfers( &self, chat_id: &str, payload: SessionCommandPayload, _transfers: Vec, + origin_chat_id: Option<&str>, ) -> Result { let handle = self.open(chat_id)?; let id = new_id(); @@ -1083,6 +1143,7 @@ impl DocHost { expires_at: Some(now + COMMAND_DEFAULT_TTL_MS), status: SessionCommandStatus::Pending, resolution: None, + origin_chat_id: origin_chat_id.map(str::to_string), }; handle.doc.queue_command(&entry)?; // Sending a message revives an archived chat: the user is acting in it @@ -1090,6 +1151,14 @@ impl DocHost { // effort — the command itself is durable regardless. if is_message { self.unarchive_on_send(chat_id); + // And it cancels any pending Resume-at-reset continuation + // (orchestration v2, ticket 07): the newer intent wins. The + // engine's own continuations never queue commands — they + // dispatch directly — so this hook only ever sees user and + // agent messages. + if let Some(sessions) = self.sessions() { + sessions.cancel_scheduled_resume(chat_id); + } } Ok(id) } @@ -1119,18 +1188,21 @@ impl DocHost { text: &str, attachments: Vec, ) -> Result { - self.queue_message_with_behavior(chat_id, text, attachments, false) + self.queue_message_with_behavior(chat_id, text, attachments, false, None) } /// Append a message while preserving the submitter's active-turn policy /// on the synchronized row. This matters when another device hosts the /// chat: the host, not the submitting UI, decides when to drain it. + /// `origin_chat_id` is the structured attribution when a chat queued this + /// through the Roboco MCP server; the promoted user message entry keeps it. pub fn queue_message_with_behavior( &self, chat_id: &str, text: &str, attachments: Vec, hold_for_turn_end: bool, + origin_chat_id: Option<&str>, ) -> Result { let handle = self.open(chat_id)?; let id = new_id(); @@ -1143,10 +1215,17 @@ impl DocHost { issued_at: now_ms(), edited_at: None, delivery_gate: None, + origin_chat_id: origin_chat_id.map(str::to_string), + completion_notice: None, })?; handle.publish_queue(); // Same reasoning as a command: the user is acting in this chat again. self.unarchive_on_send(chat_id); + // Queuing is not sending (story 41): the row waits behind the armed + // continuation — the limit that parked the queue is still the wall + // the user chose to wait out, and the continuation stays armed to + // fire at its reset. Only a NEW direct message (the send paths + // below) cancels it. Ok(id) } @@ -1693,6 +1772,13 @@ impl DocHost { let Ok(Some(head)) = handle.doc.read_queue().map(|q| q.into_iter().next()) else { return; }; + // A pause armed while this drain sat between rows (a capped + // completion notice parks its row and freezes the queue in one + // step — ticket 14) holds the head too: nothing dispatches + // while paused, whenever the pause landed. + if handle.queue_paused.load(Ordering::Acquire) { + return; + } match &head.delivery_gate { Some(QueueDeliveryGate::Editing { lease_id, @@ -1884,7 +1970,12 @@ impl DocHost { continue; }; changed = true; - if let Err(err) = handle.write_user_message(&row.id, &row.text, row.issued_at) { + if let Err(err) = handle.write_user_message( + &row.id, + &row.text, + row.issued_at, + row.origin_chat_id.as_deref(), + ) { tracing::warn!(chat = %handle.chat_id, error = %err, "cancelled held send write failed"); } } @@ -1924,9 +2015,36 @@ impl DocHost { queued_message_prompt(&item.text, &attachments) }; self.resolve_attachment_refs(&mut prompt, &mut attachments); + // A parked notice row (the loop cap's waiting form, ticket 14) + // releases as the SAME System completion-notice entry a direct + // delivery writes — never a user message (notices are engine + // artifacts, not the user's words). The entry lands BEFORE the + // dispatch, so the dispatch's own user-message write dedupes onto + // it and the turn starts on the notice's text with the notice's id + // (the loop counter still reads it as notice-started). + if let Some(sides) = item.completion_notice.clone() { + let mut entry = crate::completion_notices::notice_entry( + self.inner.config.device_id.as_str(), + &sides, + &item.text, + ); + // The row's own id (already `notice:`-prefixed): the dispatch + // below writes under `message_id`, and only a shared id dedupes. + entry.id = message_id.clone(); + if let Err(err) = handle.doc().push_message(&entry) { + tracing::warn!(chat = %chat_id, error = %err, + "parked completion-notice entry write failed"); + } + } if send == QueueSend::Steer { match sessions - .steer(chat_id, &prompt, Some(message_id.clone())) + .steer_at( + chat_id, + &prompt, + Some(message_id.clone()), + item.issued_at.min(now_ms()), + item.origin_chat_id.clone(), + ) .await? { SteerOutcome::Accepted => return Ok(()), @@ -1969,8 +2087,16 @@ impl DocHost { request.resume = None; // dispatch re-derives the harness session request.attachments = attachments; let harness = self.harness_for_request(chat_id, &request); - self.dispatch_with_source_context(&sessions, chat_id, harness, request, Some(message_id)) - .await?; + self.dispatch_with_source_context( + &sessions, + chat_id, + harness, + request, + Some(message_id), + item.origin_chat_id.clone(), + Some(item.issued_at), + ) + .await?; Ok(()) } @@ -2069,6 +2195,9 @@ impl DocHost { expires_at: Some(now + COMMAND_DEFAULT_TTL_MS), status: SessionCommandStatus::Pending, resolution: None, + // The re-issue speaks for the same message: it keeps the + // origin of the send it re-sends. + origin_chat_id: old.origin_chat_id.clone(), }; tracing::info!(chat = %chat_id, old = %old.id, new = %reissue.id, "retry re-issues a dead send attempt"); @@ -2552,9 +2681,12 @@ impl DocHost { SessionCommandPayload::Steer { prompt, .. } => prompt.clone(), _ => String::new(), }; - if let Err(err) = - handle.write_user_message(&message_id, &prompt, entry.issued_at.min(now_ms())) - { + if let Err(err) = handle.write_user_message( + &message_id, + &prompt, + entry.issued_at.min(now_ms()), + entry.origin_chat_id.as_deref(), + ) { tracing::warn!(chat = %chat_id, error = %err, "cancelled message write failed"); } tracing::info!(chat = %chat_id, "prompt sent before a Stop that has acted: cancelled"); @@ -2649,6 +2781,7 @@ impl DocHost { entry.issued_at, false, Some(&request), + entry.origin_chat_id.clone(), )?; return Ok(( SessionCommandStatus::Applied, @@ -2665,6 +2798,7 @@ impl DocHost { message_id, &request.prompt, entry.issued_at.min(now_ms()), + entry.origin_chat_id.as_deref(), ) { tracing::warn!(chat = %chat_id, error = %err, "canonical user-message write failed"); } @@ -2674,12 +2808,22 @@ impl DocHost { harness, request, Some(message_id.clone()), + entry.origin_chat_id.clone(), + Some(entry.issued_at.min(now_ms())), ) .await?; // A fresh user-authored turn is the deliberate action that - // thaws a queue frozen by Cancel. Clear only after dispatch - // succeeds so a failed send cannot silently unfreeze it. - handle.queue_paused.store(false, Ordering::Release); + // thaws a queue frozen by Cancel — and the only kind that + // may release the loop cap's pause (ticket 14): an + // agent-origin Run (a side chat or an orchestrating parent + // sending through the Roboco MCP server) must not open the + // cap's parked notice without the user, or a busy child + // lifts the very brake that waits for one. Send now / Steer + // now (explicit user queue actions) and the engine's own + // resume continuations thaw through their own paths. + if entry.origin_chat_id.is_none() { + handle.queue_paused.store(false, Ordering::Release); + } Ok((SessionCommandStatus::Applied, None)) } SessionCommandPayload::Steer { prompt, message_id } => { @@ -2689,6 +2833,7 @@ impl DocHost { prompt, message_id.clone(), entry.issued_at, + entry.origin_chat_id.clone(), ) .await } @@ -2696,6 +2841,13 @@ impl DocHost { self.await_earlier_prompts(sessions, handle, &entry.id) .await; *lock(&handle.stopped_by) = Some(entry.id.clone()); + // The user's Stop drops the notices this chat is owed as a + // parent and suppresses outcomes from side chats whose + // notifiable turns are running now (ticket 14). Stopping an + // idle chat still drops them — a stopped parent must not + // restart on its own — which `interrupt_and_pause_queue` + // would skip when there is nothing to stop. + crate::completion_notices::stop_parent(sessions, &handle.chat_id); // Sends before this Stop that are still on their way (a slow // dispatch the wait above gave up on) are cancelled with it. if let Ok(commands) = handle.doc.read_commands() @@ -2779,8 +2931,16 @@ impl DocHost { "orphaned input resolve failed"); } let harness = self.harness_for_request(chat_id, &request); - self.dispatch_with_source_context(sessions, chat_id, harness, request, None) - .await?; + self.dispatch_with_source_context( + sessions, + chat_id, + harness, + request, + None, + None, + None, + ) + .await?; Ok(( SessionCommandStatus::Applied, Some("answered as new turn".into()), @@ -2805,12 +2965,19 @@ impl DocHost { prompt: &str, message_id: Option, issued_at: i64, + origin_chat_id: Option, ) -> Result<(SessionCommandStatus, Option), EngineError> { let chat_id = &handle.chat_id; // Keep upstream's queued-attachment rewrite and send-time canonical // history while preserving the personal cut's turn-boundary queue. let prompt = self.resolve_prompt_attachments(prompt); - if self.held_until_turn_end(sessions, chat_id, &prompt, message_id.as_deref())? { + if self.held_until_turn_end( + sessions, + chat_id, + &prompt, + message_id.as_deref(), + origin_chat_id.clone(), + )? { return Ok(( SessionCommandStatus::Applied, Some("held until the turn ends".into()), @@ -2821,13 +2988,23 @@ impl DocHost { // transcript copy would duplicate it until the update finishes. if !sessions.live_run_update_pending(chat_id) && let Some(message_id) = message_id.as_deref() - && let Err(err) = - handle.write_user_message(message_id, &prompt, issued_at.min(now_ms())) + && let Err(err) = handle.write_user_message( + message_id, + &prompt, + issued_at.min(now_ms()), + origin_chat_id.as_deref(), + ) { tracing::warn!(chat = %chat_id, error = %err, "canonical user-message write failed"); } match sessions - .steer_at(chat_id, &prompt, message_id.clone(), issued_at) + .steer_at( + chat_id, + &prompt, + message_id.clone(), + issued_at, + origin_chat_id.clone(), + ) .await? { SteerOutcome::Accepted => { @@ -2836,7 +3013,12 @@ impl DocHost { } SteerOutcome::NotSteerable => { if let Some(message_id) = message_id.as_deref() { - handle.write_user_message(message_id, &prompt, issued_at.min(now_ms()))?; + handle.write_user_message( + message_id, + &prompt, + issued_at.min(now_ms()), + origin_chat_id.as_deref(), + )?; } let request = sessions .last_request(chat_id) @@ -2853,8 +3035,16 @@ impl DocHost { // images; this prompt's own refs (if any) ride its text. request.attachments = Vec::new(); let harness = self.harness_for_request(chat_id, &request); - self.dispatch_with_source_context(sessions, chat_id, harness, request, message_id) - .await?; + self.dispatch_with_source_context( + sessions, + chat_id, + harness, + request, + message_id, + origin_chat_id, + issued_at.min(now_ms()).into(), + ) + .await?; handle.queue_paused.store(false, Ordering::Release); Ok(( SessionCommandStatus::Applied, @@ -2863,7 +3053,7 @@ impl DocHost { } SteerOutcome::DeferredByUpdate => { let id = message_id.unwrap_or_else(new_id); - self.hold_until_update_ends(handle, &id, &prompt, issued_at)?; + self.hold_until_update_ends(handle, &id, &prompt, issued_at, origin_chat_id)?; // The completed turn's status publication normally re-drains // this queue. Also cover completion racing the enqueue itself. self.drain_queue(handle).await; @@ -2893,6 +3083,7 @@ impl DocHost { chat_id: &str, prompt: &str, message_id: Option<&str>, + origin_chat_id: Option, ) -> Result { // A persistent session parked BETWEEN turns DOES take the next prompt // through its mailbox (the warm-child fast path), so the question is @@ -2921,6 +3112,7 @@ impl DocHost { now_ms(), true, None, + origin_chat_id, )?; Ok(true) } @@ -2937,6 +3129,7 @@ impl DocHost { issued_at: i64, steer: bool, request: Option<&roboco_proto::RunRequest>, + origin_chat_id: Option, ) -> Result<(), EngineError> { let item = QueuedMessage { id: message_id.to_string(), @@ -2947,6 +3140,8 @@ impl DocHost { issued_at: issued_at.min(now_ms()), edited_at: None, delivery_gate: None, + origin_chat_id, + completion_notice: None, }; if handle.doc.read_queue()?.iter().any(|row| row.id == item.id) { return Ok(()); // a redelivered command: already held @@ -2976,8 +3171,9 @@ impl DocHost { id: &str, prompt: &str, issued_at: i64, + origin_chat_id: Option, ) -> Result<(), EngineError> { - self.hold_until_turn_end(handle, id, prompt, issued_at, true, None) + self.hold_until_turn_end(handle, id, prompt, issued_at, true, None, origin_chat_id) } async fn capture_source_context(&self, cwd: &str) -> Option { @@ -3001,6 +3197,7 @@ impl DocHost { /// dispatch so the conversation records the checkout the harness will /// actually observe, rather than whichever checkout state was current on /// an earlier turn. + #[allow(clippy::too_many_arguments)] pub(crate) async fn dispatch_with_source_context( &self, sessions: &SessionsEngine, @@ -3008,6 +3205,8 @@ impl DocHost { harness: HarnessId, request: roboco_proto::RunRequest, message_id: Option, + origin_chat_id: Option, + sent_at: Option, ) -> Result { if let Some(workspace) = self.workspace() && let Some(context) = self.capture_source_context(&request.cwd).await @@ -3016,7 +3215,7 @@ impl DocHost { tracing::warn!(chat = %chat_id, error = %err, "conversation source stamp failed"); } sessions - .dispatch(chat_id, harness, request, message_id) + .dispatch(chat_id, harness, request, message_id, origin_chat_id, sent_at) .await } @@ -3171,6 +3370,8 @@ impl DocHost { auto_approve: false, attachments: Vec::new(), resume: None, + provider_fork: None, + resume_fallback_history: None, worktree: None, mcp: None, }) @@ -3271,6 +3472,7 @@ mod transcript_revisit_tests { for i in 0..2000 { source .push_message(&roboco_doc::SessionMessageEntry { + origin_chat_id: None, id: format!("row-{i}"), role: roboco_doc::MessageRole::User, parts: vec![roboco_doc::MessagePart::Text { @@ -3281,6 +3483,7 @@ mod transcript_revisit_tests { device_id: "remote".into(), status: None, continuation_of: None, + provider_turn: None, duration_ms: None, }) .unwrap(); @@ -3308,7 +3511,7 @@ mod transcript_revisit_tests { let (_dir, host) = host(); let handle = host.open("cached").unwrap(); handle - .write_user_message("row", "locally persisted transcript", 0) + .write_user_message("row", "locally persisted transcript", 0, None) .unwrap(); let rx = handle.watch_messages(); assert_eq!(rx.borrow().entries.len(), 1); diff --git a/crates/engine/src/engine_settings.rs b/crates/engine/src/engine_settings.rs new file mode 100644 index 0000000000..780a986cda --- /dev/null +++ b/crates/engine/src/engine_settings.rs @@ -0,0 +1,179 @@ +//! Engine settings (orchestration v2): preferences the ENGINE itself acts +//! on — set from any client (Settings → General on desktop), persisted under +//! the engine data dir so they survive restarts and apply with the UI +//! closed. Per-device like the harness prefs, not synced. +//! +//! First setting: **Auto-resume limited chats** (ticket 07, default off) — +//! arm the Resume-at-reset continuation automatically whenever a chat +//! becomes Limited, so long work continues overnight without a click per +//! chat. Off by default: an engine that starts spending turns on its own is +//! opt-in. + +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use serde::{Deserialize, Serialize}; + +/// The engine's user-facing settings. Additive only (ADR 0005): every field +/// carries `#[serde(default)]` so settings files from older engines — and +/// older engines reading newer files — keep loading. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, ts_rs::TS)] +#[serde(default, rename_all = "camelCase")] +pub struct EngineSettings { + /// Arm **Resume at reset** automatically whenever a chat becomes + /// Limited (orchestration v2, ticket 07). Off by default. + pub auto_resume_limited: bool, +} + +/// The settings plus where they persist. `path = None` (bare engines, +/// tests) keeps everything in memory. One instance lives on the sessions +/// engine's `Inner`. +#[derive(Debug, Default)] +pub(crate) struct EngineSettingsStore { + settings: Mutex, + path: Mutex>, +} + +impl EngineSettingsStore { + /// The persisted store at `path`, loaded. Test-only now: engine + /// assembly wires the shared store in place with [`Self::attach`], so + /// production never opens a second copy. The default (no path) serves + /// bare engines and tests. + #[cfg(test)] + pub(crate) fn open(path: Option) -> Self { + let settings = path.as_deref().and_then(load_file).unwrap_or_default(); + Self { + settings: Mutex::new(settings), + path: Mutex::new(path), + } + } + + /// Point an existing store at `path` and load what is persisted there + /// (engine assembly; the store lives on the shared sessions engine, so + /// it is wired in place rather than replaced). + pub(crate) fn attach(&self, path: PathBuf) { + let settings = load_file(&path).unwrap_or_default(); + *lock(&self.settings) = settings; + *lock(&self.path) = Some(path); + } + + pub(crate) fn get(&self) -> EngineSettings { + *lock(&self.settings) + } + + /// Update in memory and on disk. Returns the persistence error (the + /// in-memory value still updated — a failed write never wedges the + /// toggle, and the next successful write heals the file). The written + /// snapshot is taken under the lock with the assignment: two racing + /// `set`s otherwise persist out of order and the file ends up behind + /// the in-memory value. + pub(crate) fn set(&self, settings: EngineSettings) -> std::io::Result<()> { + let snapshot = { + let mut current = lock(&self.settings); + *current = settings; + current.clone() + }; + let path = lock(&self.path).clone(); + match path { + Some(path) => write_file(&path, &snapshot), + None => Ok(()), + } + } +} + +fn load_file(path: &Path) -> Option { + let bytes = std::fs::read(path).ok()?; + match serde_json::from_slice::(&bytes) { + Ok(settings) => Some(settings), + Err(err) => { + // A broken file must never refuse the engine over one toggle. + tracing::warn!(path = %path.display(), error = %err, "engine settings unreadable"); + None + } + } +} + +/// Write atomically (temp file + rename): a crash mid-write must never +/// leave a truncated `engine-settings.json` that boots as defaults — the +/// Auto-resume setting is durable intent. +fn write_file(path: &Path, settings: &EngineSettings) -> std::io::Result<()> { + let bytes = serde_json::to_vec_pretty(settings) + .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?; + write_file_atomic(path, &bytes) +} + +/// Temp-file-then-rename, with the temp file in the target's directory so +/// the rename stays on one filesystem. +fn write_file_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + let dir = path.parent().unwrap_or(Path::new(".")); + let mut staged = tempfile::NamedTempFile::new_in(dir)?; + staged.write_all(bytes)?; + staged.flush()?; + staged.as_file().sync_all()?; + staged.persist(path).map(drop).map_err(|err| err.error) +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_path() -> PathBuf { + tempfile::tempdir().unwrap().keep().join("engine-settings.json") + } + + #[test] + fn defaults_are_off_and_round_trip_through_disk() { + let path = temp_path(); + let store = EngineSettingsStore::open(Some(path.clone())); + assert_eq!(store.get(), EngineSettings::default()); + assert!(!store.get().auto_resume_limited); + + store + .set(EngineSettings { + auto_resume_limited: true, + }) + .unwrap(); + let reloaded = EngineSettingsStore::open(Some(path.clone())); + assert!(reloaded.get().auto_resume_limited); + + // Back off again. + reloaded.set(EngineSettings::default()).unwrap(); + assert!(!EngineSettingsStore::open(Some(path)).get().auto_resume_limited); + } + + #[test] + fn missing_and_broken_files_default_off_and_old_settings_gain_defaults() { + // Missing. + let missing = EngineSettingsStore::open(Some(temp_path())); + assert_eq!(missing.get(), EngineSettings::default()); + // Broken. + let broken = temp_path(); + std::fs::write(&broken, b"{not json").unwrap(); + assert_eq!(EngineSettingsStore::open(Some(broken)).get(), EngineSettings::default()); + // A file from an engine with fewer settings still loads. + let sparse = temp_path(); + std::fs::write(&sparse, b"{}").unwrap(); + assert_eq!(EngineSettingsStore::open(Some(sparse)).get(), EngineSettings::default()); + // A file from a newer engine keeps its known fields. + let future = temp_path(); + std::fs::write(&future, br#"{"autoResumeLimited":true,"someFutureField":9}"#).unwrap(); + assert!(EngineSettingsStore::open(Some(future)).get().auto_resume_limited); + } + + #[test] + fn a_failed_write_never_wedges_the_in_memory_toggle() { + // A path whose parent does not exist cannot be written. + let store = EngineSettingsStore::open(Some(PathBuf::from("/nonexistent/engine-settings.json"))); + assert!(store + .set(EngineSettings { + auto_resume_limited: true, + }) + .is_err()); + assert!(store.get().auto_resume_limited); + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 241c82a562..a5d9a10ba5 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -15,11 +15,14 @@ use roboco_rpc::{RpcError, RpcReply, RpcService, methods}; use roboco_sync::DocsStore; +pub mod account_failover; pub mod agent_accounts; pub mod change_requests; mod chat_persistence; +pub mod completion_notices; pub mod diff_sync; pub mod doc_host; +pub mod engine_settings; mod fs_watch; pub mod harness_updates; mod http_error; @@ -34,6 +37,7 @@ pub mod registry; pub mod repos; pub mod rpc; pub mod run_journal; +pub mod scheduled_resumes; pub mod sessions; pub mod sidebar_state; pub mod source_control; @@ -43,6 +47,7 @@ pub mod tailcat; pub mod tailcat_relay; pub mod terminals; pub mod titles; +pub mod transcript_budget; mod transcript_history; pub mod uploads; mod web; @@ -57,6 +62,7 @@ pub use diff_sync::{ discard_working_tree, merge_base, read_diff_file_text, snapshot_tree, working_diff_base, }; pub use doc_host::{ChatDocHandle, DocHost, DocHostConfig}; +pub use engine_settings::EngineSettings; pub use instance_lock::InstanceLock; pub use profile::EngineProfile; pub use project_actions::ProjectActionsStore; @@ -229,11 +235,28 @@ impl EngineCore { doc_host.set_workspace(workspace.clone()); doc_host.set_sessions(sessions.clone()); sessions.set_doc_host(doc_host.clone()); + // Pending completion notices are undelivered outcomes + // (orchestration v2, ticket 13): load the previous engine run's + // ledger BEFORE boot recovery, so the interrupted outcomes + // recovery records for runs the restart lost land in the same + // ledger — then deliver everything once recovery has settled the + // chats. + sessions.load_pending_notices(&profile.store_root().join("completion-notices.json")); + // Engine settings (auto-resume limited chats, …) and the durable + // Resume-at-reset store both live under the engine data dir; the + // continuations' timers arm after `recover_stale` (firing needs the + // wired doc host) — orchestration v2, ticket 07. + sessions.set_engine_settings_path(data_dir.join("engine-settings.json")); + sessions.set_scheduled_resumes_path(data_dir.join("scheduled-resumes.json")); match sessions.recover_stale() { Ok(0) => {} Ok(recovered) => tracing::info!(recovered, "stale sessions recovered on boot"), Err(err) => tracing::error!(error = %err, "stale-session recovery failed"), } + sessions.deliver_pending_notices(); + // Overdue continuations fire after boot; the rest sleep until their + // reset time. Staggered with the crashed-run resumes above. + sessions.recover_scheduled_resumes(); doc_host.spawn_transcript_salvage(profile.store_root().join("journals")); let repos = Repos::new(data_dir, &device_id); doc_host.set_repos(repos.clone()); @@ -268,6 +291,12 @@ impl EngineCore { // login between engines, so no route is registered in production. let agent_accounts = AgentAccounts::with_callback_routes(agent_accounts_config, previews.callback_routes()); + // The sessions engine reads the accounts store: usage-limit records + // take the active login's cached windows (which account the limit + // hit, reset-time fill — ticket 06), and runs that fail signed-out + // mark the harness's live login on the same store the accounts RPCs + // list (ticket 08). + sessions.set_agent_accounts(agent_accounts.clone()); let harness_updates = harness_updates::HarnessUpdateCoordinator::new(data_dir, registry.clone()); harness_updates.start(); diff --git a/crates/engine/src/registry.rs b/crates/engine/src/registry.rs index 78cbf8fe9a..cdce7b3a9d 100644 --- a/crates/engine/src/registry.rs +++ b/crates/engine/src/registry.rs @@ -726,15 +726,17 @@ pub fn default_registry() -> HarnessRegistry { ); // pi over its NATIVE RPC mode (`pi --mode rpc`, the maintainer-supported // embedding path), same lazy pattern: the static descriptor mirrors - // PiHarness exactly — turn-boundary steering via pi's own steer queue, - // pi's thinking ladder minus its "off" tier, deterministic turn ends - // (agent_settled). + // PiHarness exactly — STEP-boundary steering via pi's own steer queue + // (prompt + steer behavior, mode "all" at session start), pi's thinking + // ladder minus its "off" tier, deterministic turn ends (agent_settled). + // Orchestration v2 ticket 03 flipped this from turn-boundary; the + // harness declaration and this descriptor must always agree. registry.register_lazy( HarnessDescriptor { id: HarnessId::Pi, name: "Pi".into(), supports_steering: true, - steering_mode: SteeringMode::TurnBoundary, + steering_mode: SteeringMode::StepBoundary, reasoning_levels: vec![ ReasoningLevel::Minimal, ReasoningLevel::Low, @@ -991,7 +993,7 @@ mod tests { let pi = registry.resolve(HarnessId::Pi).unwrap(); assert_eq!(pi.id(), HarnessId::Pi); assert_eq!(pi.display_name(), "Pi"); - assert_eq!(pi.steering_mode(), SteeringMode::TurnBoundary); + assert_eq!(pi.steering_mode(), SteeringMode::StepBoundary); assert_eq!( pi.reasoning_levels(), &[ @@ -1005,6 +1007,44 @@ mod tests { ); } + /// The static descriptor a lazy slot carries must mirror the harness it + /// resolves to — the descriptor-stability rule, checked for every slot + /// at once. A disagreement on `steering_mode` is exactly the bug + /// orchestration v2 ticket 03 guards against: the engine routes + /// mid-turn sends (queue holds, row actions, MCP `send_message`) off the + /// DESCRIPTOR while the driver speaks for itself, so the two must flip + /// together. (`installed`/`can_install`/`enabled` differ by design — + /// they are per-device, re-probed on `descriptors()`.) + #[test] + fn registry_descriptors_mirror_their_harness_declarations() { + let registry = default_registry(); + let descriptors = registry.descriptors(); + assert!(!descriptors.is_empty()); + for descriptor in descriptors { + let harness = registry + .resolve(descriptor.id) + .expect("every default slot resolves (construction never probes a CLI)"); + let described = describe(harness.as_ref()); + assert_eq!(descriptor.id, described.id, "slot identity"); + assert_eq!( + descriptor.steering_mode, described.steering_mode, + "{:?}: the descriptor and the harness declaration disagree on the steering mode", + descriptor.id + ); + assert_eq!( + descriptor.supports_steering, described.supports_steering, + "{:?}: steering support disagrees", + descriptor.id + ); + assert_eq!( + descriptor.reasoning_levels, described.reasoning_levels, + "{:?}: the reasoning ladder disagrees", + descriptor.id + ); + assert_eq!(descriptor.name, described.name, "{:?}: the display name disagrees", descriptor.id); + } + } + /// Catalogs serialized by engines that predate the `installed`/`enabled` /// fields must keep deserializing — installed, and enabled per the /// detection fallback. diff --git a/crates/engine/src/rpc.rs b/crates/engine/src/rpc.rs index 7c5021b167..bc14d79e11 100644 --- a/crates/engine/src/rpc.rs +++ b/crates/engine/src/rpc.rs @@ -237,6 +237,11 @@ where struct QueueCommandParams { chat_id: String, command: SessionCommandPayload, + /// The chat this Run/Steer is sent BY through the Roboco MCP server — + /// structured attribution (additive). Absent for user-typed sends; the + /// command entry and its user message entry keep it. + #[serde(default)] + origin_chat_id: Option, /// Queued attachments (bytes already committed locally as `pending://` /// refs) the engine delivers to a remote host AFTER the command is /// durably queued — never as a gate in front of it. @@ -264,6 +269,28 @@ struct QueueMessageParams { /// supports mid-turn steering. #[serde(default)] hold_for_turn_end: bool, + /// The chat this row is sent BY through the Roboco MCP server — + /// structured attribution (additive). Absent for user-typed rows. + #[serde(default)] + origin_chat_id: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct AckSideChatParams { + /// The chat that was observed — the side chat. + chat_id: String, + /// The chat that observed it: the Roboco MCP server's calling chat + /// (an agent parent). Reported after `wait_for_turn`, a send with + /// wait, `read_chat` or `get_chat` (ticket 14). + origin_chat_id: String, + /// When the observation started (epoch ms; additive — absent when the + /// caller could not say): the engine's acknowledgement watermark covers + /// only the side chat's settles that had happened by then, so an + /// outcome that landed between the read and this report — the parent's + /// UNSEEN work — still owes its notice. + #[serde(default)] + observed_at: Option, } #[derive(Debug, Deserialize)] @@ -547,6 +574,16 @@ struct AgentAccountParams { account_id: String, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ResumeOnAccountParams { + chat_id: String, + /// The account the user picked; `None` takes the engine's + /// recommendation (the best-ranked candidate). + #[serde(default)] + account_id: Option, +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct StartAgentLoginParams { @@ -645,6 +682,13 @@ pub enum MutateParams { #[serde(default)] #[ts(optional)] parent_chat_id: Option, + /// Set when an AGENT is creating this one through the Roboco MCP + /// `create_chat` / `create_chats` tools: the creating chat and its + /// notify preference (default on). Absent on UI creations — forks + /// and blank side chats never notify. + #[serde(default)] + #[ts(optional)] + agent_origin: Option, }, /// Create a space (device + folder pair). Idempotent by id; a live /// duplicate `(deviceId, path)` no-ops. `gitDetected` is seeded from the @@ -720,6 +764,16 @@ pub enum MutateParams { #[ts(optional)] at: Option, }, + /// **Resume at reset** (orchestration v2, ticket 07): arm the durable + /// continuation for a Limited chat — at the limit's reset time the + /// engine sends the continue prompt like the manual Resume. Requires + /// the chat to be Limited with a reset time still ahead. + #[serde(rename_all = "camelCase")] + ScheduleResumeAtReset { chat_id: String }, + /// Cancel the chat's pending Resume-at-reset continuation (the banner's + /// Cancel action). No-op when nothing is armed. + #[serde(rename_all = "camelCase")] + CancelScheduledResume { chat_id: String }, } pub struct EngineRpc { @@ -997,6 +1051,7 @@ impl EngineRpc { branch, cwd, parent_chat_id, + agent_origin, } => { self.workspace .create_chat_with_parent( @@ -1006,6 +1061,7 @@ impl EngineRpc { config, cwd, parent_chat_id, + agent_origin, ) .map_err(failed)?; if let Some(branch) = branch.as_deref().filter(|b| !b.is_empty()) { @@ -1077,11 +1133,29 @@ impl EngineRpc { .set_chat_host(&chat_id, &device_id) .map_err(failed) .map(drop), - MutateParams::SetChatArchived { chat_id, archived } => self - .workspace - .set_chat_archived(&chat_id, archived) - .map_err(failed) - .map(drop), + MutateParams::SetChatArchived { chat_id, archived } => { + let result = self + .workspace + .set_chat_archived(&chat_id, archived) + .map_err(failed); + // Archiving a Limited chat cancels its pending Resume-at-reset + // continuation (orchestration v2, ticket 07): the user put the + // work away — it must not resume on its own overnight. Only + // on a SUCCESSFUL archive: a refused write (the registry row + // unreadable, the doc busy) leaves the chat exactly as it + // was, continuation armed — cancelling beside a failed + // write would strand an unarchived chat with no continuation. + if archived && result.is_ok() { + self.sessions.cancel_scheduled_resume(&chat_id); + // An archived parent is owed nothing (ticket 13): its + // pending completion notices go now, and the loop cap's + // parked rows — the notice's waiting form — go with them, + // or they would sit in the archived chat's queue forever + // (nothing drains an archived parent). + crate::completion_notices::stop_parent(&self.sessions, &chat_id); + } + result.map(drop) + } MutateParams::SetChatConfig { chat_id, config } => self .workspace .set_chat_config(&chat_id, &config) @@ -1106,6 +1180,16 @@ impl EngineRpc { .map_err(failed) .map(drop) } + MutateParams::ScheduleResumeAtReset { chat_id } => { + self.sessions + .schedule_resume_at_reset(&chat_id) + .map_err(failed) + .map(drop) + } + MutateParams::CancelScheduledResume { chat_id } => { + self.sessions.cancel_scheduled_resume(&chat_id); + Ok(()) + } } } } @@ -1371,6 +1455,14 @@ impl RpcService for EngineRpc { .map_err(RpcError::Failed)?; RpcReply::value(&self.registry.title_settings()) } + methods::GET_ENGINE_SETTINGS => RpcReply::value(&self.sessions.engine_settings()), + methods::SET_ENGINE_SETTINGS => { + let p: crate::engine_settings::EngineSettings = parse_params(params)?; + self.sessions + .set_engine_settings(p) + .map_err(|e| RpcError::Failed(e.to_string()))?; + RpcReply::value(&self.sessions.engine_settings()) + } methods::SET_HARNESS_ENABLED => { let p: SetHarnessEnabledParams = parse_params(params)?; update_harness_enabled(&self.registry, p.harness, p.enabled).await?; @@ -1462,7 +1554,12 @@ impl RpcService for EngineRpc { let p: QueueCommandParams = parse_params(params)?; let command_id = self .doc_host - .queue_command_with_transfers(&p.chat_id, p.command, p.transfers) + .queue_command_with_transfers( + &p.chat_id, + p.command, + p.transfers, + p.origin_chat_id.as_deref(), + ) .map_err(|e| RpcError::Failed(e.to_string()))?; RpcReply::value(&serde_json::json!({ "commandId": command_id })) } @@ -1498,6 +1595,39 @@ impl RpcService for EngineRpc { /// side chat's parent so the copy lists as a sibling. #[serde(default)] parent_chat_id: Option, + /// The assistant entry to fork at (t3-orchestration-v2 + /// ticket 21, "Fork from here"): the copy stops at that + /// reply, inclusive, and its provider turn ref becomes + /// the native fork boundary. Any SETTLED assistant entry + /// qualifies — an earlier one, an errored one, an + /// interrupted one. Absent (the classic fork button) + /// keeps today's behaviour: the latest complete reply. + #[serde(default)] + boundary: Option, + /// Harness override (t3-orchestration-v2 ticket 22, "fork + /// into another harness"): the copy runs on this harness + /// instead of the source's, its wire name ("codex", + /// "claude-code", …). A pick of the source's own harness + /// — or no pick at all — keeps the native fork behaviour. + #[serde(default)] + harness: Option, + /// Model override (ticket 22): rides the copy's run + /// config. Without a harness change it replaces only the + /// model; into another harness the old model ids are + /// meaningless, so it (or the new harness's default at + /// run time) is all the copy gets. + #[serde(default)] + model: Option, + /// Agent-made fork (t3-orchestration-v2 ticket 23, the + /// `fork_chat` MCP tool): the chat whose agent forked + /// this one and whether it wants completion notices + /// (default on). The assignment REPLACES whatever the + /// source row carried, so a user-made fork — the UI, the + /// web client, a terminal — is never marked and never + /// notifies, even when the source itself was + /// agent-created (ticket 11's rule). + #[serde(default)] + agent_origin: Option, } let p: ForkParams = parse_params(params)?; let parent_chat_id = p @@ -1527,17 +1657,87 @@ impl RpcService for EngineRpc { .doc() .read_entries() .map_err(|e| RpcError::Failed(e.to_string()))?; - let boundary = entries - .iter() - .rposition(|entry| { - entry.role == roboco_doc::MessageRole::Assistant - && entry.status == Some(roboco_doc::MessageStatus::Complete) - }) - .ok_or_else(|| { - RpcError::Failed( - "Wait for a completed response before starting a side chat".into(), + // The reply the copy stops at, inclusive. An explicit + // boundary is any settled assistant entry — never a + // streaming one (the run is still writing it, and its turn + // ref is not stamped yet); the default stays the latest + // COMPLETE assistant reply, exactly as before this change. + let chosen = match p + .boundary + .as_deref() + .map(str::trim) + .filter(|id| !id.is_empty()) + { + Some(id) => entries + .iter() + .position(|entry| entry.id == id) + .filter(|&ix| { + entries[ix].role == roboco_doc::MessageRole::Assistant + && entries[ix].status != Some(roboco_doc::MessageStatus::Streaming) + }) + .ok_or_else(|| { + RpcError::Failed( + "Fork boundary must be a settled assistant reply".into(), + ) + })?, + None => entries + .iter() + .rposition(|entry| { + entry.role == roboco_doc::MessageRole::Assistant + && entry.status == Some(roboco_doc::MessageStatus::Complete) + }) + .ok_or_else(|| { + RpcError::Failed( + "Wait for a completed response before starting a side chat".into(), + ) + })?, + }; + // A logical message can span continuation fragments after + // its root (schema capability for oversized replies): the + // fork copies the WHOLE message the chosen entry belongs + // to, and the native boundary ref comes from its root — + // where a turn ref is stamped. + let boundary = logical_message_end(&entries, chosen); + let ref_entry = logical_message_root(&entries, chosen); + // Ticket 22's overrides: a chosen harness (and model). A + // trimmed empty string counts as absent — the web client and + // the MCP tool hand through whatever the caller typed. An + // unknown harness name is refused rather than silently + // ignored: the fork would run on the default harness while + // the user believes it runs on the one they picked. + let requested_harness = p + .harness + .as_deref() + .map(str::trim) + .filter(|harness| !harness.is_empty()); + let requested_model = p + .model + .as_deref() + .map(str::trim) + .filter(|model| !model.is_empty()); + let override_harness = match requested_harness { + None => None, + Some(name) => Some( + serde_json::from_value::( + serde_json::Value::String(name.to_owned()), ) - })?; + .map_err(|_| RpcError::Failed(format!("Unknown harness: {name}")))?, + ), + }; + // The harness the source's conversation lives on: the + // boundary entry's provider turn ref names it (the + // native-fork contract), else the row's config. An override + // onto any other harness is a CROSS-harness fork: native + // fork only works within one harness, so the copy never + // records the intent and its first send carries the budgeted + // history instead (ticket 01's builder). + let source_harness = entries[ref_entry] + .provider_turn + .as_ref() + .map(|reference| reference.harness) + .or_else(|| source.config.as_ref().map(|config| config.harness)); + let cross_harness = + override_harness.is_some_and(|harness| source_harness != Some(harness)); let mut chat = source.clone(); chat.id = p.chat_id; chat.parent_chat_id = Some(parent_chat_id); @@ -1550,6 +1750,62 @@ impl RpcService for EngineRpc { chat.harness_session_id = None; chat.harness_session_cwd = None; chat.room_gen = Some(2); + // Ticket 23: an agent-made fork records its creator and + // notify flag — the durable row the engine's completion + // notices key on. The plain assignment also strips any + // origin the source row carried, so a fork nobody marked + // for the agent stays user-made and never notifies. + chat.agent_origin = p.agent_origin; + // Ticket 22's override on the row's run config. A + // cross-harness fork keeps only harness-agnostic settings: + // the sandbox. Model, reasoning and model options are ids in + // the OLD harness's vocabulary — the override's model (or + // the new harness's own default at run time) is all the copy + // gets, mirroring the harness-switch rule. A same-harness + // pick (or a model-only override) replaces exactly the + // model and inherits the rest, like the composer's own + // model change. A chat that never carried a config is + // pinned to the harness it actually runs on (the engine + // default) so a configless source stays honest. + // + // The same drop applies when the copy's harness differs + // from the SOURCE'S CURRENT CONFIG — not just from the + // boundary's turn-ref harness `cross_harness` judges + // against: forking an older Claude reply of a chat that has + // since switched to Codex, back onto Claude, inherits + // Codex's model ids on a Claude copy. The copy's harness + // versus the config it inherits is the mismatch that + // matters; the sandbox always survives. + if override_harness.is_some() || requested_model.is_some() { + let base = chat.config.clone().unwrap_or(roboco_proto::ChatConfig { + harness: self.doc_host.harness_for(&p.source_chat_id), + model: None, + reasoning: None, + model_options: Default::default(), + sandbox: roboco_proto::SandboxLevel::WorkspaceWrite, + }); + let copy_harness = override_harness.unwrap_or(base.harness); + let config_cross = chat + .config + .as_ref() + .is_some_and(|config| config.harness != copy_harness); + let drop_vocab = cross_harness || config_cross; + chat.config = Some(roboco_proto::ChatConfig { + harness: copy_harness, + model: if drop_vocab { + requested_model.map(str::to_owned) + } else { + requested_model.map(str::to_owned).or(base.model) + }, + reasoning: if drop_vocab { None } else { base.reasoning }, + model_options: if drop_vocab { + Default::default() + } else { + base.model_options + }, + sandbox: base.sandbox, + }); + } // Missing rows open on chat2. Persist history before publishing // the registry row so a crash cannot leave a discoverable empty fork. let target = self.doc_host.open(&chat.id).map_err(failed)?; @@ -1587,7 +1843,9 @@ impl RpcService for EngineRpc { target .doc() .push_message(&roboco_doc::SessionMessageEntry { + origin_chat_id: None, duration_ms: None, + provider_turn: None, id: marker_id.clone(), role: roboco_doc::MessageRole::System, parts: vec![roboco_doc::MessagePart::Fork { @@ -1602,10 +1860,87 @@ impl RpcService for EngineRpc { }) .map_err(|e| RpcError::Failed(e.to_string()))?; } + // Native fork intent (t3-orchestration-v2 tickets 17+21): + // when the boundary entry carries a provider turn ref, the + // fork records the source provider session and the boundary + // on the new chat — its first run asks the harness to fork + // the source session at that turn (Codex `thread/fork + // {threadId, lastTurnId}`) instead of resuming it, so the + // copied history rides the provider and nothing is pasted. + // The boundary is whatever reply the caller chose — the + // latest by default, an earlier/errored/interrupted one + // explicitly — so every harness's driver (17–20) forks at + // exactly that turn. A CROSS-harness fork (ticket 22) never + // records the intent: native fork only works within one + // harness, so the copy takes the budgeted-paste path. + // No ref — old chats, or a harness that records none — + // keeps today's behavior: a fresh session plus the budgeted + // paste, cut at the boundary because the fork's own doc only + // holds the copied prefix. + // + // A boundary that is NOT the last segment of its turn gets + // the paste too (story 60): a steer-split reply's segments + // share the turn's ref, and the native fork cuts at the + // WHOLE turn — content the boundary excluded would ride the + // copy through the provider. Only the turn's last segment + // names a boundary the harness can actually cut at. + if !cross_harness + && let Some(reference) = entries[ref_entry].provider_turn.clone() + && !turn_continues_past(&entries, boundary, &reference) + { + target + .doc() + .set_provider_fork( + reference.harness, + &roboco_proto::ProviderFork { + session_id: reference.session_id, + last_turn: reference.turn_id, + fallback_history: None, + }, + ) + .map_err(|e| RpcError::Failed(e.to_string()))?; + } self.doc_host.persist_fork(&target).map_err(failed)?; self.workspace.import_chat_row(&chat).map_err(failed)?; RpcReply::value(&chat) } + methods::SWITCH_HARNESS => { + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct SwitchParams { + chat_id: String, + harness: HarnessId, + #[serde(default)] + model: Option, + #[serde(default)] + reasoning: Option, + #[serde(default)] + model_options: serde_json::Map, + } + let p: SwitchParams = parse_params(params)?; + let failed = |e: crate::EngineError| RpcError::Failed(e.to_string()); + // Sandbox is the one field the switch does not own: whatever + // the chat already runs stays (the picker never offered it). + let sandbox = self + .workspace + .chat(&p.chat_id) + .map_err(failed)? + .and_then(|chat| chat.config) + .map(|config| config.sandbox) + .unwrap_or(roboco_proto::SandboxLevel::WorkspaceWrite); + let config = ChatConfig { + harness: p.harness, + model: p.model, + reasoning: p.reasoning, + model_options: p.model_options, + sandbox, + }; + self.sessions + .switch_harness(&p.chat_id, config) + .await + .map_err(failed)?; + RpcReply::value(&serde_json::json!({})) + } methods::RELAY_COMMAND => { let p: RelayCommandParams = parse_params(params)?; let outcome = self @@ -1697,10 +2032,21 @@ impl RpcService for EngineRpc { &p.text, p.attachments, p.hold_for_turn_end, + p.origin_chat_id.as_deref(), ) .map_err(|e| RpcError::Failed(e.to_string()))?; RpcReply::value(&serde_json::json!({ "id": id })) } + methods::ACK_SIDE_CHAT => { + // An agent parent observed a chat after its last outcome + // happened (ticket 14, orchestration v2). Engine-local: + // the Roboco MCP server dials this engine's loopback, and + // both chats live here. + let p: AckSideChatParams = parse_params(params)?; + self.sessions + .ack_notice(&p.origin_chat_id, &p.chat_id, p.observed_at); + RpcReply::value(&serde_json::json!({})) + } methods::UPDATE_QUEUED_MESSAGE => { let p: QueuedMessageParams = parse_params(params)?; let changed = self @@ -2928,6 +3274,24 @@ impl RpcService for EngineRpc { .map_err(|e| RpcError::Failed(e.to_string()))?; RpcReply::value(&snapshot) } + methods::LIST_RESUME_ACCOUNT_CANDIDATES => { + let p: ChatParams = parse_params(params)?; + let candidates = self + .sessions + .resume_account_candidates(&p.chat_id) + .await + .map_err(|e| RpcError::Failed(e.to_string()))?; + RpcReply::value(&candidates) + } + methods::RESUME_ON_ANOTHER_ACCOUNT => { + let p: ResumeOnAccountParams = parse_params(params)?; + let outcome = self + .sessions + .resume_on_another_account(&p.chat_id, p.account_id.as_deref()) + .await + .map_err(|e| RpcError::Failed(e.to_string()))?; + RpcReply::value(&outcome) + } methods::FORGET_AGENT_ACCOUNT => { let p: AgentAccountParams = parse_params(params)?; let snapshot = self @@ -3029,6 +3393,63 @@ impl RpcService for EngineRpc { } } +/// The index of the ROOT entry of the logical message `at` belongs to: an +/// oversized relayed reply can split into continuation fragments +/// (`root#c1`, …) after its root, and the whole chain is one message +/// (`join_continuation_entries`). Fragments name their ROOT (`root#c2` +/// points at `root`, not at `root#c1`), so the walk resolves the named id +/// directly rather than stepping back one entry at a time — a three-plus +/// fragment chain would otherwise stop at the first fragment and lose the +/// root (and the root's provider turn ref, where a fork boundary is +/// stamped). An orphan fragment (root missing, or named forward) is its +/// own root; the walk can only move backwards, so a corrupt cycle cannot +/// spin it. +fn logical_message_root(entries: &[roboco_doc::SessionMessageEntry], at: usize) -> usize { + let mut root = at; + while let Some(parent) = entries[root].continuation_of.as_deref() { + match entries + .iter() + .position(|entry| entry.id.as_str() == parent) + { + Some(prev) if prev < root => root = prev, + _ => return root, + } + } + root +} + +/// The inclusive index of the LAST entry of the logical message `at` +/// belongs to — the end a fork copies through, so a split message never +/// copies as a truncated prefix of itself. +fn logical_message_end(entries: &[roboco_doc::SessionMessageEntry], at: usize) -> usize { + let root = logical_message_root(entries, at); + let root_id = entries[root].id.as_str(); + let mut end = root; + while entries + .get(end + 1) + .is_some_and(|next| next.continuation_of.as_deref() == Some(root_id)) + { + end += 1; + } + end.max(at) +} + +/// Whether the provider turn the boundary entry belongs to CONTINUES past +/// the copy's end (story 60): a steer-split reply's later segments carry +/// the same turn ref, and a native fork cuts at the whole turn — so a +/// boundary that is not its turn's last segment must fall back to the +/// budgeted paste (later entries the boundary excluded would otherwise +/// reach the copy through the provider session). +fn turn_continues_past( + entries: &[roboco_doc::SessionMessageEntry], + boundary: usize, + reference: &roboco_proto::ProviderTurnRef, +) -> bool { + entries[boundary.saturating_add(1)..] + .iter() + .any(|entry| entry.provider_turn.as_ref() == Some(reference)) +} + #[cfg(test)] mod tests { use super::*; @@ -3308,6 +3729,7 @@ mod tests { handle .doc() .push_message(&roboco_doc::SessionMessageEntry { + origin_chat_id: None, id: "turn".into(), role: roboco_doc::MessageRole::Assistant, parts: (0..500) @@ -3321,6 +3743,7 @@ mod tests { status: None, continuation_of: None, duration_ms: None, + provider_turn: None, }) .unwrap(); // Hold publication blocked: the opening must not await the full mirror. @@ -3350,14 +3773,14 @@ mod tests { assert_eq!(preview["reset"][0]["parts"][0]["id"], "part-372"); // Changes between preview and subscribe must appear in the full reset. handle - .write_user_message("arrived", "new local message", 1) + .write_user_message("arrived", "new local message", 1, None) .unwrap(); let full = stream.next().await.unwrap(); assert!(full.get("historyPending").is_none()); assert_eq!(full["reset"][0]["parts"].as_array().unwrap().len(), 500); assert_eq!(full["reset"][1]["id"], "arrived"); handle - .write_user_message("live", "after attach", 2) + .write_user_message("live", "after attach", 2, None) .unwrap(); let live = tokio::time::timeout(std::time::Duration::from_secs(2), stream.next()) .await @@ -3505,6 +3928,7 @@ mod context_usage_tests { let append = |id: &str| { source .push_message(&roboco_doc::SessionMessageEntry { + origin_chat_id: None, id: id.into(), role: roboco_doc::MessageRole::Assistant, parts: vec![roboco_doc::MessagePart::Text { @@ -3516,6 +3940,7 @@ mod context_usage_tests { status: Some(roboco_doc::MessageStatus::Streaming), continuation_of: None, duration_ms: None, + provider_turn: None, }) .unwrap() }; @@ -3638,6 +4063,7 @@ mod context_usage_tests { .unwrap(); }); let entry = |id: &str| roboco_doc::SessionMessageEntry { + origin_chat_id: None, id: id.into(), role: roboco_doc::MessageRole::Assistant, parts: vec![roboco_doc::MessagePart::Text { @@ -3649,6 +4075,7 @@ mod context_usage_tests { status: Some(roboco_doc::MessageStatus::Streaming), continuation_of: None, duration_ms: None, + provider_turn: None, }; handle.doc().push_message(&entry("local-before")).unwrap(); source.update_context_usage(Some(10), Some(100)).unwrap(); diff --git a/crates/engine/src/run_journal.rs b/crates/engine/src/run_journal.rs index 3a9b89cad4..ef19487db9 100644 --- a/crates/engine/src/run_journal.rs +++ b/crates/engine/src/run_journal.rs @@ -96,6 +96,50 @@ impl RunJournal { let _ = std::fs::remove_file(self.attempts_path(chat_id)); } + /// Durable record that the harness CONFIRMED consuming a steer (the + /// `Steered` boundary that retires its at-least-once ledger entry). + /// Boot auto-resume reads it so a chat revived after an engine restart + /// re-delivers only input its harness had not yet consumed: a steer + /// replayed as the revival prompt is a message the model answers twice + /// (orchestration v2, ticket 04 — restart correctness). + /// + /// One message id per line, bounded by the current turn (a cleanly + /// completed turn clears it, like the resume budget). A torn trailing + /// write can only ever read as some OTHER id, never a full match, so it + /// is tolerated like a torn journal line. + pub fn note_steer_consumed(&self, chat_id: &str, message_id: &str) { + let path = self.steers_path(chat_id); + let result = OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .and_then(|mut file| file.write_all(format!("{message_id}\n").as_bytes())); + if let Err(err) = result { + tracing::warn!(chat = %chat_id, error = %err, "consumed-steer record write failed"); + } + } + + /// Whether the harness had confirmed consuming `message_id` as a steer + /// before it was recorded — the read a boot auto-resume consults to tell + /// a consumed steer (never re-delivered) from an accepted-but-undelivered + /// one (still delivered after the resume). + pub fn steer_consumed(&self, chat_id: &str, message_id: &str) -> bool { + std::fs::read_to_string(self.steers_path(chat_id)) + .map(|lines| lines.lines().any(|line| line.trim() == message_id)) + .unwrap_or(false) + } + + /// A cleanly completed turn closes the consumed-steer record: everything + /// the harness consumed before it is settled history no revival ever + /// re-examines, so the file stays bounded by one turn's steers. + pub fn clear_consumed_steers(&self, chat_id: &str) { + let _ = std::fs::remove_file(self.steers_path(chat_id)); + } + + fn steers_path(&self, chat_id: &str) -> PathBuf { + self.dir.join(format!("{}.steers", sanitize_id(chat_id))) + } + /// Append one event; returns its journal seq. pub fn append(&self, chat_id: &str, event: &AgentEvent) -> Result { let mut files = self.lock(); @@ -206,6 +250,9 @@ impl RunJournal { if path.exists() { std::fs::remove_file(path)?; } + // The chat's own recovery sidecars go with its journal. + let _ = std::fs::remove_file(self.attempts_path(chat_id)); + let _ = std::fs::remove_file(self.steers_path(chat_id)); Ok(()) } } @@ -385,9 +432,11 @@ impl Iterator for RevEvents { } } -/// Journal (`.jsonl`) and resume-budget (`.resume`) paths for `chat_id` under an -/// arbitrary journals directory — profile import copies these files between -/// profiles without opening a `RunJournal`. +/// Journal (`.jsonl`) and resume-budget (`.resume`) paths for `chat_id` under +/// an arbitrary journals directory. Only those two: the consumed-steer record +/// (`.steers`, restart correctness) is a separate file that this helper does +/// NOT name — copy it alongside these when moving a profile, or an imported +/// journal replays a steer its harness had already consumed. pub fn journal_paths(dir: &Path, chat_id: &str) -> (PathBuf, PathBuf) { let stem = sanitize_id(chat_id); ( @@ -597,6 +646,50 @@ mod tests { assert_eq!(scan_tail(&missing).unwrap(), (1, false)); } + #[test] + fn consumed_steer_records_survive_and_clear_per_turn() { + let dir = tempfile::tempdir().unwrap(); + let journal = RunJournal::open(dir.path()).unwrap(); + assert!(!journal.steer_consumed("chat-1", "m-1"), "no record yet"); + journal.note_steer_consumed("chat-1", "m-1"); + journal.note_steer_consumed("chat-1", "m-2"); + assert!(journal.steer_consumed("chat-1", "m-1")); + assert!(journal.steer_consumed("chat-1", "m-2")); + assert!(!journal.steer_consumed("chat-1", "m-3")); + // Per-chat files, like the journal itself. + assert!(!journal.steer_consumed("chat-2", "m-1")); + // A cleanly completed turn closes the record (bounded by one turn). + journal.clear_consumed_steers("chat-1"); + assert!(!journal.steer_consumed("chat-1", "m-1")); + assert!(!journal.steer_consumed("chat-1", "m-2")); + // The record survives a reopen — the boot auto-resume read. + journal.note_steer_consumed("chat-1", "m-1"); + drop(journal); + let journal = RunJournal::open(dir.path()).unwrap(); + assert!(journal.steer_consumed("chat-1", "m-1")); + } + + #[test] + fn a_torn_consumed_steer_line_never_reads_as_a_match() { + let dir = tempfile::tempdir().unwrap(); + let journal = RunJournal::open(dir.path()).unwrap(); + journal.note_steer_consumed("chat-1", "m-full-id"); + // Crash mid-write of a second record: a truncated id is not the id — + // only a fully written id (even without its newline) can match. + let mut file = OpenOptions::new() + .append(true) + .open(dir.path().join("chat-1.steers")) + .unwrap(); + file.write_all(b"m-full-id-trunc").unwrap(); + drop(file); + let journal = RunJournal::open(dir.path()).unwrap(); + assert!(journal.steer_consumed("chat-1", "m-full-id")); + // The torn bytes are a prefix of the id being written: that id + // never matches. + assert!(!journal.steer_consumed("chat-1", "m-full-id-truncated")); + assert!(!journal.steer_consumed("chat-1", "m-full")); + } + #[test] fn stale_sessions_unchanged_by_tail_reads() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/engine/src/scheduled_resumes.rs b/crates/engine/src/scheduled_resumes.rs new file mode 100644 index 0000000000..a982c9d496 --- /dev/null +++ b/crates/engine/src/scheduled_resumes.rs @@ -0,0 +1,381 @@ +//! Resume at reset (orchestration v2, ticket 07): the durable continuation a +//! Limited chat's **Resume at reset** action arms. The engine — not the +//! client — owns it: at the limit's reset time it sends the continue prompt +//! exactly like the manual Resume (a normal run, queued messages behind it). +//! +//! One pending continuation per chat. It is cancelled by a newer message to +//! the chat, by archiving it, or by deleting it — the user's newer intent +//! always wins. It survives engine restarts (this file), and one whose time +//! passed while the engine was down fires after boot. +//! +//! This module is the store: the entries, their JSON persistence under the +//! engine data dir, and the disarm flags the timers check. The sessions +//! engine owns arming, firing and the session-row publishing +//! (`sessions.rs`). +//! +//! The loop guard: a fired continuation is consumed, never re-armed from its +//! own failure. A *new* limit record re-arms only when its reset time is +//! strictly later than the last continuation the chat fired (see +//! [`ScheduledResumes::advances`]) — a record whose reset does not advance +//! past the fired one is the same wall, and re-arming it would fire straight +//! back into the limit in a loop. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use roboco_proto::ScheduledResume; +use serde::{Deserialize, Serialize}; + +use crate::now_ms; + +/// The durable shape of `scheduled-resumes.json`. +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +struct ScheduledResumeFile { + /// The pending continuations, one per chat. + resumes: Vec, + /// When each chat's last continuation FIRED (epoch ms) — the loop guard's + /// progress floor. Survives restarts so a lying reset time cannot loop + /// across them either. + fired: HashMap, +} + +/// One pending continuation, as persisted. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ScheduledResumeRecord { + chat_id: String, + /// Epoch ms when the continuation fires. + fire_at: i64, +} + +/// One in-memory entry: the record plus the flag its timer checks. +#[derive(Debug)] +struct Entry { + fire_at: i64, + /// Set when the entry is replaced or cancelled: its timer wakes, sees + /// the flag, and goes away without firing. + disarmed: Arc, +} + +/// The engine's pending Resume-at-reset continuations. In-memory state +/// mirrors the JSON store; `path = None` (bare engines, tests) keeps +/// everything in memory. One instance lives on the sessions engine's +/// `Inner`. +#[derive(Debug, Default)] +pub(crate) struct ScheduledResumes { + entries: Mutex>, + fired: Mutex>, + path: Mutex>, +} + +impl ScheduledResumes { + /// Persist to `path` (loading anything already there) — engine assembly. + pub(crate) fn set_path(&self, path: PathBuf) { + let loaded = load_file(&path); + { + let mut entries = lock(&self.entries); + let mut fired = lock(&self.fired); + // Data only: boot recovery spawns the timers separately, after + // the doc host is wired (firing needs it). + for record in loaded.resumes { + entries.insert( + record.chat_id.clone(), + Entry { + fire_at: record.fire_at, + // No timer owns this entry yet — the disarm flag is + // for the timer boot recovery is about to spawn. + disarmed: Arc::new(AtomicBool::new(false)), + }, + ); + } + fired.extend(loaded.fired); + } + *lock(&self.path) = Some(path); + self.persist(); + } + + /// The pending continuation for `chat`, when one is armed. + pub(crate) fn scheduled(&self, chat_id: &str) -> Option { + lock(&self.entries) + .get(chat_id) + .map(|entry| ScheduledResume { + fire_at: entry.fire_at, + }) + } + + /// Arm (or re-arm) `chat`'s continuation at `fire_at`, disarming whatever + /// timer the replaced entry owned. Returns the new entry's disarm flag — + /// the timer the caller spawns for it. + pub(crate) fn arm(&self, chat_id: &str, fire_at: i64) -> Arc { + let disarmed = Arc::new(AtomicBool::new(false)); + { + let mut entries = lock(&self.entries); + if let Some(old) = entries.insert( + chat_id.to_string(), + Entry { + fire_at, + disarmed: disarmed.clone(), + }, + ) { + old.disarmed.store(true, Ordering::Release); + } + } + self.persist(); + disarmed + } + + /// Cancel `chat`'s continuation: remove, persist, disarm its timer. + /// `true` when there was one to cancel. + pub(crate) fn cancel(&self, chat_id: &str) -> bool { + let removed = lock(&self.entries).remove(chat_id); + if let Some(old) = &removed { + old.disarmed.store(true, Ordering::Release); + } + if removed.is_some() { + self.persist(); + } + removed.is_some() + } + + /// Consume the continuation this timer owns: remove and persist it — + /// BEFORE anything async happens, so a crash mid-fire can never + /// double-schedule. `false` when the entry is gone or was replaced by a + /// different one (this timer is obsolete). + pub(crate) fn consume(&self, chat_id: &str, fire_at: i64) -> bool { + let consumed = { + let mut entries = lock(&self.entries); + match entries.get(chat_id) { + Some(entry) if entry.fire_at == fire_at => { + entries.remove(chat_id); + true + } + _ => false, + } + }; + if consumed { + self.persist(); + } + consumed + } + + /// Put a consumed continuation back after a shutdown raced its dispatch: + /// the next boot re-arms it (overdue) instead of losing the user's + /// armed continuation to a graceful shutdown. + pub(crate) fn requeue(&self, chat_id: &str, fire_at: i64) { + let disarmed = Arc::new(AtomicBool::new(false)); + if let Some(old) = lock(&self.entries).insert( + chat_id.to_string(), + Entry { + fire_at, + disarmed, + }, + ) { + old.disarmed.store(true, Ordering::Release); + } + self.persist(); + } + + /// Every pending continuation with its disarm flag, for boot recovery's + /// timers. + pub(crate) fn pending(&self) -> Vec<(String, i64, Arc)> { + lock(&self.entries) + .iter() + .map(|(chat, entry)| (chat.clone(), entry.fire_at, entry.disarmed.clone())) + .collect() + } + + /// Stamp that `chat`'s continuation fired at `fire_at` — the progress + /// floor [`Self::advances`] re-arms must beat. + pub(crate) fn note_fired(&self, chat_id: &str, fire_at: i64) { + lock(&self.fired) + .entry(chat_id.to_string()) + .and_modify(|at| *at = (*at).max(fire_at)) + .or_insert(fire_at); + self.persist(); + } + + /// The loop guard: does `resets_at` advance past everything the chat + /// already fired (and the present)? A limit record whose reset does not + /// beat the last fired continuation is the same wall — re-arming it + /// would loop. See the module docs. + pub(crate) fn advances(&self, chat_id: &str, resets_at: i64) -> bool { + let floor = lock(&self.fired) + .get(chat_id) + .copied() + .unwrap_or(i64::MIN); + resets_at > floor.max(now_ms()) + } + + fn persist(&self) { + // Snapshot under the locks (both maps in one acquisition, so an + // arming racing this persist cannot write a file that mixes + // pre- and post-arm state), then write outside them. + let (path, file) = { + let path = lock(&self.path).clone(); + let file = { + let entries = lock(&self.entries); + let fired = lock(&self.fired); + ScheduledResumeFile { + resumes: entries + .iter() + .map(|(chat, entry)| ScheduledResumeRecord { + chat_id: chat.clone(), + fire_at: entry.fire_at, + }) + .collect(), + fired: fired.clone(), + } + }; + (path, file) + }; + let Some(path) = path else { + return; + }; + if let Err(err) = write_file(&path, &file) { + tracing::warn!(path = %path.display(), error = %err, "scheduled-resume persist failed"); + } + } +} + +fn load_file(path: &Path) -> ScheduledResumeFile { + std::fs::read(path) + .map_err(|err| err.to_string()) + .and_then(|bytes| { + serde_json::from_slice::(&bytes).map_err(|err| err.to_string()) + }) + .unwrap_or_else(|err| { + // Missing file = fresh engine; broken file = keep going with + // nothing armed (never refuse the engine over one feature). + if path.exists() { + tracing::warn!(path = %path.display(), error = %err, "scheduled-resume store unreadable"); + } + ScheduledResumeFile::default() + }) +} + +/// Write atomically (temp file + rename): a crash or power loss mid-write +/// must never leave a truncated `scheduled-resumes.json` that boots as +/// "unreadable — nothing armed", silently dropping every pending +/// continuation. The rename lands whole or not at all. +fn write_file(path: &Path, file: &ScheduledResumeFile) -> std::io::Result<()> { + let bytes = serde_json::to_vec_pretty(file) + .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?; + write_file_atomic(path, &bytes) +} + +/// Temp-file-then-rename, with the temp file in the target's directory so +/// the rename stays on one filesystem. +fn write_file_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + let dir = path.parent().unwrap_or(Path::new(".")); + let mut staged = tempfile::NamedTempFile::new_in(dir)?; + staged.write_all(bytes)?; + staged.flush()?; + staged.as_file().sync_all()?; + staged.persist(path).map(drop).map_err(|err| err.error) +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_path() -> PathBuf { + let dir = tempfile::tempdir().unwrap(); + let path = dir.keep().join("scheduled-resumes.json"); + path + } + + #[test] + fn arm_cancel_and_consume_track_one_entry_per_chat() { + let store = ScheduledResumes::default(); + assert!(store.scheduled("chat").is_none()); + assert!(!store.cancel("chat")); + + let first = store.arm("chat", 1_000); + assert_eq!(store.scheduled("chat").unwrap().fire_at, 1_000); + // Re-arming replaces: the old timer's flag flips. + let second = store.arm("chat", 2_000); + assert!(first.load(Ordering::Acquire)); + assert!(!second.load(Ordering::Acquire)); + assert_eq!(store.scheduled("chat").unwrap().fire_at, 2_000); + + // A timer for the replaced entry cannot consume the new one. + assert!(!store.consume("chat", 1_000)); + assert_eq!(store.scheduled("chat").unwrap().fire_at, 2_000); + // The owner consumes. + assert!(store.consume("chat", 2_000)); + assert!(store.scheduled("chat").is_none()); + // Taken is taken. + assert!(!store.consume("chat", 2_000)); + } + + #[test] + fn cancel_disarms_the_live_timer() { + let store = ScheduledResumes::default(); + let disarmed = store.arm("chat", 1_000); + assert!(store.cancel("chat")); + assert!(disarmed.load(Ordering::Acquire)); + assert!(store.scheduled("chat").is_none()); + // Nothing left to cancel. + assert!(!store.cancel("chat")); + } + + #[test] + fn entries_and_fired_floors_survive_a_store_reload() { + // `advances` also guards against the present, so the resets sit in + // the future. + let base = now_ms() + 3_600_000; + let path = temp_path(); + let first = ScheduledResumes::default(); + first.set_path(path.clone()); + first.arm("chat-live", base); + first.arm("chat-done", base + 1_000); + first.consume("chat-done", base + 1_000); + first.note_fired("chat-done", base + 1_000); + + let reloaded = ScheduledResumes::default(); + reloaded.set_path(path); + assert_eq!(reloaded.scheduled("chat-live").unwrap().fire_at, base); + assert!(reloaded.scheduled("chat-done").is_none()); + assert_eq!(reloaded.pending().len(), 1); + // The fired floor survives the restart. + assert!(reloaded.advances("chat-done", base + 2_000)); + assert!(!reloaded.advances("chat-done", base + 1_000)); + // A reset in the past never advances. + assert!(!reloaded.advances("chat-live", 1)); + } + + #[test] + fn a_broken_store_file_loads_as_nothing_armed() { + let path = temp_path(); + std::fs::write(&path, b"{not json").unwrap(); + let store = ScheduledResumes::default(); + store.set_path(path.clone()); + assert!(store.pending().is_empty()); + // And it heals: the next persist writes a valid file. + store.arm("chat", 9_000); + let healed = ScheduledResumes::default(); + healed.set_path(path); + assert_eq!(healed.scheduled("chat").unwrap().fire_at, 9_000); + } + + #[test] + fn requeue_restores_a_consumed_entry_for_the_next_boot() { + let path = temp_path(); + let store = ScheduledResumes::default(); + store.set_path(path.clone()); + store.arm("chat", 5_000); + assert!(store.consume("chat", 5_000)); + store.requeue("chat", 5_000); + let reloaded = ScheduledResumes::default(); + reloaded.set_path(path); + assert_eq!(reloaded.scheduled("chat").unwrap().fire_at, 5_000); + } +} diff --git a/crates/engine/src/sessions.rs b/crates/engine/src/sessions.rs index fc7d4c441b..f6509d9bd1 100644 --- a/crates/engine/src/sessions.rs +++ b/crates/engine/src/sessions.rs @@ -18,6 +18,7 @@ //! spawn failure, stream error, engine-restart recovery). use std::collections::HashMap; +use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex, MutexGuard, OnceLock, PoisonError}; use chrono::Utc; @@ -30,8 +31,9 @@ use roboco_doc::{ }; use roboco_harness::{CancellationToken, Harness, RunControls, SteerMessage, TurnControl}; use roboco_proto::{ - AgentEvent, DoneStatus, HarnessId, RunRequest, Session, SessionStatus, UserInputAnswer, - UserInputQuestion, + AgentErrorKind, AgentEvent, DoneStatus, HarnessId, ResumeAccountCandidates, + ResumeOnAccountOutcome, RunRequest, ScheduledResume, Session, SessionLimit, + SessionStatus, UserInputAnswer, UserInputQuestion, }; use crate::doc_host::{ChatDocHandle, DocHost}; @@ -68,6 +70,13 @@ type PendingInputs = Arc, } /// The configuration a live harness runtime launched with. The steering @@ -143,6 +152,20 @@ struct RunHandle { /// This runtime's provider session holds the fork's copied history (a /// side chat's bootstrap went out on its run or on one of its steers). fork_history_sent: Arc, + /// The credential fingerprint of the harness login this runtime's + /// processes were launched under (account failover, orchestration v2 + /// ticket 09): warm runtimes keep the token they started with, so a + /// dispatch whose fresh read of the live credentials differs never + /// routes into this runtime — it replaces it. `None` when the harness's + /// login is not fingerprinted (no account store, a Keychain login, a + /// harness roboco never reads). + credential_fingerprint: Option, + /// The parent chat this run's CURRENT turn owes a completion notice to + /// (orchestration v2): set when the turn's starting message or a steer + /// within it came from the notifying parent that created the chat, + /// reset when a user message starts a new turn. Read at the engine's + /// own turn-end states to record the outcome. + notifiable: Mutex>, /// The user stopped this runtime: it is being torn down and takes no /// more prompts. Flipped under the ledger lock, so every accepted steer /// is unambiguously before the stop (cancelled with it) or after it @@ -155,6 +178,11 @@ struct RunHandle { struct RoutedSteer { prompt: String, message_id: String, + /// The chat that sent this steer, when a chat sent it through the Roboco + /// MCP server — the re-dispatch of an orphaned steer keeps it, so the + /// turn it starts stays notifiable for its parent (ticket 11's rule, + /// extended to the orphan path by ticket 12). + origin_chat_id: Option, /// The delivered text carried the fork's copied history: its delivery /// is recorded only when the runtime confirms consuming it (`Steered`). /// An orphan re-dispatches the bare prompt, still owing the history. @@ -198,6 +226,18 @@ struct Inner { /// (roboco kept the same pair on `chats.harness_session_id`). An empty /// session id is the "do not resume" tombstone after a rejected resume. harness_sessions: Mutex>, + /// Completion-notice bookkeeping (orchestration v2): the outcomes + /// side-chat turns owe their agent parents, mirrored to a durable + /// engine-local ledger so undelivered outcomes survive restarts + /// (ticket 13). + notices: crate::completion_notices::CompletionNotices, + /// Pending Resume-at-reset continuations (orchestration v2, ticket 07): + /// the durable entries + disarm flags, persisted under the engine data + /// dir. Arming, firing and cancellation live on [`SessionsEngine`]. + scheduled_resumes: crate::scheduled_resumes::ScheduledResumes, + /// Engine settings the engine itself acts on (auto-resume limited + /// chats, …) — loaded at assembly, persisted under the data dir. + engine_settings: crate::engine_settings::EngineSettingsStore, /// Auto-titler for untitled chats (wired at engine assembly; absent in bare tests). titles: OnceLock, generated_images: OnceLock<(crate::uploads::Uploads, std::path::PathBuf)>, @@ -205,6 +245,13 @@ struct Inner { /// dispatch or accepted steer) — the diff sync snapshots the checkout tree /// for the Changes pane's "Latest turn" scope. Absent in bare tests. turn_listener: OnceLock, + /// Harness accounts (engine assembly; absent in bare tests): usage-limit + /// records read the active login's cached windows (which account the + /// limit hit, reset-time fill — ticket 06), and a run that fails with a + /// signed-out error marks the harness's live login on them (ticket 08). + /// A `Mutex` (not `OnceLock`) so a test's isolated store can swap in over + /// the detected one — every reader must see one store. + agent_accounts: Mutex>, } /// Turn-start hook: called with `(chat_id, cwd)`. @@ -241,9 +288,13 @@ impl SessionsEngine { sessions_tx, last_requests: Mutex::new(HashMap::new()), harness_sessions: Mutex::new(HashMap::new()), + notices: crate::completion_notices::CompletionNotices::default(), + scheduled_resumes: crate::scheduled_resumes::ScheduledResumes::default(), + engine_settings: crate::engine_settings::EngineSettingsStore::default(), titles: OnceLock::new(), generated_images: OnceLock::new(), turn_listener: OnceLock::new(), + agent_accounts: Mutex::new(None), }), } } @@ -294,6 +345,93 @@ impl SessionsEngine { let _ = self.inner.turn_listener.set(listener); } + /// Wire the harness accounts (called once at engine assembly; a test + /// swaps in an isolated store through the same call). Usage-limit records + /// read the active login's cached windows (see [`Inner::limit_record`]), + /// and runs that fail signed-out mark the live login on them. + pub fn set_agent_accounts(&self, accounts: crate::agent_accounts::AgentAccounts) { + *lock(&self.inner.agent_accounts) = Some(accounts); + } + + /// The engine's harness-account store (crate-internal): the account + /// failover (orchestration v2, ticket 09) ranks, probes and switches + /// accounts on it, and every dispatch fingerprints the live login. + pub(crate) fn agent_accounts(&self) -> Option { + lock(&self.inner.agent_accounts).clone() + } + + /// The harness-native session a run of `for_harness` on `chat_id` from + /// `cwd` would resume (crate-internal; the failover's continuation names + /// it explicitly so its fallback history can pair with it). + pub(crate) fn harness_session_to_resume( + &self, + chat_id: &str, + cwd: &str, + for_harness: HarnessId, + ) -> Option { + self.inner.resume_for(chat_id, cwd, for_harness) + } + + /// The ranked switch targets for one Limited chat's "Resume on another + /// account" (orchestration v2, ticket 09): the cached harness accounts + /// of the chat's harness, best first, plus the harness's signed-out + /// rows (listed with a "Sign in again" offer — stories 47/50). Both + /// lists empty when every other account is limited, an API key, or + /// unswitchable — then only Resume at reset is offered. + pub async fn resume_account_candidates( + &self, + chat_id: &str, + ) -> Result { + crate::account_failover::candidates(self, chat_id).await + } + + /// **Resume on another account** (ticket 09): rank the cached harness + /// accounts, probe the chosen one (or the best), activate it, retire the + /// chat's warm runtime, and queue the continuation — the same harness + /// session resumed with the continue prompt, the budgeted history + /// (ticket 01) riding as the fallback for a refused native resume. A + /// signed-out failure after the switch moves to the next candidate; + /// any other failure restores the previously active account. + pub async fn resume_on_another_account( + &self, + chat_id: &str, + chosen: Option<&str>, + ) -> Result { + crate::account_failover::resume_on_another_account(self, chat_id, chosen).await + } + + /// Persist the Resume-at-reset store under `path`, loading anything + /// already there (engine assembly — `EngineCore::assemble`). The + /// entries carry no timers until [`Self::recover_scheduled_resumes`] + /// arms them, because firing needs the doc host this runs before. + pub fn set_scheduled_resumes_path(&self, path: std::path::PathBuf) { + self.inner.scheduled_resumes.set_path(path); + } + + /// Persist the engine settings under `path`, loading anything already + /// there (engine assembly; a bare engine keeps the in-memory default — + /// auto-resume off). + pub fn set_engine_settings_path(&self, path: std::path::PathBuf) { + self.inner.engine_settings.attach(path); + } + + /// The engine's settings (see [`crate::engine_settings::EngineSettings`]). + pub fn engine_settings(&self) -> crate::engine_settings::EngineSettings { + self.inner.engine_settings.get() + } + + /// Update the engine's settings, persisting them when a store path is + /// wired. A failed disk write still applies in memory (and is reported). + pub fn set_engine_settings( + &self, + settings: crate::engine_settings::EngineSettings, + ) -> Result<(), EngineError> { + self.inner + .engine_settings + .set(settings) + .map_err(|err| EngineError::Other(format!("engine settings write failed: {err}"))) + } + fn note_turn_start(&self, chat_id: &str, cwd: &str) { if let Some(listener) = self.inner.turn_listener.get() { listener(chat_id, cwd); @@ -317,6 +455,166 @@ impl SessionsEngine { lock(&self.inner.statuses).get(chat_id).cloned() } + /// The completion-notice state (crate-internal; orchestration v2). + pub(crate) fn notices(&self) -> &crate::completion_notices::CompletionNotices { + &self.inner.notices + } + + /// Load the durable completion-notice ledger a previous engine run + /// left (ticket 13): outcomes recorded but never delivered survive the + /// restart. Called at assembly, BEFORE boot recovery, so the + /// interrupted outcomes recovery records for runs the restart lost + /// land in the same ledger. + pub(crate) fn load_pending_notices(&self, path: &std::path::Path) { + self.inner.notices.load(path); + } + + /// Boot delivery (ticket 13): every parent the ledger owes gets its + /// notice now, unless it is already busy again. A parent whose crashed + /// run is auto-resumed may still read idle here — the staggered + /// revival lands up to a couple of seconds later — in which case the + /// notice turn starts and the revival joins it like any later message; + /// either order loses nothing, and the outcome always reaches the + /// parent. + pub(crate) fn deliver_pending_notices(&self) { + for parent in self.inner.notices.parents() { + spawn_notice_delivery(&self.inner, &parent); + } + } + + /// The doc host, for the completion-notice delivery path (crate-internal). + pub(crate) fn host(&self) -> Option { + self.inner.doc_host() + } + + /// Whether the engine is tearing down (crate-internal): notice + /// delivery defers — outcomes stay durably pending, and the next boot + /// delivers them. + pub(crate) fn shutting_down(&self) -> bool { + self.inner + .shutting_down + .load(std::sync::atomic::Ordering::Acquire) + } + + /// An observation report from the Roboco MCP server (ticket 14): the + /// chat `parent` — the server's calling chat — just observed `side_chat` + /// after its last outcome happened, through `wait_for_turn`, a send + /// with wait, `read_chat` or `get_chat`. `observed_at` is the moment + /// the observation started (epoch ms), when the caller knows it: the + /// watermark covers only the settles that had happened by then, so an + /// outcome that landed between the read and this report — the parent's + /// unseen work — survives. Outcomes the parent already observed owe no + /// notice: they are dropped, and a record that lands right after the + /// report raced it is dropped too. + pub(crate) fn ack_notice(&self, parent: &str, side_chat: &str, observed_at: Option) { + let watermark = self.inner.notices.ack(parent, side_chat, observed_at); + tracing::debug!( + parent, + side_chat, + watermark, + "completion-notice observation acknowledged" + ); + } + + /// Clear the notifiable-parent claim on every live run whose current + /// turn owes a notice to `parent` (crate-internal; ticket 14's + /// stopped-parent rule). The side chats keep running; their turns + /// simply stop owing outcomes — a later parent message re-arms the + /// claim through `mark_notifiable`. + pub(crate) fn suppress_notifiable_runs(&self, parent: &str) { + let runs = lock(&self.inner.runs); + for run in runs.values() { + let mut notifiable = lock(&run.notifiable); + if notifiable.as_deref() == Some(parent) { + *notifiable = None; + } + } + } + + /// Loop-cap bookkeeping for a message that just landed (ticket 14): a + /// turn whose starting message id carries the engine's notice prefix + /// counts as notice-started (the consecutive streak grows); a + /// user-authored message — no origin chat, not one of the engine's own + /// artifacts — breaks the streak and resets the count. A message from + /// another chat does neither: the cap is about notice loops, and it + /// waits for the user, not for third-party traffic. + fn note_loop_counter(&self, chat_id: &str, message_id: &str, origin_chat_id: Option<&str>) { + if message_id.starts_with(crate::completion_notices::NOTICE_ID_PREFIX) { + self.inner.notices.notice_turn_started(chat_id); + } else if origin_chat_id.is_none() { + self.inner.notices.note_user_message(chat_id); + } + } + + /// The parent a message from `origin` makes a turn of `chat_id` + /// notifiable for: only the chat the durable agent-origin row + /// (`roboco_proto::AgentOrigin` on the chat) names, and only while it + /// did not opt out of notices at creation. `None` for user-typed sends + /// (no origin), third-party sends, user-created side chats (no agent + /// origin) and opted-out creators — those never notify. `sent_at` (the + /// moment the message was sent, when the caller knows it) gates the + /// stopped-parent rule: a message queued or sent BEFORE the parent's + /// last Stop does not re-arm the chat (story 24 counts only later + /// messages); `None` (time unknown) never gates. + fn notifiable_parent_for( + &self, + chat_id: &str, + origin: Option<&str>, + sent_at: Option, + ) -> Option { + let origin = origin?; + let agent_origin = self + .inner + .workspace() + .and_then(|ws| ws.chat(chat_id).ok().flatten()) + .and_then(|chat| chat.agent_origin)?; + agent_origin + .notify + .then(|| agent_origin.chat_id) + .filter(|parent| parent == origin) + .filter(|parent| { + sent_at.is_none_or(|at| { + self.inner + .notices + .parent_stopped_at(parent) + .is_none_or(|stop| at > stop) + }) + }) + } + + /// Stamp a live run's notifiable parent for a message that just joined + /// it. A message into a parked (idle) session starts a NEW turn: the + /// turn's notifiability is that message's own. One into a working turn + /// only ever ARMS notices — a parent message makes the turn notifiable; + /// anything else (a user's steer) leaves the starting message's claim + /// intact. `sent_at` (the message's send time, when known) carries the + /// stopped-parent rule through: a message sent before the parent's last + /// Stop must not re-arm. + fn mark_notifiable( + &self, + chat_id: &str, + run_id: &str, + origin: Option<&str>, + sent_at: Option, + ) { + let mid_turn = matches!( + self.session_status(chat_id).map(|session| session.status), + Some(SessionStatus::Working) | Some(SessionStatus::AwaitingInput) + ); + let from_parent = self.notifiable_parent_for(chat_id, origin, sent_at); + let mut handle = lock(&self.inner.runs); + let Some(run) = handle.get_mut(chat_id).filter(|h| h.run_id == *run_id) else { + return; + }; + let mut notifiable = lock(&run.notifiable); + let current = notifiable.take(); + *notifiable = if mid_turn { + from_parent.or(current) + } else { + from_parent + }; + } + /// Whether this harness takes a prompt *during* a turn, rather than only at /// the boundary between turns. The queue drain's central question: a /// mid-turn agent gets the message now, everything else holds it until the @@ -439,15 +737,37 @@ impl SessionsEngine { /// - A live steerable run receives the prompt as its next turn via the mailbox /// (roboco's persistent-session routing); otherwise any live run is interrupted /// first — never two runtimes driving one chat. + /// + /// `origin_chat_id` is the structured attribution when a chat sent this + /// through the Roboco MCP server; the doc-host command path has usually + /// written the entry already (idempotent by id), and the queue-drain + /// path relies on this write to stamp it. + /// + /// `sent_at` is the message's send time (epoch ms) when the caller knows + /// it — a queued row's or a command's issue time. It carries the + /// stopped-parent re-arm rule (ticket 14): a message sent before the + /// origin chat's last Stop does not re-arm that side chat's notices. + /// `None` (unknown, or an engine-authored send) never gates. + #[allow(clippy::too_many_arguments)] pub async fn dispatch( &self, chat_id: &str, harness_id: HarnessId, request: RunRequest, message_id: Option, + origin_chat_id: Option, + sent_at: Option, ) -> Result { - self.dispatch_with(chat_id, harness_id, request, message_id, false) - .await + self.dispatch_with( + chat_id, + harness_id, + request, + message_id, + origin_chat_id, + false, + sent_at, + ) + .await } /// [`Self::dispatch`] with the startup-crash retry marker: the retry @@ -455,24 +775,38 @@ impl SessionsEngine { /// final (its own startup death surfaces instead of retrying again). /// Boxed future: `drive_run` re-enters this for that retry, and the /// erasure breaks the opaque-type cycle the recursion would otherwise form. + #[allow(clippy::too_many_arguments)] fn dispatch_with<'a>( &'a self, chat_id: &'a str, harness_id: HarnessId, request: RunRequest, message_id: Option, + origin_chat_id: Option, startup_retry: bool, + sent_at: Option, ) -> futures::future::BoxFuture<'a, Result> { - Box::pin(self.dispatch_inner(chat_id, harness_id, request, message_id, startup_retry)) + Box::pin(self.dispatch_inner( + chat_id, + harness_id, + request, + message_id, + origin_chat_id, + startup_retry, + sent_at, + )) } + #[allow(clippy::too_many_arguments)] async fn dispatch_inner( &self, chat_id: &str, harness_id: HarnessId, mut request: RunRequest, mut message_id: Option, + origin_chat_id: Option, startup_retry: bool, + sent_at: Option, ) -> Result { // Project-less chats store cwd `~` (the creating device can't know the // host's home); expand it here, on the host, where the run spawns. @@ -482,11 +816,29 @@ impl SessionsEngine { // cross-harness delivery before recording or routing the user turn. roboco_proto::invocation::validate_harness_invocations(&request.prompt, harness_id) .map_err(EngineError::Other)?; + // The live login's fingerprint, fresh from the CLI store (no value + // when the harness's login is not fingerprinted). Compared against + // the warm runtime's stamp below: a runtime started under another + // login is never reused — it holds the old token in memory (ticket 09). + let live_login = self.inner.live_login_fingerprint(harness_id); let routed = lock(&self.inner.runs).get(chat_id).map(|h| { + // A fingerprint mismatch (a switch the engine observed since this + // runtime launched) downgrades ANY route — including an in-place + // reconfiguration — to "replace the runtime": the warm process + // must not run the next turn on the old token. + let same_login = h.credential_fingerprint.as_deref().is_none_or( + |fingerprint| { + live_login + .as_deref() + .is_none_or(|live| live == fingerprint) + }, + ); ( h.run_id.clone(), h.steerable && !h.retiring.load(std::sync::atomic::Ordering::Acquire), - h.runtime_config.route(harness_id, &request), + h.runtime_config + .route(harness_id, &request) + .filter(|_| same_login), h.steer_tx.clone(), h.routed_steers.clone(), h.fork_history_sent.clone(), @@ -534,6 +886,7 @@ impl SessionsEngine { pending.push_back(RoutedSteer { prompt: request.prompt.clone(), message_id: user_id.clone(), + origin_chat_id: origin_chat_id.clone(), fork_history: bootstrap.is_some(), after_stop: retiring.load(std::sync::atomic::Ordering::Acquire), request: Some(Box::new(request.clone())), @@ -545,6 +898,11 @@ impl SessionsEngine { false }; if accepted { + // A warm dispatch is a steer: the message joins the turn in + // flight (arming its completion notice) or starts the parked + // session's next turn (whose notifiability is the message's + // own) — orchestration v2. + self.mark_notifiable(chat_id, &run_id, origin_chat_id.as_deref(), sent_at); if reconfigure { // The runtime runs this configuration from now on: later // sends compare against it, and a re-dispatch of an @@ -558,7 +916,16 @@ impl SessionsEngine { lock(&self.inner.last_requests).insert(chat_id.to_string(), request.clone()); } let handle = self.doc_handle(chat_id)?; - handle.write_user_message(&user_id, &request.prompt, now_ms())?; + handle.write_user_message( + &user_id, + &request.prompt, + now_ms(), + origin_chat_id.as_deref(), + )?; + // The warm send still counts for the notice loop (ticket + // 14): a notice row routing into a live turn is + // notice-started; a user's message resets the streak. + self.note_loop_counter(chat_id, &user_id, origin_chat_id.as_deref()); // A turn counts from acceptance — a prompt deferred behind an // update must not snapshot a turn that has not started. self.note_turn_start(chat_id, &request.cwd); @@ -617,7 +984,17 @@ impl SessionsEngine { let harness = self.inner.registry.resolve(harness_id)?; let handle = self.doc_handle(chat_id)?; let user_id = message_id.unwrap_or_else(new_id); - handle.write_user_message(&user_id, &request.prompt, now_ms())?; + handle.write_user_message( + &user_id, + &request.prompt, + now_ms(), + origin_chat_id.as_deref(), + )?; + // Ticket 14's loop cap: a notice turn — the starting message is one + // of the engine's notice artifacts (a delivered notice entry or a + // released parked row) — grows the consecutive count; a + // user-authored message resets it. + self.note_loop_counter(chat_id, &user_id, origin_chat_id.as_deref()); // Engine-owned resume (roboco sessions.ts:736 — every dispatch read the // chat's stored harness session): callers always send `resume: None`; @@ -628,9 +1005,24 @@ impl SessionsEngine { // and starting the retry fresh silently dropped a good conversation. let mut resume_injected = false; if request.resume.is_none() { - request.resume = self.inner.resume_for(chat_id, &request.cwd); + request.resume = self + .inner + .resume_for(chat_id, &request.cwd, harness_id); resume_injected = request.resume.is_some(); } + // A fork's first run continues the SOURCE's provider session + // natively (t3-orchestration-v2 ticket 17): the ForkSideChat RPC + // recorded the fork source and boundary on the chat, and the driver + // forks the provider session there — the copied history rides the + // provider, so nothing is pasted. The intent is consumed once; a + // missing ref (old chats), a changed harness, or a refused fork + // falls back to the budgeted transcript from ticket 01. Consumed + // even when the harness mismatches: the native path never re-arms. + if request.provider_fork.is_none() && request.resume.is_none() { + if let Some(fork) = handle.doc().take_provider_fork(harness_id) { + request.provider_fork = Some(fork); + } + } lock(&self.inner.last_requests).insert(chat_id.to_string(), request.clone()); let run_id = new_id(); @@ -704,6 +1096,13 @@ impl SessionsEngine { pending_inputs, routed_steers: Arc::new(Mutex::new(std::collections::VecDeque::new())), fork_history_sent: fork_history_sent.clone(), + // Stamped at launch with the login the runtime's processes + // start under: later dispatches compare against a fresh read + // and replace this runtime when the login has changed. + credential_fingerprint: live_login.clone(), + notifiable: Mutex::new( + self.notifiable_parent_for(chat_id, origin_chat_id.as_deref(), sent_at), + ), retiring: Arc::new(std::sync::atomic::AtomicBool::new(false)), }, ); @@ -779,18 +1178,23 @@ impl SessionsEngine { chat_id: &str, prompt: &str, message_id: Option, + origin_chat_id: Option, ) -> Result { - self.steer_at(chat_id, prompt, message_id, now_ms()).await + self.steer_at(chat_id, prompt, message_id, now_ms(), origin_chat_id) + .await } /// [`Self::steer`] with the prompt's issue time: a queued row promoted /// into a live run keeps its original send time on the transcript. + /// `origin_chat_id` is the structured attribution when a chat sent this + /// through the Roboco MCP server; the user entry it writes keeps it. pub(crate) async fn steer_at( &self, chat_id: &str, prompt: &str, message_id: Option, issued_at: i64, + origin_chat_id: Option, ) -> Result { let target = lock(&self.inner.runs) .get(chat_id) @@ -842,6 +1246,7 @@ impl SessionsEngine { pending.push_back(RoutedSteer { prompt: prompt.to_string(), message_id: user_id.clone(), + origin_chat_id: origin_chat_id.clone(), fork_history: bootstrap.is_some(), after_stop: retiring.load(std::sync::atomic::Ordering::Acquire), request: sent_with, @@ -856,8 +1261,23 @@ impl SessionsEngine { _ => SteerOutcome::DeferredByUpdate, }); }; + // A parent's steer within the turn keeps the turn notifiable; one + // into a parked session defines the next turn's notifiability — + // orchestration v2 completion notices. + self.mark_notifiable(chat_id, &run_id, origin_chat_id.as_deref(), Some(issued_at)); let handle = self.doc_handle(chat_id)?; - handle.write_user_message(&user_id, prompt, issued_at.min(now_ms()))?; + handle.write_user_message( + &user_id, + prompt, + issued_at.min(now_ms()), + origin_chat_id.as_deref(), + )?; + // A steer is never one of the engine's notice artifacts: a user's + // steer breaks the notice-turn streak (ticket 14); a chat's steer + // leaves the cap alone. + if origin_chat_id.is_none() { + self.inner.notices.note_user_message(chat_id); + } // A routed steer is a turn too. Fired here (not only on the confirmed // path) — a reclaim falls back to dispatch, which just re-snapshots. if let Some(request) = self.last_request(chat_id) { @@ -906,7 +1326,13 @@ impl SessionsEngine { /// shells, everything the agent runs. The next message resumes the /// conversation in a fresh process. (Sending, steering and switching /// models never come here: they keep the runtime and its work.) + /// + /// Stopping also drops the notices this chat is owed as a parent and + /// suppresses outcomes from side chats whose notifiable turns are + /// running now (ticket 14): a stopped parent must not restart on its + /// own. Side chats keep running — Stop does not cascade. pub async fn interrupt(&self, chat_id: &str) -> Result { + crate::completion_notices::stop_parent(self, chat_id); self.terminate(chat_id).await } @@ -921,8 +1347,14 @@ impl SessionsEngine { /// — still running, and the next prompt routes into it. Drivers without /// that, runs whose harness has not started yet, and stops that do not /// settle fall back to [`Self::terminate`]. + /// + /// Like [`Self::interrupt`], this is the user stopping the work in front + /// of them: the notices the chat is owed as a parent go with the turn it + /// kills (ticket 14) — a stopped parent must not be woken by a notice for + /// work the stop just ended. pub async fn stop_turn(&self, chat_id: &str) -> Result { const STOP_SETTLE: std::time::Duration = std::time::Duration::from_secs(15); + crate::completion_notices::stop_parent(self, chat_id); let target = lock(&self.inner.runs) .get(chat_id) .filter(|h| { @@ -997,6 +1429,92 @@ impl SessionsEngine { Ok(true) } + /// One-way harness switch between turns (t3-orchestration-v2 ticket 24). + /// Writes the "Switched to " seam into the transcript, drops + /// the chat's harness session (the tombstone also blocks the journal + /// scan from resurrecting the old harness's id), retires any stale + /// native-fork intent, and saves `config` — the caller's resolved picks + /// (model options reset to the new harness's defaults unless the user + /// chose them; the engine only stores what arrived). A warm runtime from + /// the old harness is torn down by the next dispatch (its run + /// configuration no longer routes). The next send starts a fresh + /// provider session on the new harness and carries the budgeted history + /// of everything above the seam. Switching back is just another switch. + /// + /// Refused while a turn is in flight (the picker stays locked while the + /// chat works), for an unknown chat, and for a switch onto the harness + /// the chat already runs. + pub async fn switch_harness( + &self, + chat_id: &str, + config: roboco_proto::ChatConfig, + ) -> Result<(), EngineError> { + let Some(ws) = self.inner.workspace() else { + return Err(EngineError::Other("The registry is unavailable".into())); + }; + let chat = ws + .chat(chat_id)? + .ok_or_else(|| EngineError::Other("Chat no longer exists".into()))?; + if !ws.is_host(chat_id) { + return Err(EngineError::Other( + "Harness switch must run on the chat's host device".into(), + )); + } + if self.turn_in_flight(chat_id) { + return Err(EngineError::Other( + "The chat is working; switch harnesses between turns".into(), + )); + } + // The target harness must resolve first: the switch is pointless (and + // the next dispatch would fail) against an unregistered harness, and + // its display name names it in the refusal below. + let harness = self.inner.registry.resolve(config.harness)?; + if chat.config.as_ref().is_some_and(|existing| existing.harness == config.harness) { + return Err(EngineError::Other(format!( + "Chat is already on {}", + harness.display_name() + ))); + } + // The seam first: once the row names the new harness, a crash between + // the two writes would at worst dispatch the next send on the new + // harness with the history still riding (the seam is what arms it). + let handle = self.doc_handle(chat_id)?; + handle.doc().clear_provider_fork(); + let marker_id = format!("switch:{}", new_id()); + handle + .doc() + .push_message(&SessionMessageEntry { + origin_chat_id: None, + duration_ms: None, + provider_turn: None, + id: marker_id.clone(), + role: MessageRole::System, + parts: vec![MessagePart::HarnessSwitch { + id: marker_id, + harness: config.harness, + }], + created_at: now_ms(), + device_id: self.inner.device_id.clone(), + status: Some(MessageStatus::Complete), + continuation_of: None, + }) + .map_err(|e| EngineError::Other(e.to_string()))?; + // Drop the session from every resume source: the live cache, the + // durable row (the empty-string tombstone also stops the journal + // scan — the old harness's session ids must never ride a resume on + // the new harness), and the native-fork intent above. + lock(&self.inner.harness_sessions).remove(chat_id); + ws.set_chat_harness_session(chat_id, "", ""); + // Sends held for the turn end carry the FULL request they were sent + // with — the old harness, its model, its options — and the drain + // prefers them over the chat row: rewrite them onto the switch's + // config, or the first message after the switch would run on the + // old harness (ticket 24). + handle.rewrite_held_sends_for_switch(&config); + ws.set_chat_config(chat_id, &config)?; + Ok(()) + } + /// Resolve a pending `request_input` question set. Returns `false` when no such /// request is pending (unknown id, or the run already settled). pub fn respond_input( @@ -1053,23 +1571,78 @@ impl SessionsEngine { // same harness conversation (roboco recoverDraft, sessions.ts:538). if let Some((session_id, cwd)) = self.inner.journal_harness_session(&chat_id) { self.inner - .remember_harness_session(&chat_id, &session_id, &cwd); + .remember_harness_session(&chat_id, &session_id, &cwd, None); } - // The revival prompt: the last user message (idempotent re-dispatch - // under the SAME id — `write_user_message` dedupes by id, so the - // transcript never shows a duplicate). + // The revival prompt: the turn's starting message. Its structured + // origin rides along, so a parent-started turn that is revived stays + // notifiable for its completion notice (orchestration v2). A + // notice-started turn's starting entry is the engine's SYSTEM + // notice — the loop cap's parked row or a delivered notice — so it + // is considered here too: without it, the revival would pick the + // USER message under the notice and re-send the parent's stale + // instruction for work the notice already answered. let prompt = handle.doc().read_entries().ok().and_then(|entries| { entries .iter() .rev() - .find(|e| e.role == MessageRole::User) + .find(|e| { + e.role == MessageRole::User + || (e.role == MessageRole::System + && e.id + .starts_with(crate::completion_notices::NOTICE_ID_PREFIX)) + }) .and_then(|e| { e.parts.iter().find_map(|p| match p { - MessagePart::Text { text, .. } => Some((e.id.clone(), text.clone())), + MessagePart::Text { text, .. } => Some(( + e.id.clone(), + text.clone(), + e.origin_chat_id.clone(), + )), _ => None, }) }) }); + // A notice-started turn revives with the engine's continue prompt + // (the same rule a consumed steer gets, ticket 04) under the + // notice entry's own id: `write_user_message` dedupes onto the + // System entry that is already there, and the loop counter still + // reads the revival as notice-started — never as the user's fresh + // message — so the cap's streak is not reset by a crash. + let notice_revival = prompt.as_ref().is_some_and(|(id, _, _)| { + id.starts_with(crate::completion_notices::NOTICE_ID_PREFIX) + }); + // Restart correctness (orchestration v2, ticket 04): a steer the + // harness had already confirmed consuming before the crash is + // NOT re-delivered — the model answered it once, and a replay + // makes it answer twice (the user's orchestrator chat + // acknowledged the same steer again after a restart). The durable + // consumed-steer record tells; the revival then continues the + // crashed turn with the engine's continue prompt — the same one + // Resume and Resume-at-reset send — under a deterministic id + // (`write_user_message` dedupes by id, so a crash between the + // revival's dispatch and its persistence re-lands as one entry). + // Everything else about the revival is unchanged: budget, + // staggering, the "— resuming" note, the resumed harness session, + // and the consumed steer's own structured origin, so a parent's + // revived turn still notifies when it ends. + let steer_consumed = prompt.as_ref().is_some_and(|(user_id, _, _)| { + self.inner.journal.steer_consumed(&chat_id, user_id) + }); + let prompt = match prompt { + Some((id, _text, origin)) if notice_revival => Some(( + id, + roboco_proto::RESUME_CONTINUE_PROMPT.to_string(), + origin, + )), + Some((user_id, _text, origin)) if steer_consumed => Some(( + format!("auto-resume-continue:{chat_id}:{user_id}"), + roboco_proto::RESUME_CONTINUE_PROMPT.to_string(), + origin, + )), + // Idempotent re-dispatch under the SAME id — `write_user_message` + // dedupes by id, so the transcript never shows a duplicate. + other => other, + }; let attempts = self.inner.journal.resume_attempts(&chat_id); let fresh = handle .doc() @@ -1103,11 +1676,48 @@ impl SessionsEngine { }; self.inner.publish(&chat_id, &done); let stamped = handle.mark_abandoned_streams(note)?.len(); + // Stamp the settle serial before the recovered chat reads idle + // (ticket 14): the interrupted outcome belongs to this settle, + // so an observation of the chat after boot covers it. + let settle = self.inner.notices.settle(&chat_id); self.set_status(&chat_id, SessionStatus::Idle, false); tracing::info!(chat = %chat_id, stamped, will_resume, attempts, "recovered stale session journal"); recovered += 1; if !will_resume { + // Completion notices (orchestration v2, ticket 13): a + // notifiable run lost to the restart with no auto-resume + // owes its parent an interrupted outcome — the work never + // finished, and the parent may want to restart it. Recorded + // synchronously here (not through `spawn_notice_record`) + // so the boot delivery that runs after recovery sees it in + // the ledger. Archived chats are quiet both ways: an + // archived side chat sends nothing, an archived parent is + // owed nothing. + let notifiable = prompt + .as_ref() + .and_then(|(_, _, origin)| origin.as_deref()) + .and_then(|origin| { + self.notifiable_parent_for(&chat_id, Some(origin), None) + }) + .filter(|parent| { + !crate::completion_notices::chat_archived(self, &chat_id) + && !crate::completion_notices::chat_archived(self, parent) + }); + if let Some(parent) = notifiable { + self.inner.notices.record( + &parent, + crate::completion_notices::PendingOutcome { + side_chat_id: chat_id.clone(), + settle, + outcome: crate::completion_notices::Outcome::Interrupted { + reason: Some( + crate::completion_notices::INTERRUPTED_REASON.into(), + ), + }, + }, + ); + } continue; } let attempt = self.inner.journal.note_resume_attempt(&chat_id); @@ -1115,7 +1725,7 @@ impl SessionsEngine { resume_ix * RESUME_STAGGER_MS + resume_jitter_ms(&chat_id), ); resume_ix += 1; - let (user_id, prompt_text) = prompt.expect("gated by will_resume"); + let (user_id, prompt_text, origin_chat_id) = prompt.expect("gated by will_resume"); let sessions = self.clone(); let gate = resume_gate.clone(); tokio::spawn(async move { @@ -1145,6 +1755,8 @@ impl SessionsEngine { auto_approve: false, attachments: Vec::new(), resume: None, + provider_fork: None, + resume_fallback_history: None, worktree: None, mcp: None, }) @@ -1164,6 +1776,8 @@ impl SessionsEngine { harness_id, request, Some(user_id), + origin_chat_id, + None, ) .await { @@ -1179,6 +1793,339 @@ impl SessionsEngine { Ok(recovered) } + // ── Resume at reset (orchestration v2, ticket 07) ───────────────────── + + /// The chat's pending Resume-at-reset continuation, when one is armed. + /// (The session row carries the same record for clients; this is the + /// engine-side read.) + pub fn scheduled_resume(&self, chat_id: &str) -> Option { + self.inner.scheduled_resumes.scheduled(chat_id) + } + + /// **Resume at reset**: arm the durable continuation for a Limited chat, + /// firing the continue prompt at its limit's reset time. Requires the + /// chat to be Limited with a reset time still ahead — otherwise the + /// honest offer is the manual Resume, and the caller is told so. + pub fn schedule_resume_at_reset(&self, chat_id: &str) -> Result<(), EngineError> { + let limit = lock(&self.inner.statuses) + .get(chat_id) + .and_then(|row| row.limit.clone()) + .ok_or_else(|| { + EngineError::Other(format!("chat {chat_id} is not Limited: no limit record")) + })?; + let fire_at = limit + .resets_at + .ok_or_else(|| EngineError::Other( + "the limit's reset time is unknown — use Resume instead".into(), + ))?; + if fire_at <= now_ms() { + return Err(EngineError::Other( + "the limit has already reset — use Resume instead".into(), + )); + } + self.arm_scheduled_resume(chat_id, fire_at); + Ok(()) + } + + /// Cancel the chat's pending Resume-at-reset continuation. A new message + /// to the chat, archiving it or deleting it all land here — the user's + /// newer intent wins. `true` when there was one to cancel. + pub fn cancel_scheduled_resume(&self, chat_id: &str) -> bool { + let cancelled = self.inner.scheduled_resumes.cancel(chat_id); + if cancelled { + self.inner.set_scheduled_resume_row(chat_id, None); + tracing::info!(chat = %chat_id, "scheduled resume cancelled"); + } + cancelled + } + + /// Cancel the continuation as the chat is DELETED: the store entry and + /// timer go, the in-memory row clears, but nothing mirrors into the + /// workspace doc — its sessions row is being deleted with the chat, and + /// a mirror would resurrect it. + pub(crate) fn cancel_scheduled_resume_with_chat(&self, chat_id: &str) { + if self.inner.scheduled_resumes.cancel(chat_id) { + self.inner.set_scheduled_resume_row_local(chat_id, None); + tracing::info!(chat = %chat_id, "scheduled resume cancelled with the deleted chat"); + } + } + + /// Boot recovery: arm timers for every continuation that survived the + /// restart — an overdue one fires now, after the engine is up. Called + /// from assembly once the doc host is wired (firing needs it). + /// + /// It also REPUBLISHES the client-visible rows a restart cleared: the + /// in-memory status map (the merged WatchSessions source for this + /// device's chats) starts empty at boot, so a still-Limited chat and + /// its armed continuation would vanish from the banner — "resumes in…", + /// Cancel, and `ScheduleResumeAtReset` (which reads the in-memory limit + /// record) all refused. The durable workspace rows still carry both, so + /// the boot re-creates the in-memory rows from them: still-Limited + /// chats keep their limit record, chats with an armed continuation keep + /// its row, and a fresh transition overwrites the republished row as + /// usual. + pub fn recover_scheduled_resumes(&self) { + self.republish_durable_rows(); + let pending = self.inner.scheduled_resumes.pending(); + for (ix, (chat_id, fire_at, disarmed)) in pending.into_iter().enumerate() { + // Same stagger discipline as the crashed-run auto-resume: a + // fleet of overdue continuations re-dispatches spaced out, not + // as one boot burst of CLI spawns. + let delay = std::time::Duration::from_millis(ix as u64 * RESUME_STAGGER_MS); + self.spawn_resume_timer(chat_id.clone(), fire_at, disarmed, delay); + // The armed continuation's row rides the republished status row + // (the durable row may predate the restart's last write): the + // banner keeps its countdown and its Cancel. + self.inner + .set_scheduled_resume_row(&chat_id, Some(ScheduledResume { fire_at })); + tracing::info!( + chat = %chat_id, + fire_at, + overdue = fire_at <= now_ms(), + "scheduled resume armed at boot" + ); + } + } + + /// Re-create the in-memory status rows a restart wiped, from the durable + /// workspace rows: only chats that are still Limited (errored with a + /// limit record) — a row that says Working or AwaitingInput belongs to + /// `recover_stale`'s revival, and an idle row owes clients nothing. The + /// settle serial of a republished row is unknown (the serials restart + /// with the engine); the row's limit record is what the banner and + /// `schedule_resume_at_reset` read. + fn republish_durable_rows(&self) { + let Some(ws) = self.inner.workspace() else { + return; + }; + let rows = match ws.read_sessions() { + Ok(rows) => rows, + Err(err) => { + tracing::warn!(error = %err, "boot republish of session rows failed"); + return; + } + }; + let mut statuses = lock(&self.inner.statuses); + let mut republished = 0usize; + for row in rows { + if row.status != SessionStatus::Errored || row.limit.is_none() { + continue; + } + // A run the boot revival is about to own: its transition lands + // through the normal path; do not pre-seed a row the revival's + // own set_status overwrites anyway. + if statuses.contains_key(&row.chat_id) { + continue; + } + let chat_id = row.chat_id.clone(); + statuses.insert(chat_id, row); + republished += 1; + } + if republished > 0 { + let mut list: Vec = statuses.values().cloned().collect(); + list.sort_by(|a, b| a.chat_id.cmp(&b.chat_id)); + self.inner.sessions_tx.send_replace(list); + tracing::info!(republished, "limited session rows republished at boot"); + } + } + + /// Arm (or re-arm) the chat's Resume-at-reset continuation at `fire_at`: + /// store it durably, publish it on the status row, spawn its timer. + fn arm_scheduled_resume(&self, chat_id: &str, fire_at: i64) { + let disarmed = self.inner.scheduled_resumes.arm(chat_id, fire_at); + self.inner + .set_scheduled_resume_row(chat_id, Some(ScheduledResume { fire_at })); + self.spawn_resume_timer( + chat_id.to_string(), + fire_at, + disarmed, + std::time::Duration::ZERO, + ); + } + + /// The **Auto-resume limited chats** setting's arm: a fresh limit record + /// re-arms the continuation only when its reset time is still ahead AND + /// advances past the last continuation this chat fired — the loop guard. + /// A record whose reset does not beat it is the same wall, and re-arming + /// would fire straight back into the limit in a loop (T3's model: the + /// scheduler only ever works from a fresh persisted limit record). + fn maybe_auto_arm_resume(&self, chat_id: &str, limit: &SessionLimit) { + if !self.inner.engine_settings.get().auto_resume_limited { + return; + } + let Some(resets_at) = limit.resets_at else { + return; + }; + if !self.inner.scheduled_resumes.advances(chat_id, resets_at) { + return; + } + tracing::info!( + chat = %chat_id, + resets_at, + "auto-resume limited chats: arming the continuation" + ); + self.arm_scheduled_resume(chat_id, resets_at); + } + + /// One continuation's timer: wait out the delay plus the remaining time + /// to `fire_at`, then hand the firing to the engine — unless the entry + /// was replaced or cancelled while it slept (the flag). `consume` + /// re-checks under the store's lock either way, so an obsolete timer can + /// never fire. + fn spawn_resume_timer( + &self, + chat_id: String, + fire_at: i64, + disarmed: Arc, + delay: std::time::Duration, + ) { + let engine = self.clone(); + tokio::spawn(async move { + if !delay.is_zero() { + tokio::time::sleep(delay).await; + } + // Clock skew and a reset time minted on a lagging client clamp + // to "due now": never a panic, never a negative sleep. + let wait_ms = fire_at.saturating_sub(now_ms()).max(0) as u64; + tokio::time::sleep(std::time::Duration::from_millis(wait_ms)).await; + if disarmed.load(std::sync::atomic::Ordering::Acquire) { + return; + } + engine.fire_scheduled_resume(&chat_id, fire_at).await; + }); + } + + /// Fire one armed continuation: send the continue prompt exactly like + /// the manual Resume — a normal run, the queued messages behind it. + /// Owned by the timer task spawned in [`Self::spawn_resume_timer`]. + async fn fire_scheduled_resume(&self, chat_id: &str, fire_at: i64) { + // A shutdown that began before the timer woke leaves the record + // armed for the next boot — shutting down never consumes one. + if self + .inner + .shutting_down + .load(std::sync::atomic::Ordering::Acquire) + { + return; + } + // Consume BEFORE anything async: the record is gone before the run + // starts, so a crash mid-fire can never double-schedule it. + if !self.inner.scheduled_resumes.consume(chat_id, fire_at) { + return; // cancelled or replaced while this timer slept + } + // The chat fired this continuation — the loop guard's progress + // floor (a new limit record must beat it to re-arm). + self.inner.scheduled_resumes.note_fired(chat_id, fire_at); + self.inner.set_scheduled_resume_row(chat_id, None); + // A shutdown that raced the consume: put the record back — the next + // boot fires it overdue instead of losing the armed continuation. + if self + .inner + .shutting_down + .load(std::sync::atomic::Ordering::Acquire) + { + self.inner.scheduled_resumes.requeue(chat_id, fire_at); + return; + } + // A turn already in flight is the user's newer intent — its send + // cancelled this continuation on the way in and the timer raced + // the cancellation. Ours loses. + if self.turn_in_flight(chat_id) { + tracing::info!( + chat = %chat_id, + "scheduled resume dropped: a turn is already in flight" + ); + return; + } + let Some(host) = self.inner.doc_host() else { + tracing::warn!(chat = %chat_id, "scheduled resume dropped: no doc host"); + return; + }; + // The chat row is gone — deleted here or on another device (the + // row syncs through the workspace doc), or swept with its space — + // or the chat was archived, possibly from another device whose + // sync never ran the archive RPC here: the continuation dies with + // the chat either way, and archived work must not resume on its + // own overnight. A bare host with no workspace to ask falls + // through to the last-request configuration below. + let chat = host.workspace().and_then(|ws| ws.chat(chat_id).ok()); + match chat { + Some(None) => { + tracing::info!(chat = %chat_id, "scheduled resume dropped: the chat is gone"); + return; + } + Some(Some(chat)) if chat.archived => { + tracing::info!(chat = %chat_id, "scheduled resume dropped: the chat is archived"); + return; + } + _ => {} + } + // The continuation runs on the chat's own configuration, like the + // completion-notice turn: the chat row when there is one, else the + // last live request. `dispatch` re-derives the harness session, so + // the continuation continues the same harness conversation. + let prompt = roboco_proto::RESUME_CONTINUE_PROMPT; + let request = host.request_from_chat_row(chat_id, prompt).or_else(|| { + self.last_request(chat_id).map(|mut request| { + request.prompt = prompt.to_string(); + // Dispatch re-injects the chat's remembered harness session — + // never carry a stale resume ref into the continuation. + request.resume = None; + request.attachments = Vec::new(); + request + }) + }); + let Some(request) = request else { + tracing::warn!( + chat = %chat_id, + "scheduled resume dropped: no run configuration for the chat" + ); + return; + }; + let harness = host.harness_for_request(chat_id, &request); + // Deterministic id: a re-fire after a crash between consume and + // persistence lands as the same transcript entry (dedupe by id). + let message_id = format!("resume-at-reset:{fire_at}:{chat_id}"); + tracing::info!(chat = %chat_id, fire_at, "scheduled resume firing"); + match host + .dispatch_with_source_context( + self, + chat_id, + harness, + request, + Some(message_id), + None, + None, + ) + .await + { + Ok(_) => { + // The continuation is the explicit prompt that thaws the + // queue the limited chat holds — the same deliberate action + // the manual Resume's run is (queued messages follow it). + if let Ok(handle) = host.open(chat_id) { + handle.thaw_queue(); + } + } + Err(err) => { + if self + .inner + .shutting_down + .load(std::sync::atomic::Ordering::Acquire) + { + // The engine went down under the dispatch: requeue for + // the next boot rather than drop the armed continuation. + self.inner.scheduled_resumes.requeue(chat_id, fire_at); + } + tracing::warn!( + chat = %chat_id, + error = %err, + "scheduled resume turn failed to start" + ); + } + } + } + /// Graceful shutdown: interrupt every live run so streaming entries settle. pub async fn shutdown(&self) { self.inner @@ -1266,6 +2213,7 @@ impl Inner { }, AgentEvent::Error { message: "Generated image unavailable".into(), + kind: None, }, ] } @@ -1287,6 +2235,26 @@ impl Inner { .collect() } + /// A run failed with ticket 05's structured signed-out error: mark the + /// harness's live login's account signed out (ticket 08) — the account + /// every new run on this harness would use, so the account switcher + /// stops showing it as healthy. Best-effort by design: a marking failure + /// is logged and never disturbs the run's own error path, and the next + /// sign-in or successful usage probe clears the mark. + async fn note_run_signed_out(&self, chat_id: &str, harness: HarnessId) { + let Some(accounts) = lock(&self.agent_accounts).clone() else { + return; + }; + if let Err(err) = accounts.mark_live_signed_out(harness).await { + tracing::warn!( + chat = %chat_id, + provider = ?harness, + error = %err, + "could not mark the harness account signed out" + ); + } + } + fn publish(&self, chat_id: &str, event: &AgentEvent) -> u64 { let seq = match self.journal.append(chat_id, event) { Ok(seq) => seq, @@ -1339,6 +2307,50 @@ impl Inner { self.set_status_with_completion(chat_id, status, fresh_start, None); } + /// Complete a usage-limit record for a turn that ended on one: which + /// harness account it hit (the active login — resolved when the turn + /// settles; warm runtimes keep the token they started with, so that is + /// the login the run used in practice), and — when the harness's own + /// error carried no reset time — the fill from that account: the window + /// whose LABEL matches the limit's (Codex's meters name the driver's + /// windows; Claude's do not), else the account's `limited_until` (the + /// latest reset among its exhausted windows — ticket 08's derivation, + /// one shared meaning for "the account is limited until"). Engines + /// without an accounts store (bare tests) record without either. + /// `None` when the classification was not a usage limit. + async fn limit_record( + &self, + harness: HarnessId, + kind: AgentErrorKind, + ) -> Option { + let AgentErrorKind::UsageLimit { window, resets_at } = kind else { + return None; + }; + let accounts = lock(&self.agent_accounts).clone(); + let active = match accounts { + Some(accounts) => accounts.active_account(harness).await, + None => None, + }; + let resets_at = resets_at.or_else(|| { + let account = active.as_ref()?; + // Same window, when the account's meters name it. + let by_label = account + .usage_windows + .iter() + .find(|w| w.resets_at.is_some() && w.label.eq_ignore_ascii_case(&window)); + by_label + .and_then(|w| w.resets_at) + .or(account.limited_until) + .map(|at| at.timestamp_millis()) + }); + Some(SessionLimit { + harness, + window, + resets_at, + account_id: active.map(|account| account.id), + }) + } + fn has_pending_steers(&self, chat_id: &str, run_id: &str) -> bool { lock(&self.runs) .get(chat_id) @@ -1354,6 +2366,23 @@ impl Inner { status: SessionStatus, fresh_start: bool, completed_turn: Option, + ) { + self.set_status_with_limit(chat_id, status, fresh_start, completed_turn, None); + } + + /// The full status transition: everything above, plus the usage-limit + /// record a limited turn ends with. `None` CLEARS any previous record — + /// the marker lives exactly as long as the errored status that recorded + /// it, so a new turn (any Working transition) or any other settled + /// outcome drops it. Only the run task's limit-errored exit passes + /// `Some`. + fn set_status_with_limit( + &self, + chat_id: &str, + status: SessionStatus, + fresh_start: bool, + completed_turn: Option, + limit: Option, ) { let now = Utc::now(); let session = { @@ -1367,6 +2396,8 @@ impl Inner { status, started_at: None, updated_at: now, + limit: None, + scheduled_resume: None, }); // `started_at` is the elapsed-timer base and must only ever mean // "this turn". Entering Working from a settled state always @@ -1383,6 +2414,7 @@ impl Inner { } entry.status = status; entry.updated_at = now; + entry.limit = limit; match status { SessionStatus::Working if fresh_start || !was_active => { entry.started_at = Some(now); @@ -1407,6 +2439,40 @@ impl Inner { } } + /// Publish the chat's pending Resume-at-reset continuation onto its + /// status row (arming and cancellation both land here), leaving the + /// status itself untouched, and mirror it into the workspace doc so + /// remote devices' banners see it. No row yet — nothing armed, nothing + /// to clear; the arm paths only ever run for a chat with a row. + fn set_scheduled_resume_row(&self, chat_id: &str, scheduled: Option) { + if let Some(session) = self.set_scheduled_resume_row_local(chat_id, scheduled) { + if let Some(ws) = self.workspace() { + ws.record_session(&session); + } + } + } + + /// [`set_scheduled_resume_row`] without the workspace mirror: the + /// in-memory row and the watch list only. The delete paths use this — + /// they are deleting the chat's sessions row, which a mirror would + /// resurrect. Returns the updated row when there was one. + fn set_scheduled_resume_row_local( + &self, + chat_id: &str, + scheduled: Option, + ) -> Option { + let now = Utc::now(); + let mut statuses = lock(&self.statuses); + let entry = statuses.get_mut(chat_id)?; + entry.scheduled_resume = scheduled; + entry.updated_at = now; + let session = entry.clone(); + let mut list: Vec = statuses.values().cloned().collect(); + list.sort_by(|a, b| a.chat_id.cmp(&b.chat_id)); + self.sessions_tx.send_replace(list); + Some(session) + } + /// The doc host, once wired. `None` before assembly or after retirement. fn doc_host(&self) -> Option { lock(&self.doc_host).clone() @@ -1448,6 +2514,16 @@ impl Inner { self.doc_host().and_then(|host| host.workspace().cloned()) } + /// The harness login's current credential fingerprint, read fresh from + /// the CLI store (account failover, orchestration v2 ticket 09). `None` + /// when no account store is wired or the harness's login is not + /// file-readable — then no runtime is ever retired by fingerprint. + fn live_login_fingerprint(&self, harness: HarnessId) -> Option { + lock(&self.agent_accounts) + .clone() + .and_then(|accounts| accounts.live_credentials_fingerprint(harness)) + } + /// Sidebar freshness: push a message-persist preview into the chat's workspace row. fn note_message(&self, chat_id: &str, text: &str) { if text.is_empty() { @@ -1460,8 +2536,16 @@ impl Inner { /// Record the chat's harness-native session id (and its cwd): live-process /// cache plus the durable workspace chat row — the row is what survives an - /// engine restart (roboco sessions.ts:1039). - fn remember_harness_session(&self, chat_id: &str, session_id: &str, cwd: &str) { + /// engine restart (roboco sessions.ts:1039). `harness` is the harness the + /// session belongs to when the caller knows it (a run's own + /// SessionStarted always does); `None` (journal-derived) trusts the id. + fn remember_harness_session( + &self, + chat_id: &str, + session_id: &str, + cwd: &str, + harness: Option, + ) { if session_id.is_empty() { return; } @@ -1470,6 +2554,7 @@ impl Inner { HarnessSessionRef { session_id: session_id.to_string(), cwd: cwd.to_string(), + harness, }, ); if let Some(ws) = self.workspace() { @@ -1484,15 +2569,18 @@ impl Inner { // conversations (user incident 2026-08-13). A truly stale id simply // yields a fresh session whose SessionStarted overwrites the row. - /// The session id to resume for a run in `chat_id` launching from `cwd` - /// (roboco sessions.ts:736, looked up on every dispatch): - /// live-process cache → workspace chat row → journal scan (the crash path - /// where the debounced row write never landed — SessionStarted/Done events - /// are journaled per event, flushed immediately). Cwd-gated throughout: - /// harness session stores are keyed by cwd, so a session created elsewhere - /// never rides `--resume`. An empty stored id is the explicit tombstone — - /// no resume, no falling through to staler sources. - fn resume_for(&self, chat_id: &str, cwd: &str) -> Option { + /// The session id to resume for a run of `for_harness` in `chat_id` + /// launching from `cwd` (roboco sessions.ts:736, looked up on every + /// dispatch): live-process cache → workspace chat row → journal scan (the + /// crash path where the debounced row write never landed — + /// SessionStarted/Done events are journaled per event, flushed + /// immediately). Cwd-gated throughout: harness session stores are keyed + /// by cwd, so a session created elsewhere never rides `--resume`. And + /// HARNESS-gated: a session recorded under another harness (a switch's + /// clear raced, or the row config changed) never rides this resume — the + /// new harness gets a fresh session. An empty stored id is the explicit + /// tombstone — no resume, no falling through to staler sources. + fn resume_for(&self, chat_id: &str, cwd: &str, for_harness: HarnessId) -> Option { // The same directory, however it is spelled: an agent reports where // it runs with symlinks resolved (`/private/var/…` for `/var/…` on // macOS), and until a turn's Done restores the request's spelling a @@ -1505,18 +2593,36 @@ impl Inner { .is_ok_and(|a| std::fs::canonicalize(cwd).is_ok_and(|b| a == b)) }; if let Some(known) = lock(&self.harness_sessions).get(chat_id).cloned() { - return (!known.session_id.is_empty() && cwd_ok(&known.cwd)) - .then_some(known.session_id); + if known.harness.is_some_and(|recorded| recorded != for_harness) { + // Another harness's conversation: fall through to the row, + // never resume it here. + } else { + return (!known.session_id.is_empty() && cwd_ok(&known.cwd)) + .then_some(known.session_id); + } } - if let Some(ws) = self.workspace() - && let Some((session_id, session_cwd)) = ws.chat_harness_session(chat_id) - { - return (!session_id.is_empty() && cwd_ok(session_cwd.as_deref().unwrap_or(""))) - .then_some(session_id); + if let Some(ws) = self.workspace() { + let row_harness = ws + .chat(chat_id) + .ok() + .flatten() + .and_then(|chat| chat.config) + .map(|config| config.harness); + if let Some((session_id, session_cwd)) = ws.chat_harness_session(chat_id) { + if row_harness.is_some_and(|recorded| recorded != for_harness) { + // The row's own config names another harness: its stored + // session id is that harness's, not this run's. + } else { + return (!session_id.is_empty() && cwd_ok(session_cwd.as_deref().unwrap_or(""))) + .then_some(session_id); + } + } } let (session_id, session_cwd) = self.journal_harness_session(chat_id)?; - // Cache the journal hit (memory + row) so later dispatches skip the scan. - self.remember_harness_session(chat_id, &session_id, &session_cwd); + // Cache the journal hit (memory + row) so later dispatches skip the + // scan. The journal records no harness: trusted as-is (the scan is + // the last resort, and a switched chat's tombstone never reaches it). + self.remember_harness_session(chat_id, &session_id, &session_cwd, None); cwd_ok(&session_cwd).then_some(session_id) } @@ -1602,11 +2708,16 @@ fn note_fork_history_session(doc: &SessionDoc, carried: bool, session_id: &str) } impl Inner { - /// `prompt` with the conversation a side chat's provider session lacks in - /// front of it, or `None` when it lacks nothing. A fresh session gets the - /// whole transcript (`current` excluded); a continued one only the fork's - /// copied history before the seam, until that went out to that session. - /// Native commands are never wrapped: providers route a leading `/name`. + /// `prompt` with the transcript a provider session lacks in front of it, + /// or `None` when it lacks nothing. A side chat's fresh session gets its + /// budgeted copied transcript (`current` excluded); a main chat's fresh + /// session gets the budgeted transcript only after a harness switch (the + /// seam marks what the dropped session held); a continued one only the + /// history above the chat's newest seam, until that went out to that + /// session. The budget and the selection rules live in + /// [`crate::transcript_budget`]. Native commands are never wrapped — + /// providers route a leading `/name`, so their history rides the first + /// ordinary send instead. fn fork_history_prompt( &self, chat_id: &str, @@ -1616,20 +2727,53 @@ impl Inner { current: Option<&str>, provider: ProviderSession<'_>, ) -> Option { - if native_command(prompt, harness_id) - || !self - .workspace() - .and_then(|ws| ws.chat(chat_id).ok().flatten()) - .is_some_and(|chat| chat.parent_chat_id.is_some()) - { + self.fork_history_prefix(chat_id, doc, harness_id, prompt, current, provider) + .map(|prefix| format!("{prefix}{prompt}")) + } + + /// The transcript prefix [`Self::fork_history_prompt`] would deliver — + /// the budgeted selection alone, without the prompt appended. The + /// native-fork path (ticket 17) rides this as the request's fallback + /// history: the driver pastes it in front of the prompt only when the + /// harness refuses the fork, so a successful fork delivers nothing. + fn fork_history_prefix( + &self, + chat_id: &str, + doc: &SessionDoc, + harness_id: HarnessId, + prompt: &str, + current: Option<&str>, + provider: ProviderSession<'_>, + ) -> Option { + if native_command(prompt, harness_id) { return None; } let entries: Vec<_> = doc .read_entries() .unwrap_or_default() .into_iter() - .filter(|entry| Some(entry.id.as_str()) != current) + // `current` — and any continuation fragment of it — belongs to + // the turn being sent, never to the transcript in front of it. + .filter(|entry| { + !current.is_some_and(|current| { + entry.id == current || entry.continuation_of.as_deref() == Some(current) + }) + }) .collect(); + // A side chat owes its copied transcript to any session that lacks + // it. A main chat owes its transcript only after a one-way harness + // switch (t3-orchestration-v2 ticket 24): the seam marks the + // conversation the new harness's fresh session never saw, so the + // first send after the switch carries the budgeted history of + // everything above it; without a seam a main chat's fresh session + // starts empty (nothing was copied, nothing was dropped). + let side_chat = self + .workspace() + .and_then(|ws| ws.chat(chat_id).ok().flatten()) + .is_some_and(|chat| chat.parent_chat_id.is_some()); + if !side_chat && !entries.iter().any(is_harness_switch_seam) { + return None; + } let end = match provider { ProviderSession::Fresh => entries.len(), ProviderSession::Continued(session) => { @@ -1637,46 +2781,68 @@ impl Inner { if session.is_some() && delivered.as_deref() == session { return None; } - entries.iter().position(|entry| { - entry - .parts - .iter() - .any(|part| matches!(part, roboco_doc::MessagePart::Fork { .. })) - })? + // The owed unit ends at the NEWEST seam, whichever kind: a + // fork's copied transcript above its seam, or the dropped + // conversation above a harness-switch seam (ticket 24). A + // session that changed below the newest seam lacks + // everything above it — including the case where the first + // post-switch send was a native command (never wrapped), so + // the history rides the first ordinary send instead. + entries.iter().rposition(is_history_seam)? } }; - let history: Vec<_> = entries[..end] + // The chat the omitted messages are readable in: the newest fork + // seam names the chat this transcript was copied from (its + // transcript holds everything above the seam); a seam-less side + // chat points at itself. + let source = entries .iter() - .map(|entry| { - let text = entry - .parts - .iter() - .filter_map(|part| match part { - roboco_doc::MessagePart::Text { text, .. } => Some(text.clone()), - roboco_doc::MessagePart::Tool { call, output, .. } => Some(format!( - "Tool: {}\n{}", - serde_json::to_string(call).unwrap_or_default(), - output.clone().unwrap_or_default() - )), - _ => None, - }) - .collect::>() - .join("\n"); - (entry.role, text) - }) - .filter(|(_, text)| !text.is_empty()) - .map(|(role, text)| serde_json::json!({ "role": role, "text": text })) - .collect(); - (!history.is_empty()).then(|| { - format!( - "Continue this side conversation using the following prior conversation as context.\n\n{}\n\n\n{}", - serde_json::to_string(&history).unwrap_or_default(), - prompt - ) - }) + .rev() + .find_map(fork_seam_source) + .unwrap_or_else(|| chat_id.to_owned()); + crate::transcript_budget::budgeted_transcript_prefix(&entries[..end], Some(&source)) } } +/// A fork seam entry: the transcript divider above which messages were +/// copied from another chat. +fn is_fork_seam(entry: &SessionMessageEntry) -> bool { + entry + .parts + .iter() + .any(|part| matches!(part, MessagePart::Fork { .. })) +} + +/// A history seam of either kind: the divider below which the chat's +/// provider session started fresh — a fork's copy boundary or a harness +/// switch's drop boundary. The newest one bounds what a changed session is +/// owed. +fn is_history_seam(entry: &SessionMessageEntry) -> bool { + is_fork_seam(entry) || is_harness_switch_seam(entry) +} + +/// A harness-switch seam entry (t3-orchestration-v2 ticket 24): the divider +/// below which the chat runs on the harness the switch named. Everything +/// above it belongs to the provider session the switch dropped, so it is +/// the history the new harness's first session is owed. +fn is_harness_switch_seam(entry: &SessionMessageEntry) -> bool { + entry + .parts + .iter() + .any(|part| matches!(part, MessagePart::HarnessSwitch { .. })) +} + +/// The chat a fork seam copied its transcript from. +fn fork_seam_source(entry: &SessionMessageEntry) -> Option { + entry + .parts + .iter() + .find_map(|part| match part { + MessagePart::Fork { source_chat_id, .. } => Some(source_chat_id.clone()), + _ => None, + }) +} + /// Whether the provider routes this prompt as a native command: its delivered /// text leads with `/name` (Codex `command_request`, OpenCode commands, Claude /// slash commands). Anything put in front of it would make it model input. @@ -1813,6 +2979,7 @@ impl SubagentSink { tracing::warn!(doc = %self.doc_id, error = %err, "subagent segment close failed"); } let entry = SessionMessageEntry { + origin_chat_id: None, id: new_id(), role: MessageRole::User, parts: vec![MessagePart::Text { @@ -1824,6 +2991,7 @@ impl SubagentSink { status: Some(MessageStatus::Complete), continuation_of: None, duration_ms: None, + provider_turn: None, }; if let Err(err) = self.doc.push_message(&entry) { tracing::warn!(doc = %self.doc_id, error = %err, "subagent steer write failed"); @@ -1948,6 +3116,7 @@ fn sync_segment<'a>( Ok(()) } +#[allow(clippy::too_many_arguments)] fn finish_segment<'a>( doc: &'a SessionDoc, writer: Option>, @@ -1956,15 +3125,26 @@ fn finish_segment<'a>( started_at: i64, folded: &[MessagePart], status: MessageStatus, + turn_ref: Option<&roboco_proto::ProviderTurnRef>, ) -> Result<(), DocError> { let rendered = render_parts(folded); - match writer { + let outcome = match writer { Some(w) => w.finish(&rendered, status), None if !folded.is_empty() => { SegmentWriter::begin(doc, entry_id, device_id, started_at)?.finish(&rendered, status) } None => Ok(()), + }?; + // The provider turn the finished entry belongs to (ticket 17): stamped + // after the write, a no-op for an entry that never streamed. A failure + // here costs the ref (a fork falls back to the budgeted paste), never + // the segment. + if let Some(reference) = turn_ref + && let Err(err) = doc.stamp_provider_turn(entry_id, reference) + { + tracing::warn!(chat = %doc.chat_id().unwrap_or_default(), error = %err, "provider turn ref stamp failed"); } + Ok(outcome) } /// Resume bookkeeping for one run task: which user entry the run answers (so @@ -2043,6 +3223,131 @@ fn cursor_unstarted_history( )) } +/// Whether `event` carries ticket 05's signed-out error kind — directly, or +/// tagged inside a subagent event (a subagent runs on the same login, so its +/// rejection is equally hard evidence the login is dead). +fn event_signs_out(event: &AgentEvent) -> bool { + let leaf = match event { + AgentEvent::Subagent { event, .. } => event.as_ref(), + event => event, + }; + matches!( + leaf, + AgentEvent::Error { + kind: Some(AgentErrorKind::SignedOut), + .. + } + ) +} + +/// The parent the run's CURRENT turn owes a completion notice to, read at +/// the engine's own turn-end states (orchestration v2). +fn run_notifiable_parent(inner: &Inner, chat_id: &str, run_id: &str) -> Option { + lock(&inner.runs) + .get(chat_id) + .filter(|h| h.run_id == run_id) + .and_then(|h| lock(&h.notifiable).clone()) +} + +/// Record a completion-notice outcome, fire-and-forget: the side chat's +/// event stream never waits on the parent's notice turn. `settle` is the +/// side chat's settle serial stamped before its settled status published +/// (ticket 14) — the acknowledgement compares against it. Engine teardown +/// records nothing — the runs it kills were not user-visible outcomes; +/// a restart's lost runs are covered at boot (interrupted outcomes), +/// where the crash state is durable enough to read. +fn spawn_notice_record( + inner: Arc, + side_chat: String, + parent: String, + settle: u64, + outcome: crate::completion_notices::Outcome, +) { + if inner + .shutting_down + .load(std::sync::atomic::Ordering::Acquire) + { + return; + } + let engine = SessionsEngine { inner }; + tokio::spawn(async move { + crate::completion_notices::record_and_deliver(&engine, &side_chat, &parent, settle, outcome).await; + }); +} + +/// Deliver the completion notices `chat_id` is owed as a PARENT, now that +/// its turn has ended (ticket 13's busy-parent rule): the outcomes that +/// piled up while it worked were never steered in, so they start the +/// chat's next turn now. Fire-and-forget, like the records: the run loop +/// never waits on another chat's turn. Delivery re-checks `turn_in_flight` +/// (a queued user message may have claimed the boundary first — then the +/// notice waits for that turn's own end) and `shutting_down` (teardown +/// never starts recovered work). A turn ended by an interrupt is the +/// user's Stop: no wakeup — ticket 14 owns the stopped-parent rule. +fn spawn_notice_delivery(inner: &Arc, chat_id: &str) { + if inner + .shutting_down + .load(std::sync::atomic::Ordering::Acquire) + { + return; + } + let engine = SessionsEngine { + inner: inner.clone(), + }; + let chat = chat_id.to_string(); + tokio::spawn(async move { + crate::completion_notices::deliver(&engine, &chat).await; + }); +} + +/// Record the outcome of a turn that settled INSIDE the run loop (a parked +/// persistent session, the quiesce watchdog): only real turn ends record — +/// an internal handoff (a pending steer or a sent message owns the +/// continuation) notifies nothing. `settle` is the settle serial the +/// caller stamped before the settled status published (ticket 14). +fn record_parked_turn( + inner: &Arc, + chat_id: &str, + run_id: &str, + completed: bool, + stopped_in_place: bool, + settle: u64, +) { + let outcome = if stopped_in_place { + Some(crate::completion_notices::Outcome::Stopped) + } else if completed { + Some(crate::completion_notices::Outcome::Completed) + } else { + None + }; + if let Some(outcome) = outcome + && let Some(parent) = run_notifiable_parent(inner, chat_id, run_id) + { + spawn_notice_record(inner.clone(), chat_id.to_string(), parent, settle, outcome); + } +} + +/// The outcome for the run's FINAL turn-end, from the engine's own recorded +/// state: the Done status and error, and whether the completion marker was +/// minted (a Done with a pending steer or a sent message on its way is an +/// internal handoff, not a completion). +fn final_outcome( + status: DoneStatus, + error: Option<&str>, + completed_turn: bool, +) -> Option { + match status { + DoneStatus::Completed if completed_turn => { + Some(crate::completion_notices::Outcome::Completed) + } + DoneStatus::Completed => None, + DoneStatus::Errored => Some(crate::completion_notices::Outcome::Errored { + reason: crate::completion_notices::one_line(error), + }), + DoneStatus::Interrupted => Some(crate::completion_notices::Outcome::Stopped), + } +} + #[allow(clippy::too_many_arguments)] async fn drive_run( inner: Arc, @@ -2072,7 +3377,9 @@ async fn drive_run( } // Kept whole for the startup-crash retry (same user entry; dispatch // re-injects the stored resume id). Option so the retry branch (inside - // the event loop) can take ownership. + // the event loop) can take ownership. The native-fork directive spreads + // in from `request` — a startup crash is transient, so the retry + // re-attempts the fork and re-derives the fallback history. let mut retry_request = Some(RunRequest { resume: None, ..request.clone() @@ -2080,23 +3387,51 @@ async fn drive_run( // A side chat owns a fresh provider session. Bootstrap it from the frozen // conversation, never resume (and mutate) the parent's provider session. let mut carried_history = false; - let provider = match request.resume.as_deref() { - None => ProviderSession::Fresh, - resumed => ProviderSession::Continued(resumed), - }; - if let Some(prompt) = inner.fork_history_prompt( - &chat_id, - &doc, - harness_id, - &request.prompt, - Some(&resume_state.user_message_id), - provider, - ) { - request.prompt = prompt; + if let Some(fork) = request.provider_fork.as_mut() { + // A native fork (t3-orchestration-v2 ticket 17): the driver forks the + // source provider session at the boundary, so the copied history + // rides the provider and the delivered prompt carries nothing. The + // budgeted transcript still rides the request — the driver pastes it + // in front of the prompt on the fresh session it starts when the + // harness refuses the fork. Either way the session this run reports + // holds the copied history, so warm sends never re-deliver it. + if fork.fallback_history.is_none() { + // The Fresh selection: a session the fork refused to continue + // knows nothing of the chat, so the fallback is the whole + // budgeted transcript (the seam and everything after it are + // structure, not transcript). + fork.fallback_history = inner.fork_history_prefix( + &chat_id, + &doc, + harness_id, + &request.prompt, + Some(&resume_state.user_message_id), + ProviderSession::Fresh, + ); + } carried_history = true; resume_state .fork_history_sent .store(true, std::sync::atomic::Ordering::Release); + } else { + let provider = match request.resume.as_deref() { + None => ProviderSession::Fresh, + resumed => ProviderSession::Continued(resumed), + }; + if let Some(prompt) = inner.fork_history_prompt( + &chat_id, + &doc, + harness_id, + &request.prompt, + Some(&resume_state.user_message_id), + provider, + ) { + request.prompt = prompt; + carried_history = true; + resume_state + .fork_history_sent + .store(true, std::sync::atomic::Ordering::Release); + } } // Startup can stop before the SDK saves user text, with no new session // ID or receipt. Bridge that unacknowledged tail from our transcript; @@ -2138,6 +3473,24 @@ async fn drive_run( if !matches!(harness_id, HarnessId::Cursor | HarnessId::Opencode) { wire_request.prompt = roboco_proto::invocation::harness_prompt(&wire_request.prompt, harness_id); + // The fork fallback rides the same conversion the + // pasted prompt would get (Codex: file mentions), so a + // refused fork's fresh session reads a transcript as + // close to the native paste as the driver can deliver. + // The resume fallback (ticket 09's account-failover + // continuation) converts the same way. + if let Some(fork) = wire_request.provider_fork.as_mut() + && let Some(history) = fork.fallback_history.clone() + { + fork.fallback_history = Some( + roboco_proto::invocation::harness_prompt(&history, harness_id), + ); + } + if let Some(history) = wire_request.resume_fallback_history.clone() { + wire_request.resume_fallback_history = Some( + roboco_proto::invocation::harness_prompt(&history, harness_id), + ); + } } harness.run(wire_request, controls).await } @@ -2179,6 +3532,8 @@ async fn drive_run( now_ms(), &parts, MessageStatus::Complete, + // The run never started: no provider turn ever streamed. + None, ) { tracing::warn!(chat = %chat_id, error = %err, "start failure entry failed"); } @@ -2186,6 +3541,7 @@ async fn drive_run( &chat_id, &AgentEvent::Error { message: message.clone(), + kind: None, }, ); inner.publish( @@ -2193,12 +3549,27 @@ async fn drive_run( &AgentEvent::Done { status: DoneStatus::Errored, result: None, - error: Some(message), + error: Some(message.clone()), session_id: None, }, ); + // Completion notices (orchestration v2): a notifiable turn that + // never started is still a turn the parent is owed — it errored. + // The settle serial stamps before the errored status publishes + // (ticket 14): a wait resolving off that flip observes it. + let settle = inner.notices.settle(&chat_id); + if let Some(parent) = run_notifiable_parent(&inner, &chat_id, &run_id) { + let outcome = crate::completion_notices::Outcome::Errored { + reason: crate::completion_notices::one_line(Some(message.as_str())), + }; + spawn_notice_record(inner.clone(), chat_id.clone(), parent, settle, outcome); + } inner.remove_run(&chat_id, &run_id); inner.set_status(&chat_id, SessionStatus::Errored, false); + // The chat's own turn is over (it errored before it started): + // notices that piled up while it was working start its next + // turn now (ticket 13). + spawn_notice_delivery(&inner, &chat_id); return; } }; @@ -2226,6 +3597,11 @@ async fn drive_run( let mut prepared_events = std::collections::VecDeque::new(); let mut entry_id = new_id(); let mut segment_started = now_ms(); + // The provider turn the entry currently streaming belongs to (ticket 17): + // set by a TurnRef event, stamped onto the entry when its segment + // finishes, so a fork can continue the provider session at exactly that + // turn. Survives steer boundaries — a steer mid-turn keeps the turn. + let mut turn_ref: Option = None; let mut writer: Option> = None; let mut dirty = false; let mut flush_at = tokio::time::Instant::now(); @@ -2234,6 +3610,10 @@ async fn drive_run( let mut interrupt_deadline: Option = None; let mut interrupted = false; let mut saw_session_started = false; + // One signed-out mark per run: repeats and retries all describe the same + // dead login, and each marking pass resolves the live login (a detection + // sweep) — once is enough. + let mut marked_signed_out = false; // Liveness heartbeat: this loop RUNNING is proof the harness stream is // open, so freshness must not depend on events arriving. Silent stretches // are normal and UNBOUNDED — a long tool call, redacted thinking, an @@ -2346,6 +3726,15 @@ async fn drive_run( let mut last_subagent_activity: Option = None; let mut final_completed_turn = None; + // The run's LAST error classification. Harnesses emit the classified + // `AgentEvent::Error` (usage limit / signed out) before their terminal + // `Done`; when that Done is Errored, this is what turns the settled row + // into a Limited one (the record on the session status). + let mut last_error_kind: Option = None; + // The run's FINAL Done (status + error), captured at the break that ends + // the run — the engine's own recorded turn-end state for the completion + // notice. Parked settles record inside the loop instead. + let mut final_done: Option<(DoneStatus, Option)> = None; let final_status = loop { let event: AgentEvent = if let Some(event) = prepared_events.pop_front() { event @@ -2542,6 +3931,7 @@ async fn drive_run( segment_started, &folded, MessageStatus::Complete, + turn_ref.as_ref(), ) { tracing::warn!(chat = %chat_id, error = %err, "quiesce segment finish failed"); } @@ -2554,9 +3944,28 @@ async fn drive_run( idle_since = Some(tokio::time::Instant::now()); reap_deferred_until = None; self_continued_turn = false; + // The watchdog finalized a completed turn (no native + // Done): same rule as a Done — the turn's ref must not + // leak into whatever turn follows in this runtime. + turn_ref = None; + let settle = inner.notices.settle(&chat_id); inner.set_status_with_completion( - &chat_id, SessionStatus::Idle, false, completed_turn, + &chat_id, SessionStatus::Idle, false, completed_turn.clone(), ); + // The watchdog finalized a completed turn (no native + // Done): it owes its parent the same notice a Done would. + record_parked_turn( + &inner, + &chat_id, + &run_id, + completed_turn.is_some(), + false, + settle, + ); + // The watchdog-finalized turn is over: notices that + // piled up while the chat worked start its next turn + // (ticket 13). + spawn_notice_delivery(&inner, &chat_id); continue; } }; @@ -2572,6 +3981,25 @@ async fn drive_run( event }; + // Sign-out evidence (ticket 05's structured kind, ticket 08's + // marking): the harness rejected its login — the login is dead, not + // stale. Checked BEFORE the subagent routing and the parked gate so + // every arrival shape counts (a subagent running on the same login + // fails signed-out just as hard). Spawned, not awaited: the mark + // queues behind the accounts ops lock, which a usage probe can hold + // across a network round trip — awaiting it inline stalls this loop + // (streaming, Done) for seconds. The run's own terminal state + // settles independently; the failover watcher re-checks the mark + // after the settle, bounded, so a mark that lands late is still seen. + if !marked_signed_out && event_signs_out(&event) { + marked_signed_out = true; + let inner = inner.clone(); + let chat = chat_id.clone(); + tokio::spawn(async move { + inner.note_run_signed_out(&chat, harness_id).await; + }); + } + // ── subagent routing ─────────────────────────────────────────── // Tagged events NEVER fold into the parent transcript: they stream // into the subagent's own doc, and the parent keeps only the spawn @@ -2936,7 +4364,7 @@ async fn drive_run( // The user entry write inside dispatch is idempotent by // message id; `startup_retry` makes this attempt final. if let Err(err) = engine - .dispatch_with(&chat, harness_id, retry, Some(message_id), true) + .dispatch_with(&chat, harness_id, retry, Some(message_id), None, true, None) .await { tracing::error!(chat = %chat, error = %err, "startup-crash retry dispatch failed"); @@ -2956,6 +4384,20 @@ async fn drive_run( .. } = &event { + // The boundary confirms delivery of the oldest accepted steer — + // retire its at-least-once ledger entry FIRST, and record the + // consumption durably before the boundary is even journaled: the + // harness has already consumed this input, so a crash from here + // on must leave boot auto-resume able to tell — or the revival + // would replay the steer and the model would answer it twice + // (restart correctness, orchestration v2 ticket 04). + let confirmed = lock(&inner.runs) + .get(&chat_id) + .filter(|h| h.run_id == run_id) + .and_then(|h| lock(&h.routed_steers).pop_front()); + if let Some(steer) = &confirmed { + inner.journal.note_steer_consumed(&chat_id, &steer.message_id); + } inner.publish(&chat_id, &event); // A steer boundary means a real prompt owns the turn again — its // Done will come; the short self-continued window stands down. @@ -2968,6 +4410,11 @@ async fn drive_run( segment_started, &folded, MessageStatus::Complete, + // The segment closes at the steer, but the provider turn it + // belongs to carries on: the ref survives the rotation so the + // NEXT segment of the same turn keeps it (a new turn's TurnRef + // replaces it before that segment finishes). + turn_ref.as_ref(), ) { tracing::warn!(chat = %chat_id, error = %err, "segment finish failed"); } @@ -2980,12 +4427,6 @@ async fn drive_run( // steer boundary restarts it (matches the parked-resume path and // the composer's optimistic overlay, which already reads 0:00). inner.set_status(&chat_id, SessionStatus::Working, true); - // The boundary confirms delivery of the oldest accepted steer — - // retire its at-least-once ledger entry. - let confirmed = lock(&inner.runs) - .get(&chat_id) - .filter(|h| h.run_id == run_id) - .and_then(|h| lock(&h.routed_steers).pop_front()); // Consumed: a steer carrying the fork history delivered it to // this runtime's provider session. if confirmed.is_some_and(|steer| steer.fork_history) @@ -3006,24 +4447,88 @@ async fn drive_run( saw_session_started = true; // The event's own cwd (where the harness actually created the // session) scopes the stored id, not the request's. - inner.remember_harness_session(&chat_id, session_id, cwd); + inner.remember_harness_session(&chat_id, session_id, cwd, Some(harness_id)); note_fork_history_session(&doc, carried_history, session_id); } + AgentEvent::TurnRef { reference } => { + // The provider turn now streaming into `entry_id`: held for + // the stamp at segment finish (below). Like SessionStarted, + // it is engine bookkeeping — the fold below renders nothing + // for it. + turn_ref = Some(reference.clone()); + } AgentEvent::Done { session_id: Some(session_id), .. } => { - inner.remember_harness_session(&chat_id, session_id, &run_cwd); + inner.remember_harness_session(&chat_id, session_id, &run_cwd, Some(harness_id)); note_fork_history_session(&doc, carried_history, session_id); } - AgentEvent::InputRequested { .. } => { + AgentEvent::InputRequested { questions, .. } => { // Known-id guaranteed: the unknown-id twin was dropped above, // before the parked gate. + let was_awaiting = lock(&inner.statuses) + .get(&chat_id) + .is_some_and(|s| s.status == SessionStatus::AwaitingInput); + // Stamp the settle serial before the awaiting status + // publishes (ticket 14): a wait resolving at this flip — or + // a read after it — has observed the question this outcome + // reports. Only a NEW awaiting stretch settles; a twin + // question does not. + let settle = (!was_awaiting).then(|| inner.notices.settle(&chat_id)); inner.set_status(&chat_id, SessionStatus::AwaitingInput, false); + // A notifiable turn reaching awaiting input owes its parent a + // notice with a one-line summary of the question — once per + // awaiting stretch, not per twin question. + if let Some(settle) = settle + && let Some(parent) = run_notifiable_parent(&inner, &chat_id, &run_id) + { + let summary = crate::completion_notices::question_summary(questions); + spawn_notice_record( + inner.clone(), + chat_id.clone(), + parent, + settle, + crate::completion_notices::Outcome::WaitingForInput { summary }, + ); + } } AgentEvent::InputResolved { .. } => { inner.set_status(&chat_id, SessionStatus::Working, false); } + AgentEvent::Error { kind, .. } => { + // Track the latest classification; a later unclassified + // error supersedes an earlier limit one (the turn then ended + // on that failure, not the limit). A later limit that is + // LESS specific than the earlier one never wins: Claude's + // terse `rate_limit` code (window "usage", no reset) lands + // after the detailed `rate_limit_event` that named the + // window and the reset — keep the earlier detail when the + // later lacks it. + last_error_kind = match (last_error_kind.take(), kind.clone()) { + ( + Some(AgentErrorKind::UsageLimit { + window: earlier_window, + resets_at: earlier_reset, + }), + Some(AgentErrorKind::UsageLimit { + window: later_window, + resets_at: later_reset, + }), + ) => { + let window = if later_window != "usage" || earlier_window == "usage" { + later_window + } else { + earlier_window + }; + Some(AgentErrorKind::UsageLimit { + window, + resets_at: later_reset.or(earlier_reset), + }) + } + (_, later) => later, + }; + } _ => {} } @@ -3042,7 +4547,7 @@ async fn drive_run( // still in place and tested. } - if let AgentEvent::Done { status, .. } = &event { + if let AgentEvent::Done { status, error, .. } = &event { // A question still pending at turn end can never be legitimately // answered (its turn is over): drain the resolvers NOW, or a late // `respond_input` finds one, emits InputResolved, and un-parks @@ -3084,6 +4589,7 @@ async fn drive_run( segment_started, &folded, message_status, + turn_ref.as_ref(), ) { tracing::warn!(chat = %chat_id, error = %err, "final segment finish failed"); } @@ -3093,6 +4599,10 @@ async fn drive_run( // A cleanly completed turn resets the auto-resume revival // budget: only consecutive crash-revive-crash cycles spend it. inner.journal.clear_resume_attempts(&chat_id); + // …and closes the consumed-steer record with it: the record + // only ever speaks for the turn a crash interrupts, so it + // stays bounded by one turn's steers. + inner.journal.clear_consumed_steers(&chat_id); } // Exchange completed on an untitled chat → name it (fire-and-forget; // interrupted/errored turns never trigger naming). @@ -3145,15 +4655,42 @@ async fn drive_run( idle_since = Some(tokio::time::Instant::now()); reap_deferred_until = None; self_continued_turn = false; + // The turn is over: its provider turn ref must not leak into + // the next turn's segments. Harnesses that emit the ref + // before their segments (Claude, Codex, OpenCode) re-stamp + // it at the next turn's start; Pi emits its ref at settle, so + // without this clear a steer-split reply in the NEXT turn + // would carry THIS turn's ref (ticket 17's fork boundary + // would cut the wrong place). + turn_ref = None; + // The settle serial stamps before the idle status publishes + // (ticket 14): an observer of the parked turn's completion + // has observed this settle. + let settle = inner.notices.settle(&chat_id); inner.set_status_with_completion( &chat_id, SessionStatus::Idle, false, - completed_turn, + completed_turn.clone(), ); + // The parked turn settled: a completed turn notifies its + // parent, a stop-in-place notifies as stopped, an internal + // handoff (a pending steer owns the continuation) does not. + record_parked_turn( + &inner, + &chat_id, + &run_id, + completed_turn.is_some(), + stopped_in_place, + settle, + ); + // The parked turn is over: notices that piled up while the + // chat worked start its next turn (ticket 13). + spawn_notice_delivery(&inner, &chat_id); continue; } final_completed_turn = completed_turn; + final_done = Some((*status, error.clone())); break match status { DoneStatus::Errored => SessionStatus::Errored, _ => SessionStatus::Idle, @@ -3194,8 +4731,62 @@ async fn drive_run( .filter(|h| h.run_id == run_id) .map(|h| std::mem::take(&mut *lock(&h.routed_steers)).into()) .unwrap_or_default(); + // A turn that ended on a usage limit records the additive limit row + // (account + reset-time fill) while this run still owns the chat — the + // accounts lookup is the only await in the exit path, so it runs before + // the handle goes away rather than racing a fresh dispatch's status. + let limit = match (final_status, last_error_kind) { + (SessionStatus::Errored, Some(kind)) => { + inner.limit_record(harness_id, kind).await + } + _ => None, + }; + if limit.is_some() { + // Queued messages wait BEHIND the continuation instead of dispatching + // into the same wall: a limited chat holds its queue exactly like a + // Stop does, and the next explicit prompt (Resume's continue prompt, + // or the user's own message) thaws it when its run dispatches. + if let Some(host) = inner.doc_host() + && let Ok(handle) = host.open(&chat_id) + { + handle.pause_queue(); + } + } + // Completion notices (orchestration v2): the run's final turn ended — + // read the notifiable parent before the handle goes away, then record + // the outcome the engine's own state observed. The settle serial stamps + // before the final status publishes (ticket 14): a wait resolving off + // that flip — or a read after it — has observed this turn's outcome. + let notifiable = run_notifiable_parent(&inner, &chat_id, &run_id); + let final_settle = inner.notices.settle(&chat_id); inner.remove_run(&chat_id, &run_id); - inner.set_status_with_completion(&chat_id, final_status, false, final_completed_turn); + let final_turn_completed = final_completed_turn.is_some(); + let auto_arm_limit = limit.clone(); + inner.set_status_with_limit(&chat_id, final_status, false, final_completed_turn, limit); + if let Some(limit) = auto_arm_limit + .as_ref() + .filter(|_| final_status == SessionStatus::Errored) + { + // Auto-resume limited chats (orchestration v2, ticket 07): the fresh + // limit record arms the continuation when the setting is on — only + // ever via a NEW record whose reset advances past the last fired + // continuation (the loop guard), never from the failure of the + // continuation itself. + SessionsEngine { inner: inner.clone() }.maybe_auto_arm_resume(&chat_id, limit); + } + if let (Some(parent), Some((status, error))) = (notifiable, final_done) { + let completed_turn = final_turn_completed; + if let Some(outcome) = final_outcome(status, error.as_deref(), completed_turn) { + spawn_notice_record(inner.clone(), chat_id.clone(), parent, final_settle, outcome); + } + } + // A busy parent's notices waited — never steered mid-turn (ticket 13): + // its turn is over, so they start its next one now. A turn ended by an + // interrupt is the user's Stop: no wakeup (ticket 14 owns the full + // stopped-parent rule). + if !interrupted { + spawn_notice_delivery(&inner, &chat_id); + } // A Stop cancels the messages it found waiting; one accepted after it // began is still the user's next message and must run. let orphans: Vec = if inner @@ -3243,6 +4834,8 @@ async fn drive_run( harness_id, request, Some(steer.message_id.clone()), + steer.origin_chat_id.clone(), + None, ) .await { @@ -3371,6 +4964,7 @@ mod tests { .link(); let previous = format!("Previous {skill}\nSecond line with \\ and \"quotes\""); doc.push_message(&roboco_doc::SessionMessageEntry { + origin_chat_id: None, id: "u1".into(), role: roboco_doc::MessageRole::User, parts: vec![roboco_doc::MessagePart::Text { @@ -3382,6 +4976,7 @@ mod tests { status: None, continuation_of: None, duration_ms: None, + provider_turn: None, }) .unwrap(); let current = format!("Current {skill}\nKeep **Markdown**"); @@ -3412,6 +5007,7 @@ mod tests { ("u3", roboco_doc::MessageRole::User, "future pending request"), ] { doc.push_message(&roboco_doc::SessionMessageEntry { + origin_chat_id: None, id: id.into(), role, parts: vec![roboco_doc::MessagePart::Text { @@ -3423,6 +5019,7 @@ mod tests { status: None, continuation_of: None, duration_ms: None, + provider_turn: None, }) .unwrap(); } @@ -3604,6 +5201,8 @@ mod tests { sandbox: SandboxLevel::WorkspaceWrite, auto_approve: true, resume: None, + provider_fork: None, + resume_fallback_history: None, attachments: Vec::new(), worktree: None, mcp: None, @@ -3802,7 +5401,7 @@ mod tests { .unwrap(); let chat = "chat-sink-spin"; core.sessions - .dispatch(chat, HarnessId::Mock, request(), None) + .dispatch(chat, HarnessId::Mock, request(), None, None, None) .await .unwrap(); feed.send(AgentEvent::SessionStarted { diff --git a/crates/engine/src/titles.rs b/crates/engine/src/titles.rs index 99984d5b69..310fd52d2d 100644 --- a/crates/engine/src/titles.rs +++ b/crates/engine/src/titles.rs @@ -210,6 +210,8 @@ impl TitleGenerator { auto_approve: false, attachments: Vec::new(), resume: None, + provider_fork: None, + resume_fallback_history: None, worktree: None, mcp: None, }; @@ -298,7 +300,7 @@ async fn collect_text( "title generation attempted to use a tool".into(), )); } - AgentEvent::Error { message } => { + AgentEvent::Error { message, .. } => { return Err(EngineError::Other(format!("titling run error: {message}"))); } AgentEvent::Done { status, error, .. } => { @@ -380,6 +382,8 @@ mod tests { resume: None, attachments: vec![], worktree: None, + provider_fork: None, + resume_fallback_history: None, mcp: None, }; let result = collect_text(&harness, request, None).await; diff --git a/crates/engine/src/transcript_budget.rs b/crates/engine/src/transcript_budget.rs new file mode 100644 index 0000000000..314f5e0298 --- /dev/null +++ b/crates/engine/src/transcript_budget.rs @@ -0,0 +1,765 @@ +//! Budgeted transcript selection: the messages delivered in front of a +//! prompt that starts a provider session lacking the chat's prior context. +//! +//! A forked side chat's first prompt used to paste the entire source +//! transcript — every tool call included — as raw JSON in front of the new +//! message, with no size limit: long chats burned the new session's context +//! before it started (t3-orchestration-v2 spec story 58, ticket 01). The +//! transcript in front of the message is now a budgeted selection: +//! +//! 1. the latest user message, the latest assistant message and the first +//! user message; +//! 2. then the newest remaining messages, newest first, until the budget is +//! spent. +//! +//! **Contents:** user and assistant text, command runs (input, exit code, +//! output), errors and file-change names. Reasoning (the model's private +//! scratchpad) and attachments are left out. Long items are truncated. +//! +//! **Budget:** 16k tokens, counted at one token per UTF-8 byte — a +//! deliberately pessimistic proxy, not a tokenizer. It bounds everything the +//! wrapper puts in front of the new user message; the new message itself is +//! never counted and never shortened. +//! +//! A transcript that fits the budget whole is delivered complete, with no +//! note. One that does not carries a **coverage header** naming how many +//! messages were omitted and how to read them back with the `read_chat` MCP +//! tool using the source chat's id. +//! +//! One engine piece on purpose: every path that must hand a chat's prior +//! transcript to a session that lacks it reuses +//! [`budgeted_transcript_prompt`] (or its prefix form, +//! [`budgeted_transcript_prefix`], which the native-fork fallback rides) — +//! the fork bootstrap (ticket 01), the account-failover fallback (ticket +//! 09), the native-fork fallbacks (17–20), forks into another harness (22) +//! and the one-way harness switch (24). + +use roboco_doc::{join_continuation_entries, MessagePart, MessageRole, SessionMessageEntry}; +use roboco_proto::{ToolCall, ToolDiff}; + +/// The transcript budget: 16k "tokens" at one token per UTF-8 byte, bounding +/// everything delivered in front of the new user message (wrapper, coverage +/// header and transcript — never the new message itself). +pub const TRANSCRIPT_BUDGET_BYTES: usize = 16 * 1024; + +/// Cap for any single delivered message once the transcript must be +/// selected: no one message may eat more than a quarter of the budget, so +/// the three priority messages always fit together with room for recent +/// context. A transcript that fits whole is never truncated, whatever this +/// cap says. +const ITEM_CAP_BYTES: usize = TRANSCRIPT_BUDGET_BYTES / 4; + +/// Smallest truncated delivery worth its bytes: below this an item is +/// omitted rather than shipped as a meaningless stub. +const MIN_ITEM_BYTES: usize = 64; + +/// The lead is shared by every budgeted-history use (ticket 01): a side +/// chat's fork bootstrap, a main chat's harness switch, and an account +/// failover's fresh-session fallback — so it names neither of them, only +/// the chat and the transcript. +const LEAD: &str = "Continue this chat using the following prior transcript as context."; +const TRANSCRIPT_OPEN: &str = "\n"; +const TRANSCRIPT_CLOSE: &str = "\n\n\n"; +const TRUNCATION_MARKER: &str = " …[truncated]"; + +/// One transcript entry rendered for handoff: its role plus the parts the +/// contents policy keeps. +struct TranscriptItem { + role: MessageRole, + text: String, +} + +/// The delivered prompt: the budgeted transcript of `entries` in front of +/// `prompt`, or `None` when the entries carry no deliverable messages (the +/// caller then sends `prompt` as is). +/// +/// `source_chat_id` is the chat whose transcript the omitted messages are +/// readable in — the fork's source for the fork bootstrap, the chat itself +/// for harness switches and account failovers. +#[cfg(test)] +pub fn budgeted_transcript_prompt( + entries: &[SessionMessageEntry], + source_chat_id: Option<&str>, + prompt: &str, +) -> Option { + budgeted_transcript_prefix(entries, source_chat_id) + .map(|prefix| format!("{prefix}{prompt}")) +} + +/// The budgeted transcript alone (ticket 17): everything +/// [`budgeted_transcript_prompt`] would put in front of the new user +/// message, without the message. The native-fork path rides this as the +/// request's fallback history — the driver pastes it in front of the prompt +/// only when the harness refuses the fork, so a successful fork pastes +/// nothing at all. +pub fn budgeted_transcript_prefix( + entries: &[SessionMessageEntry], + source_chat_id: Option<&str>, +) -> Option { + let items = transcript_items(entries); + let lines: Vec = items + .iter() + .map(|item| format!("{}: {}", role_label(item.role), item.text)) + .collect(); + if lines.is_empty() { + return None; + } + let wrapper_fixed = LEAD.len() + 1 + TRANSCRIPT_OPEN.len() + TRANSCRIPT_CLOSE.len(); + let transcript = lines.join("\n"); + if wrapper_fixed + transcript.len() <= TRANSCRIPT_BUDGET_BYTES { + // The whole transcript fits: complete, verbatim, no coverage note. + return Some(wrap(None, &lines)); + } + + // Selection mode. The header's length depends on the omitted count, so + // reserve its worst case up front (the largest count is all items); + // intermediate selections then cannot grow past the budget. + let header_max = coverage_header_max(items.len(), source_chat_id); + let mut remaining = TRANSCRIPT_BUDGET_BYTES + .saturating_sub(wrapper_fixed) + .saturating_sub(header_max); + let mut delivered: Vec> = vec![None; lines.len()]; + + // Priorities, in order: the latest user message, the latest assistant + // message, the first user message. Never dropped while any room remains + // — an oversized one is truncated to the smaller of the per-item cap and + // what is left. + let last_user = items.iter().rposition(|item| item.role == MessageRole::User); + let last_assistant = items + .iter() + .rposition(|item| item.role == MessageRole::Assistant); + let first_user = items.iter().position(|item| item.role == MessageRole::User); + for idx in [last_user, last_assistant, first_user].into_iter().flatten() { + if delivered[idx].is_some() { + continue; + } + let cap = ITEM_CAP_BYTES.min(remaining.saturating_sub(1)); + if cap < MIN_ITEM_BYTES { + break; // nothing meaningful fits any more + } + let line = truncate_line(&lines[idx], cap); + remaining -= line.len() + 1; + delivered[idx] = Some(line); + } + + // Fill with the newest remaining messages, newest first. An item that + // does not fit (after its per-item cap) is skipped whole — counted as + // omitted — and older items still get their chance. + for idx in (0..lines.len()).rev() { + if delivered[idx].is_some() { + continue; + } + let Some(line) = fill_line(&lines[idx], remaining) else { + continue; + }; + remaining -= line.len() + 1; + delivered[idx] = Some(line); + } + + let omitted = delivered.iter().filter(|slot| slot.is_none()).count(); + let selected: Vec = delivered.into_iter().flatten().collect(); + if selected.is_empty() { + // Unreachable with the shipped budget (three capped priorities + // always fit); degrade to the verbatim transcript rather than send + // an empty block. + return Some(wrap(None, &lines)); + } + let header = coverage_header(omitted, source_chat_id); + Some(wrap(Some(header), &selected)) +} + +/// Render transcript entries into handoff items. Continuation entries are +/// fragments of one logical message (entries split at `MSG_INLINE_MAX`), so +/// they are joined first: the budget and the omission count must see +/// messages, not fragments. System entries (fork seams, system notices) are +/// structure, not transcript, and never render. +fn transcript_items(entries: &[SessionMessageEntry]) -> Vec { + let joined = join_continuation_entries(entries.to_vec()); + let mut out = Vec::with_capacity(joined.len()); + for entry in joined { + if entry.role == MessageRole::System { + continue; + } + let text = render_parts(&entry.parts); + if text.trim().is_empty() { + continue; + } + out.push(TranscriptItem { + role: entry.role, + text, + }); + } + out +} + +/// The delivered label for a role. +fn role_label(role: MessageRole) -> &'static str { + match role { + MessageRole::User => "user", + MessageRole::Assistant => "assistant", + // Unreachable: system entries never become items. + MessageRole::System => "system", + } +} + +/// The parts of one entry, joined: the contents policy in code. +fn render_parts(parts: &[MessagePart]) -> String { + let mut blocks: Vec = Vec::new(); + for part in parts { + let block = match part { + MessagePart::Text { text, .. } => { + if text.trim().is_empty() { + continue; + } + text.clone() + } + MessagePart::Error { message, .. } => format!("Error: {message}"), + MessagePart::Tool { .. } => render_command_run(part), + // Reasoning is the model's private scratchpad; images are + // attachments; input requests, fork seams, harness-switch seams + // and completion notices are structure — none of them belong in + // a budgeted summary. + MessagePart::Reasoning { .. } + | MessagePart::Image { .. } + | MessagePart::Input { .. } + | MessagePart::Fork { .. } + | MessagePart::HarnessSwitch { .. } + | MessagePart::CompletionNotice { .. } => continue, + }; + blocks.push(block); + } + blocks.join("\n") +} + +/// One command run: the input, the exit code, the output, the file changes. +/// +/// The doc records no numeric exit status — the fold keeps the failure flag, +/// which carries the same signal for a handoff (`0` ok, `1` failed, +/// `unknown` while the call is unresolved). `output` is the persisted +/// one-line summary; full outputs live outside the doc. `diff_stats` (and +/// the legacy inline `diff`) name the files the run changed. +fn render_command_run(part: &MessagePart) -> String { + let MessagePart::Tool { + call, + is_error, + resolved, + output, + diff, + diff_stats, + .. + } = part + else { + return String::new(); + }; + let mut lines = vec![format!("Command: {}", describe_call(call))]; + let exit = match (*resolved, *is_error) { + (true, false) => "0", + (true, true) => "1", + (false, _) => "unknown", + }; + lines.push(format!("Exit code: {exit}")); + if let Some(output) = output.as_deref().filter(|output| !output.trim().is_empty()) { + lines.push(output.to_owned()); + } + let mut paths: Vec<&str> = Vec::new(); + if let Some(stats) = diff_stats { + paths.extend(stats.iter().map(|stat| stat.path.as_str())); + } + if let Some(diff) = diff { + paths.push(diff.path.as_str()); // pre-strip docs kept inline diffs + } + if !paths.is_empty() { + lines.push(format!("File changes: {}", paths.join(", "))); + } + lines.join("\n") +} + +/// Compact rendering of a (already privacy-sanitized) tool call — the input +/// line of a command run. Heavy inputs never reach the doc (the render +/// policy strips them), so this stays short by construction. +fn describe_call(call: &ToolCall) -> String { + match call { + ToolCall::Exec { command } => command.clone(), + ToolCall::ReadFile { path } => format!("read {path}"), + ToolCall::WriteFile { path, .. } => format!("write {path}"), + ToolCall::EditFile { path, .. } => format!("edit {path}"), + ToolCall::ApplyPatch { path } => match path { + Some(path) => format!("patch {path}"), + None => "patch".to_owned(), + }, + ToolCall::Search { pattern, .. } => format!("search {pattern}"), + ToolCall::Glob { pattern } => format!("glob {pattern}"), + ToolCall::WebFetch { url, .. } => format!("fetch {url}"), + ToolCall::WebSearch { query } => format!("search web {query}"), + ToolCall::Todo { items } => format!("todo ({} items)", items.len()), + ToolCall::Mcp { server, tool, .. } => format!("{server}/{tool}"), + ToolCall::Unknown { name, .. } => name.clone(), + } +} + +/// A fill candidate: its per-item-capped form, included only when it fits +/// the remaining budget — otherwise the message is omitted whole. +fn fill_line(line: &str, remaining: usize) -> Option { + if remaining <= MIN_ITEM_BYTES { + return None; + } + let capped = truncate_line(line, ITEM_CAP_BYTES); + (capped.len() + 1 <= remaining).then_some(capped) +} + +/// Truncate to at most `cap` bytes (marker included), at a UTF-8 char +/// boundary, marking the cut so the reader knows there was more. +fn truncate_line(line: &str, cap: usize) -> String { + if line.len() <= cap { + return line.to_owned(); + } + let keep = cap.saturating_sub(TRUNCATION_MARKER.len()); + let mut end = keep.min(line.len()); + while end > 0 && !line.is_char_boundary(end) { + end -= 1; + } + let mut out = line[..end].to_owned(); + out.push_str(TRUNCATION_MARKER); + out +} + +/// The coverage header for a partial transcript: how many messages were +/// omitted, and how to read them back. A selection that omitted nothing but +/// truncated something still points at `read_chat` — the cut text is just +/// as recoverable as the omitted messages. +fn coverage_header(omitted: usize, source_chat_id: Option<&str>) -> String { + let lead = if omitted > 0 { + format!("This transcript is partial: {omitted} messages were omitted.") + } else { + "This transcript is partial: long messages were truncated.".to_owned() + }; + match source_chat_id { + Some(chat_id) => format!( + "{lead} Read the full transcript with the `read_chat` MCP tool using chat id \"{chat_id}\"." + ), + None => lead, + } +} + +/// The worst-case header length for a given item count: the largest omitted +/// count is all items (widest counter), plus the count-free truncation +/// variant, plus the trailing newline. +fn coverage_header_max(items: usize, source_chat_id: Option<&str>) -> usize { + coverage_header(items, source_chat_id) + .len() + .max(coverage_header(0, source_chat_id).len()) + + 1 +} + +/// Assemble the budgeted transcript: lead, optional coverage header, and +/// the transcript block — ending with the blank line that separates it +/// from whatever follows (the new user message, whole and always last). +fn wrap(header: Option, lines: &[String]) -> String { + let mut out = + String::with_capacity(LEAD.len() + TRANSCRIPT_OPEN.len() + TRANSCRIPT_CLOSE.len() + 256); + out.push_str(LEAD); + out.push('\n'); + if let Some(header) = header { + out.push_str(&header); + out.push('\n'); + } + out.push_str(TRANSCRIPT_OPEN); + out.push_str(&lines.join("\n")); + out.push_str(TRANSCRIPT_CLOSE); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use roboco_doc::{MessageStatus, ToolDiffStat}; + + fn entry(id: &str, role: MessageRole, parts: Vec) -> SessionMessageEntry { + SessionMessageEntry { + duration_ms: None, + origin_chat_id: None, + provider_turn: None, + id: id.into(), + role, + parts, + created_at: 1, + device_id: "device".into(), + status: Some(MessageStatus::Complete), + continuation_of: None, + } + } + + fn user(id: &str, body: &str) -> SessionMessageEntry { + entry( + id, + MessageRole::User, + vec![MessagePart::Text { + id: format!("{id}-t"), + text: body.into(), + }], + ) + } + + fn assistant(id: &str, body: &str) -> SessionMessageEntry { + entry( + id, + MessageRole::Assistant, + vec![MessagePart::Text { + id: format!("{id}-t"), + text: body.into(), + }], + ) + } + + fn tool_part(id: &str, command: &str) -> MessagePart { + MessagePart::Tool { + id: id.into(), + call: ToolCall::Exec { + command: command.into(), + }, + is_error: false, + resolved: true, + output: None, + diff: None, + output_ref: None, + output_bytes: None, + diff_ref: None, + diff_stats: None, + subagent_ref: None, + subagent_status: None, + subagent_tail: None, + } + } + + const NEW_MESSAGE: &str = "NEW: what next?"; + + /// Everything the wrapper puts in front of the new message (the + /// budgeted part). + fn delivered_prefix(prompt: &str) -> &str { + &prompt[..prompt.len() - NEW_MESSAGE.len()] + } + + /// The count inside "N messages were omitted". + fn omitted_count(prompt: &str) -> usize { + let at = prompt + .find(" messages were omitted") + .expect("coverage header"); + let start = prompt[..at] + .char_indices() + .rev() + .find(|(_, c)| !c.is_ascii_digit()) + .map(|(i, _)| i + 1) + .unwrap_or(0); + prompt[start..at].parse().expect("omitted count") + } + + #[test] + fn a_fitting_transcript_rides_complete_with_no_note() { + let entries = vec![ + user("u1", "Remember PINEAPPLE"), + assistant("a1", "I remember PINEAPPLE"), + user("u2", "And the code?"), + ]; + let prompt = budgeted_transcript_prompt(&entries, Some("main"), NEW_MESSAGE).unwrap(); + assert!(prompt.ends_with(NEW_MESSAGE)); + for body in ["Remember PINEAPPLE", "I remember PINEAPPLE", "And the code?"] { + assert!(prompt.contains(body), "missing {body}: {prompt}"); + } + // Complete means complete: no omission note, no read pointer, no cuts. + assert!(!prompt.contains("omitted"), "{prompt}"); + assert!(!prompt.contains("read_chat"), "{prompt}"); + assert!(!prompt.contains(TRUNCATION_MARKER), "{prompt}"); + assert!(prompt.contains(LEAD), "{prompt}"); + assert!(prompt.contains(""), "{prompt}"); + // Delivered in transcript order. + assert!( + prompt.find("Remember PINEAPPLE").unwrap() < prompt.find("And the code?").unwrap() + ); + } + + #[test] + fn selection_keeps_priorities_then_fills_newest_first_within_the_budget() { + let mut entries = vec![ + user("u0", "ORIGINAL: build the parser"), + assistant("a0", "on it"), + ]; + for i in 1..=20 { + entries.push(user(&format!("u{i}"), &format!("fill-{i}: {}", "x".repeat(900)))); + entries.push(assistant(&format!("a{i}"), &format!("note-{i}: {}", "y".repeat(90)))); + } + entries.push(user("u99", "LATEST-Q: ship it")); + entries.push(assistant("a99", "LATEST-A: done")); + let prompt = budgeted_transcript_prompt(&entries, Some("main"), NEW_MESSAGE).unwrap(); + // The three priorities survive every budget. + for body in ["ORIGINAL: build the parser", "LATEST-Q: ship it", "LATEST-A: done"] { + assert!(prompt.contains(body), "missing priority {body}: {prompt}"); + } + // Recent context is preferred over the middle of the transcript. + assert!(prompt.contains("fill-20:"), "{prompt}"); + assert!(prompt.contains("note-20:"), "{prompt}"); + assert!(!prompt.contains("fill-2:"), "{prompt}"); + // The coverage header names the omitted count and the source chat. + assert!(omitted_count(&prompt) >= 1); + assert!(prompt.contains("read_chat"), "{prompt}"); + assert!(prompt.contains("chat id \"main\""), "{prompt}"); + // Everything in front of the new message fits the budget. + assert!( + delivered_prefix(&prompt).len() <= TRANSCRIPT_BUDGET_BYTES, + "delivered {} bytes", + delivered_prefix(&prompt).len() + ); + assert!(prompt.ends_with(NEW_MESSAGE)); + } + + #[test] + fn an_oversized_message_is_truncated_not_dropped() { + let entries = vec![ + user("u0", "FIRST: the original request"), + assistant("a0", "acknowledged"), + user("u1", &format!("BIG {}", "z".repeat(30_000))), + assistant("a1", "done"), + ]; + let prompt = budgeted_transcript_prompt(&entries, Some("main"), NEW_MESSAGE).unwrap(); + assert!(prompt.contains("BIG "), "{prompt}"); + assert!(prompt.contains(TRUNCATION_MARKER), "{prompt}"); + // The 30k body was cut to a bounded prefix: most of it is gone, and + // nothing close to the full body survived. + assert!(prompt.contains(&"z".repeat(4000)), "{prompt}"); + assert!(!prompt.contains(&"z".repeat(5000)), "{prompt}"); + // The other messages still ride along. + assert!(prompt.contains("FIRST: the original request"), "{prompt}"); + assert!(prompt.contains("done"), "{prompt}"); + assert!( + delivered_prefix(&prompt).len() <= TRANSCRIPT_BUDGET_BYTES, + "delivered {} bytes", + delivered_prefix(&prompt).len() + ); + // Truncated-but-nothing-omitted still points at read_chat. + assert!(!prompt.contains("messages were omitted"), "{prompt}"); + assert!(prompt.contains("read_chat"), "{prompt}"); + } + + #[test] + fn the_new_user_message_is_never_shortened() { + let entries = vec![ + user("u0", "ORIGINAL: build the parser"), + assistant("a0", "on it"), + user("u1", &format!("BIG {}", "z".repeat(30_000))), + assistant("a1", "done"), + ]; + let new_message = format!("{} {}", "NEW:", "n".repeat(60_000)); + let prompt = budgeted_transcript_prompt(&entries, Some("main"), &new_message).unwrap(); + assert!(prompt.ends_with(&new_message)); + assert!(prompt.contains(&new_message)); + // The oversized new message never ate the transcript's budget. + assert!(prompt.contains("ORIGINAL: build the parser"), "{prompt}"); + assert!( + prompt.len() - new_message.len() <= TRANSCRIPT_BUDGET_BYTES, + "delivered {} bytes", + prompt.len() - new_message.len() + ); + } + + #[test] + fn contents_policy_keeps_text_runs_errors_and_file_changes_only() { + let entries = vec![ + entry( + "u1", + MessageRole::User, + vec![ + MessagePart::Text { + id: "u1-t".into(), + text: "look at this".into(), + }, + MessagePart::Reasoning { + id: "u1-r".into(), + text: "PRIVATE SCRATCHPAD".into(), + }, + MessagePart::Image { + id: "u1-i".into(), + path: "/uploads/p.png".into(), + name: "p.png".into(), + mime_type: "image/png".into(), + }, + MessagePart::Input { + id: "u1-in".into(), + request_id: "req-1".into(), + questions: vec![], + resolved: true, + }, + ], + ), + entry( + "a1", + MessageRole::Assistant, + vec![ + MessagePart::Tool { + id: "a1-tool".into(), + call: ToolCall::Exec { + command: "cargo test".into(), + }, + is_error: true, + resolved: true, + output: Some("3 failed".into()), + diff: None, + output_ref: None, + output_bytes: None, + diff_ref: None, + diff_stats: Some(vec![ToolDiffStat { + path: "/w/a.rs".into(), + additions: 2, + deletions: 1, + }]), + subagent_ref: None, + subagent_status: None, + subagent_tail: None, + }, + MessagePart::Error { + id: "a1-e".into(), + message: "boom".into(), + }, + MessagePart::Text { + id: "a1-t".into(), + text: "here is what happened".into(), + }, + ], + ), + // System entries (the fork seam) are structure, not transcript. + entry("seam", MessageRole::System, vec![MessagePart::Fork { + id: "fork:x".into(), + source_chat_id: "main".into(), + source_title: "Main".into(), + }]), + // An entry that renders blank (reasoning only) is no message. + entry( + "a2", + MessageRole::Assistant, + vec![MessagePart::Reasoning { + id: "a2-r".into(), + text: "thinking only".into(), + }], + ), + ]; + let prompt = budgeted_transcript_prompt(&entries, Some("main"), NEW_MESSAGE).unwrap(); + assert!(prompt.contains("look at this"), "{prompt}"); + assert!(prompt.contains("Command: cargo test"), "{prompt}"); + assert!(prompt.contains("Exit code: 1"), "{prompt}"); + assert!(prompt.contains("3 failed"), "{prompt}"); + assert!(prompt.contains("File changes: /w/a.rs"), "{prompt}"); + assert!(prompt.contains("Error: boom"), "{prompt}"); + assert!(prompt.contains("here is what happened"), "{prompt}"); + assert!(prompt.contains("user: look at this"), "{prompt}"); + assert!(prompt.contains("assistant: Command: cargo test"), "{prompt}"); + assert!(!prompt.contains("PRIVATE SCRATCHPAD"), "{prompt}"); + assert!(!prompt.contains("p.png"), "{prompt}"); + assert!(!prompt.contains("req-1"), "{prompt}"); + assert!(!prompt.contains("fork:x"), "{prompt}"); + assert!(!prompt.contains("Main"), "{prompt}"); + assert!(!prompt.contains("thinking only"), "{prompt}"); + } + + #[test] + fn unresolved_calls_report_an_unknown_exit_code() { + let mut part = tool_part("t", "ls"); + if let MessagePart::Tool { + resolved, is_error, .. + } = &mut part + { + *resolved = false; + *is_error = false; + } + let entries = vec![entry( + "a1", + MessageRole::Assistant, + vec![part, tool_part("t2", "pwd")], + )]; + let prompt = budgeted_transcript_prompt(&entries, None, NEW_MESSAGE).unwrap(); + assert!(prompt.contains("Command: ls"), "{prompt}"); + assert!(prompt.contains("Exit code: unknown"), "{prompt}"); + assert!(prompt.contains("Command: pwd"), "{prompt}"); + assert!(prompt.contains("Exit code: 0"), "{prompt}"); + } + + #[test] + fn truncation_cuts_at_char_boundaries() { + let line = format!("user: {}", "é".repeat(6_000)); + let cut = truncate_line(&line, ITEM_CAP_BYTES); + assert!(cut.len() <= ITEM_CAP_BYTES, "{}", cut.len()); + assert!(cut.ends_with(TRUNCATION_MARKER)); + // Slicing at a char boundary keeps the string valid UTF-8 whole. + let reassembled: String = cut.chars().collect(); + assert_eq!(reassembled, cut); + // A line under the cap is untouched. + assert_eq!(truncate_line("short", ITEM_CAP_BYTES), "short"); + } + + #[test] + fn continuation_fragments_join_into_one_message() { + let mut root = user("m1", "first half"); + root.parts.push(MessagePart::Text { + id: "m1-t".into(), + text: "second half".into(), + }); + let tail = entry( + "m1#c1", + MessageRole::User, + vec![MessagePart::Text { + id: "m1-t2".into(), + text: "third half".into(), + }], + ); + let tail = SessionMessageEntry { + continuation_of: Some("m1".into()), + ..tail + }; + let prompt = budgeted_transcript_prompt(&[root, tail], Some("main"), NEW_MESSAGE).unwrap(); + // One message, delivered once, parts in order. + assert_eq!(prompt.matches("user:").count(), 1, "{prompt}"); + assert!(prompt.contains("first half\nsecond half\nthird half"), "{prompt}"); + } + + #[test] + fn no_deliverable_transcript_returns_none() { + let entries = vec![ + entry( + "seam", + MessageRole::System, + vec![MessagePart::Fork { + id: "fork:x".into(), + source_chat_id: "main".into(), + source_title: "Main".into(), + }], + ), + user("u1", " "), + ]; + assert!(budgeted_transcript_prompt(&entries, Some("main"), NEW_MESSAGE).is_none()); + assert!(budgeted_transcript_prompt(&[], Some("main"), NEW_MESSAGE).is_none()); + } + + #[test] + fn the_prefix_is_the_prompt_without_its_tail() { + // The native-fork fallback (ticket 17) rides the prefix: the driver + // appends the user message itself, so the prefix must be exactly the + // delivered prompt minus the message — same selection, same header, + // same trailing blank line, never counting the message. + let mut entries = vec![ + user("u0", "ORIGINAL: build the parser"), + assistant("a0", "on it"), + ]; + for i in 1..=20 { + entries.push(user(&format!("u{i}"), &format!("fill-{i}: {}", "x".repeat(900)))); + entries.push(assistant(&format!("a{i}"), &format!("note-{i}: {}", "y".repeat(90)))); + } + let prefix = budgeted_transcript_prefix(&entries, Some("main")).unwrap(); + let prompt = budgeted_transcript_prompt(&entries, Some("main"), NEW_MESSAGE).unwrap(); + assert!(prompt.starts_with(&prefix), "prefix must head the prompt"); + assert!(prompt.ends_with(NEW_MESSAGE)); + assert_eq!(&prompt[..prefix.len()], &prefix); + assert!(prefix.ends_with("\n\n")); + assert!(!prefix.contains(NEW_MESSAGE)); + assert!(prefix.contains("read_chat")); + assert!(prefix.len() <= TRANSCRIPT_BUDGET_BYTES); + // An undeliverable transcript has no prefix either. + assert!(budgeted_transcript_prefix(&[], Some("main")).is_none()); + } +} diff --git a/crates/engine/src/transcript_history.rs b/crates/engine/src/transcript_history.rs index 63945e6162..9f3662008a 100644 --- a/crates/engine/src/transcript_history.rs +++ b/crates/engine/src/transcript_history.rs @@ -141,6 +141,7 @@ mod tests { fn entry(id: &str) -> SessionMessageEntry { SessionMessageEntry { + origin_chat_id: None, id: id.into(), role: MessageRole::Assistant, parts: vec![MessagePart::Text { @@ -151,6 +152,7 @@ mod tests { device_id: "host".into(), status: Some(MessageStatus::Streaming), continuation_of: None, + provider_turn: None, duration_ms: None, } } diff --git a/crates/engine/src/workspace_host.rs b/crates/engine/src/workspace_host.rs index 60fe873218..30aee1f379 100644 --- a/crates/engine/src/workspace_host.rs +++ b/crates/engine/src/workspace_host.rs @@ -456,13 +456,18 @@ impl WorkspaceHost { config: Option, cwd: Option, ) -> Result<(), EngineError> { - self.create_chat_with_parent(chat_id, space_id, device_id, config, cwd, None) + self.create_chat_with_parent(chat_id, space_id, device_id, config, cwd, None, None) } /// [`create_chat`](Self::create_chat) recording the chat this one hangs /// off (`parentChatId`) — a side chat forked from it, or the chat whose /// agent spawned this one through the Roboco MCP server. Children list /// under their parent in the explorer instead of the main sidebar. + /// `agent_origin` is set when an AGENT created this one through the + /// Roboco MCP `create_chat`/`create_chats` tools (the creator chat and + /// its notify preference); absent on user-created chats, which never + /// notify. It rides the same row as `parentChatId` and survives + /// restarts with it. pub fn create_chat_with_parent( &self, chat_id: &str, @@ -471,6 +476,7 @@ impl WorkspaceHost { config: Option, cwd: Option, parent_chat_id: Option, + agent_origin: Option, ) -> Result<(), EngineError> { if self.read(|doc| doc.chat(chat_id))?.is_some() { return Ok(()); // idempotent: optimistic client retries never duplicate @@ -519,6 +525,7 @@ impl WorkspaceHost { space_id: space.as_ref().map(|s| s.id.clone()), last_seen_at: None, parent_chat_id: parent_chat_id.filter(|p| !p.trim().is_empty()), + agent_origin, }) })?; Ok(()) @@ -1075,6 +1082,8 @@ mod tests { last_completed_turn: None, started_at: None, updated_at: at, + limit: None, + scheduled_resume: None, } } diff --git a/crates/engine/tests/account_failover.rs b/crates/engine/tests/account_failover.rs new file mode 100644 index 0000000000..7a51db8fd8 --- /dev/null +++ b/crates/engine/tests/account_failover.rs @@ -0,0 +1,1006 @@ +//! Resume on another account (orchestration v2, ticket 09): the engine +//! operation behind a Limited Claude Code chat's account failover. +//! +//! The candidates come from a SEEDED, fully isolated harness-account store +//! (slot files + usage cache + a live Claude login — no network: the cache +//! is fresh, so the failover's one probe reads it), and the harness seam is +//! a scripted stand-in that records every run request it receives and plays +//! the driver's part: a refused native resume starts a fresh session and +//! pastes the budgeted history in front of the prompt, exactly the way the +//! Codex driver does. +//! +//! Coverage (the ticket's checklist, at the engine RPC seam): +//! - the ranking (excludes signed-out, limited, API-key and the live +//! account; lowest usage first, same-org accounts ranked lower); +//! - the switch itself (activate, retire the warm runtime by credential +//! fingerprint, resume the SAME harness session id with the continue +//! prompt and the budgeted fallback history); +//! - the refused-resume fresh-session fallback; +//! - a signed-out failure after the switch moving to the next candidate; +//! a plain failure restoring the previous account; +//! - no healthy candidate → `noCandidates` (only Resume at reset). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use futures::StreamExt; +use futures::stream::BoxStream; + +use roboco_doc::SessionCommandPayload; +use roboco_engine::{AgentAccounts, AgentAccountsConfig, EngineCore, HarnessRegistry}; +use roboco_harness::{Harness, HarnessError, RunControls}; +use roboco_proto::{ + AgentEvent, DoneStatus, HarnessId, Model, ReasoningLevel, ResumeAccountCandidates, + ResumeOnAccountOutcome, RunRequest, SandboxLevel, SessionStatus, SteeringMode, +}; +use roboco_rpc::methods; + +const LIMITED: &str = "chat-limited"; +const WARM: &str = "chat-warm"; + +/// What the scripted harness plays for the chat's continuation run (run +/// index 1 on the limited chat). +#[derive(Clone, Copy, PartialEq, Eq)] +enum Continuation { + /// The native resume works: the same session id answers. + Resumed, + /// The harness REFUSES the native resume: a fresh session id answers, + /// the budgeted history pasted in front of the prompt (the driver's + /// part — what Codex does on a failed `thread/resume`). + Refused, + /// The continuation fails with ticket 05's signed-out kind. + SignedOut, + /// The continuation fails with an unclassified error. + PlainFailure, +} + +/// One run the scripted harness received, as the engine handed it over plus +/// what a driver would have delivered (the fallback pasted on a refusal). +#[derive(Clone, Debug)] +struct RecordedRun { + chat: String, + run: usize, + prompt: String, + resume: Option, + fallback: Option, + delivered_prompt: String, + session_id: String, +} + +/// A persistent, steerable Claude-shaped stand-in. Run 0 of the limited +/// chat ends on a classified usage-limit error (the driver shape ticket 05 +/// landed, the settle ticket 06 records); run 0 of the warm chat completes +/// and PARKS (the stream never ends) so its runtime stays warm; run 1 of +/// the limited chat plays [`Continuation`]; later runs complete. +struct FailoverHarness { + continuation: Continuation, + runs: Arc>>, + counts: Arc>>, +} + +impl FailoverHarness { + fn new(continuation: Continuation) -> Arc { + Arc::new(Self { + continuation, + runs: Arc::new(Mutex::new(Vec::new())), + counts: Arc::new(Mutex::new(HashMap::new())), + }) + } + + fn runs_for(&self, chat: &str) -> Vec { + self.runs + .lock() + .unwrap() + .iter() + .filter(|run| run.chat == chat) + .cloned() + .collect() + } + + fn run_count(&self, chat: &str) -> usize { + self.counts.lock().unwrap().get(chat).copied().unwrap_or(0) + } +} + +#[async_trait] +impl Harness for FailoverHarness { + fn id(&self) -> HarnessId { + HarnessId::ClaudeCode + } + fn display_name(&self) -> &str { + "Claude Code" + } + fn supports_steering(&self) -> bool { + true + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::TurnBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[ReasoningLevel::Medium] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + fn deterministic_turn_end(&self) -> bool { + true + } + async fn run( + &self, + request: RunRequest, + _controls: RunControls, + ) -> Result>, HarnessError> { + // The engine stamps the originating chat on the injected MCP server. + let chat = request + .mcp + .as_ref() + .and_then(|mcp| mcp.env.get("ROBOCO_CHAT_ID").cloned()) + .unwrap_or_default(); + let run = { + let mut counts = self.counts.lock().unwrap(); + let next = counts.entry(chat.clone()).or_insert(0); + *next += 1; + *next - 1 + }; + // The driver's delivery rule: a refused native resume pastes the + // budgeted fallback in front of the prompt on the fresh session. + let refuse = run == 1 && chat == LIMITED && self.continuation == Continuation::Refused; + let delivered_prompt = match (&request.resume_fallback_history, refuse) { + (Some(history), true) => format!("{history}{}", request.prompt), + _ => request.prompt.clone(), + }; + let session_id = match (chat.as_str(), run) { + (LIMITED, 0) => "sess-0".to_string(), + (LIMITED, 1) if self.continuation == Continuation::Refused => "fresh-9".to_string(), + (LIMITED, 1) => "sess-0".to_string(), + (LIMITED, _) => "sess-0".to_string(), + (WARM, 0) => "warm-sess".to_string(), + (WARM, _) => format!("warm-sess-{run}"), + _ => "sess".to_string(), + }; + self.runs.lock().unwrap().push(RecordedRun { + chat: chat.clone(), + run, + prompt: request.prompt.clone(), + resume: request.resume.clone(), + fallback: request.resume_fallback_history.clone(), + delivered_prompt, + session_id: session_id.clone(), + }); + let harness = HarnessId::ClaudeCode; + let cwd = request.cwd.clone(); + let mut events = vec![Ok(AgentEvent::SessionStarted { + harness, + model: "claude-test".into(), + tools: vec![], + cwd, + session_id: session_id.clone(), + assistant_message_id: format!("a-{chat}-{run}"), + })]; + match (chat.as_str(), run) { + (LIMITED, 0) => { + // The classified limit error, then the errored Done — the + // shape that turns the chat Limited (ticket 06). + events.push(Ok(AgentEvent::Error { + message: "Claude 5-hour limit reached — try again after it resets.".into(), + kind: Some(roboco_proto::AgentErrorKind::UsageLimit { + window: "5-hour".into(), + resets_at: Some(1_762_675_400_000), + }), + })); + events.push(Ok(AgentEvent::Done { + status: DoneStatus::Errored, + result: None, + error: Some("the turn failed".into()), + session_id: Some(session_id), + })); + Ok(futures::stream::iter(events).boxed()) + } + (LIMITED, 1) => match self.continuation { + Continuation::Resumed | Continuation::Refused => { + events.push(Ok(AgentEvent::TextDelta { + text: "Picked the work back up on another account.".into(), + })); + events.push(Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some(session_id), + })); + Ok(futures::stream::iter(events).boxed()) + } + Continuation::SignedOut => { + events.push(Ok(AgentEvent::Error { + message: "Authentication failed — sign in to Claude again.".into(), + kind: Some(roboco_proto::AgentErrorKind::SignedOut), + })); + events.push(Ok(AgentEvent::Done { + status: DoneStatus::Errored, + result: None, + error: Some("Authentication failed — sign in to Claude again.".into()), + session_id: Some(session_id), + })); + Ok(futures::stream::iter(events).boxed()) + } + Continuation::PlainFailure => { + events.push(Ok(AgentEvent::Error { + message: "The agent child crashed.".into(), + kind: None, + })); + events.push(Ok(AgentEvent::Done { + status: DoneStatus::Errored, + result: None, + error: Some("the turn failed".into()), + session_id: Some(session_id), + })); + Ok(futures::stream::iter(events).boxed()) + } + }, + (WARM, 0) => { + // A cleanly completed turn on a steerable harness PARKS: the + // stream stays open, the runtime stays warm for the next + // routed dispatch. + events.push(Ok(AgentEvent::TextDelta { + text: "warm turn done".into(), + })); + events.push(Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some(session_id), + })); + Ok(futures::stream::iter(events) + .chain(futures::stream::pending::>()) + .boxed()) + } + _ => { + // The chain's later runs (and the warm chat's replaced + // runtime): plain completions. + events.push(Ok(AgentEvent::TextDelta { + text: "later turn done".into(), + })); + events.push(Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some(session_id), + })); + Ok(futures::stream::iter(events).boxed()) + } + } + } +} + +/// A local stand-in for the usage endpoint: nothing should ever reach it +/// (the seeded cache is fresh, so the failover's probe reads it), but a +/// stray probe answers green instead of touching the network. +async fn usage_server() -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("bind mock usage server"); + let base = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + tokio::spawn(async move { + let mut request = vec![0u8; 16 * 1024]; + let _ = socket.read(&mut request).await.unwrap_or(0); + let body = + r#"{"five_hour":{"utilization":6.0,"resets_at":"2026-10-18T08:00:00Z"}}"#; + let reply = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: \ + {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = socket.write_all(reply.as_bytes()).await; + let _ = socket.shutdown().await; + }); + } + }); + base +} + +/// The live Claude login (the account the limit hits) under `config`. +fn write_claude_login(config: &AgentAccountsConfig) { + std::fs::create_dir_all(&config.claude_config_dir).expect("claude config dir"); + std::fs::write( + &config.claude_config_file, + serde_json::json!({ + "oauthAccount": { + "accountUuid": "uuid-alice", + "emailAddress": "alice@example.com", + "organizationName": "Acme", + }, + }) + .to_string(), + ) + .expect("claude config"); + std::fs::write( + config.claude_config_dir.join(".credentials.json"), + serde_json::json!({ + "claudeAiOauth": { + "accessToken": "alice-token", + "refreshToken": "refresh-alice-token", + // Far-future expiry: probes must never try to rotate it. + "expiresAt": 4_102_444_800_000i64, + } + }) + .to_string(), + ) + .expect("claude creds"); +} + +/// One saved Claude slot + its cached usage. `windows`/`sign_in` seed the +/// usage cache keyed by the slot's account key; a FRESH `checked_at` (now) +/// keeps the failover's probe reading the cache instead of the network. +fn write_slot( + config: &AgentAccountsConfig, + id: &str, + account_key: &str, + email: &str, + organization: Option<&str>, + auth_kind: &str, + usage: &str, + sign_in: Option<&str>, +) { + let dir = config.data_dir.join("agent-accounts").join("claude-code"); + std::fs::create_dir_all(&dir).expect("slots dir"); + let token = format!("{}-token", account_key); + let slot = serde_json::json!({ + "id": id, + "harness": "claude-code", + "accountKey": account_key, + "profile": { + "email": email, + "organization": organization, + "plan": "Pro", + "authKind": auth_kind, + }, + "credentials": { + "claudeAiOauth": { + "accessToken": token, + "refreshToken": format!("refresh-{token}"), + "expiresAt": 4_102_444_800_000i64, + } + }, + "savedAt": 1_762_000_000_000i64, + }); + std::fs::write(dir.join(format!("{id}.json")), slot.to_string()).expect("slot file"); + + // The usage cache: keyed `claude-code:{account_key}`, fresh so nothing + // probes. `usage` is the windows array JSON; `sign_in` the hard evidence. + let cache_path = config.data_dir.join("agent-accounts").join("usage-cache.json"); + let mut cache: serde_json::Value = std::fs::read_to_string(&cache_path) + .ok() + .and_then(|raw| serde_json::from_str(&raw).ok()) + .unwrap_or_else(|| serde_json::json!({ "entries": {} })); + cache["entries"][&format!("claude-code:{account_key}")] = serde_json::json!({ + "usage": { "windows": serde_json::from_str::(usage).unwrap() }, + "fetchedAt": chrono::Utc::now().timestamp_millis(), + "checkedAt": chrono::Utc::now().timestamp_millis(), + "signIn": sign_in, + }); + std::fs::write(&cache_path, cache.to_string()).expect("usage cache"); +} + +/// The full candidate zoo: carol (0.05, other org — the recommendation), +/// bob (0.2, no org), gina (0.05, the limited account's org — ranked below +/// bob), and the three exclusions (signed-out dave, still-limited erin, +/// API-key frank). +fn seed_store(config: &AgentAccountsConfig) { + write_claude_login(config); + write_slot( + config, + "slot-carol", + "uuid-carol", + "carol@example.com", + Some("Globex"), + "oauth", + r#"[{"label":"Session","usedFraction":0.05,"resetsAt":"2026-10-18T08:00:00Z"}]"#, + Some("signed-in"), + ); + write_slot( + config, + "slot-bob", + "uuid-bob", + "bob@example.com", + None, + "oauth", + r#"[{"label":"Session","usedFraction":0.2,"resetsAt":"2026-10-18T08:00:00Z"}]"#, + Some("signed-in"), + ); + write_slot( + config, + "slot-gina", + "uuid-gina", + "gina@example.com", + Some("Acme"), + "oauth", + r#"[{"label":"Session","usedFraction":0.05,"resetsAt":"2026-10-18T08:00:00Z"}]"#, + Some("signed-in"), + ); + write_slot( + config, + "slot-dave", + "uuid-dave", + "dave@example.com", + None, + "oauth", + r#"[{"label":"Session","usedFraction":0.1,"resetsAt":"2026-10-18T08:00:00Z"}]"#, + Some("signed-out"), + ); + // Erin's exhausted window resets in the future: still limited. + let reset = (chrono::Utc::now() + chrono::TimeDelta::hours(2)) + .to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + write_slot( + config, + "slot-erin", + "uuid-erin", + "erin@example.com", + None, + "oauth", + &format!(r#"[{{"label":"Session","usedFraction":1.0,"resetsAt":"{reset}"}}]"#), + Some("signed-in"), + ); + write_slot( + config, + "slot-frank", + "uuid-frank", + "frank@example.com", + None, + "api-key", + r#"[]"#, + None, + ); +} + +/// Which store the engine under test gets. Every variant starts from the +/// limited live login (alice); the rest is the candidate set the test +/// needs. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Store { + /// The full candidate zoo (the ranking test). + Full, + /// Two healthy candidates (carol 0.05, bob 0.2) — the failover and the + /// chain tests, whose shape depends on a known next account. + Two, + /// Nothing else (no candidate remains). + Bare, +} + +fn seed(config: &AgentAccountsConfig, store: Store) { + match store { + Store::Full => seed_store(config), + Store::Two => { + write_claude_login(config); + for (id, key, email, org, usage) in [ + ( + "slot-carol", + "uuid-carol", + "carol@example.com", + Some("Globex"), + 0.05, + ), + ("slot-bob", "uuid-bob", "bob@example.com", None, 0.2), + ] { + write_slot( + config, + id, + key, + email, + org, + "oauth", + &format!( + r#"[{{"label":"Session","usedFraction":{usage},"resetsAt":"2026-10-18T08:00:00Z"}}]"# + ), + Some("signed-in"), + ); + } + } + Store::Bare => write_claude_login(config), + } +} + +/// The engine under test: the scripted Claude harness registered under +/// `claude-code`, the seeded isolated account store wired into BOTH the +/// sessions engine (ranking, probing, switching) and the RPC surface. +async fn setup( + continuation: Continuation, + store: Store, +) -> (EngineCore, Arc, roboco_rpc::RpcClient) { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.keep(); + let registry = HarnessRegistry::new(); + let harness = FailoverHarness::new(continuation); + registry.register(harness.clone()); + let mut core = EngineCore::assemble(&root, Arc::new(registry), HarnessId::ClaudeCode) + .expect("engine core assembles"); + // The mocked usage endpoint only ever sees a stray probe. + let base = usage_server().await; + let config = AgentAccountsConfig::isolated(&root); + seed(&config, store); + let accounts = AgentAccounts::with_mocked_endpoints(config, &base); + // One store, two handles: the RPC lists from it, the failover switches + // on it — replacing the detected store `assemble` wired in. + core.agent_accounts = accounts.clone(); + core.sessions.set_agent_accounts(accounts); + // The injected MCP server names the chat, so the scripted harness can + // tell the two chats' runs apart. + core.sessions.set_ipc_port(27699); + let client = roboco_rpc::memory_client(core.rpc_service()); + (core, harness, client) +} + +async fn create_chat(core: &EngineCore, chat: &str) { + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": chat, + "deviceId": core.device_id, + }), + ) + .await + .expect("createChat"); + // Pre-titled, so the auto-titler never dispatches a harness request of + // its own (it would land on the scripted harness as an extra run). + core.workspace + .rename_chat(chat, &format!("Pre-titled {chat}")) + .expect("rename chat"); +} + +fn run_request(prompt: &str) -> RunRequest { + RunRequest { + prompt: prompt.into(), + harness: Some(HarnessId::ClaudeCode), + model: None, + reasoning: None, + model_options: Default::default(), + cwd: ".".into(), + sandbox: SandboxLevel::WorkspaceWrite, + auto_approve: true, + attachments: Vec::new(), + worktree: None, + mcp: None, + resume: None, + provider_fork: None, + resume_fallback_history: None, + } +} + +/// Queue a Run command the way the composer's send does (the durable +/// command plane, ahead of any held queue). +fn queue_run(core: &EngineCore, chat: &str, prompt: &str, message_id: &str) { + core.doc_host + .queue_command( + chat, + SessionCommandPayload::Run { + request: run_request(prompt), + message_id: message_id.into(), + }, + ) + .expect("queue run command"); +} + +async fn wait_for(mut predicate: impl FnMut() -> bool, what: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while !predicate() { + assert!( + tokio::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(Duration::from_millis(25)).await; + } +} + +/// [`wait_for`] for predicates that must await the engine (an RPC round +/// trip per probe). +async fn wait_for_async(mut predicate: F, what: &str) +where + F: FnMut() -> Fut, + Fut: std::future::Future, +{ + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while !predicate().await { + assert!( + tokio::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +async fn wait_for_status(core: &EngineCore, chat: &str, status: SessionStatus) { + wait_for( + || { + core.sessions + .session_status(chat) + .is_some_and(|s| s.status == status) + }, + "the chat to settle", + ) + .await; +} + +/// Run the chat's first turn into the usage limit and wait for the Limited +/// record (the failover's precondition). +async fn drive_into_limit(core: &EngineCore, harness: &FailoverHarness, chat: &str) { + queue_run(core, chat, "do the work", "m-open"); + wait_for( + || harness.run_count(chat) >= 1, + "the limited run to start", + ) + .await; + wait_for( + || { + core.sessions + .session_status(chat) + .is_some_and(|s| s.status == SessionStatus::Errored && s.limit.is_some()) + }, + "the chat to go Limited", + ) + .await; +} + +/// The candidates as the RPC ranks them. +async fn candidates( + client: &roboco_rpc::RpcClient, + chat: &str, +) -> ResumeAccountCandidates { + let value = client + .call( + methods::LIST_RESUME_ACCOUNT_CANDIDATES, + serde_json::json!({ "chatId": chat }), + ) + .await + .expect("ListResumeAccountCandidates"); + serde_json::from_value(value).expect("candidates reply parses") +} + +/// The failover RPC. +async fn resume_on_another_account( + client: &roboco_rpc::RpcClient, + chat: &str, + account_id: Option<&str>, +) -> ResumeOnAccountOutcome { + let mut params = serde_json::json!({ "chatId": chat }); + if let (Some(id), Some(object)) = (account_id, params.as_object_mut()) { + object.insert("accountId".into(), serde_json::json!(id)); + } + let value = client + .call(methods::RESUME_ON_ANOTHER_ACCOUNT, params) + .await + .expect("ResumeOnAnotherAccount"); + serde_json::from_value(value).expect("outcome reply parses") +} + +/// The active Claude account's row, through the accounts RPC. +async fn active_row( + client: &roboco_rpc::RpcClient, +) -> Option { + let value = client + .call( + methods::LIST_AGENT_ACCOUNTS, + serde_json::json!({ "forceUsage": false }), + ) + .await + .expect("ListAgentAccounts"); + let snapshot: roboco_proto::AgentAccountsSnapshot = + serde_json::from_value(value).expect("snapshot parses"); + snapshot + .accounts + .into_iter() + .find(|row| row.harness == HarnessId::ClaudeCode && row.active) +} + +/// The candidate ranking (ticket checklist 1): the limited account, signed-out, +/// still-limited and API-key rows never surface; lowest usage first, and the +/// account sharing the limited one's org ranks below a higher-usage account +/// from another org. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn candidates_exclude_dead_rows_and_rank_by_usage_then_org() { + let (core, _harness, client) = setup(Continuation::Resumed, Store::Full).await; + create_chat(&core, LIMITED).await; + drive_into_limit(&core, &_harness, LIMITED).await; + + let ranked = candidates(&client, LIMITED).await; + assert_eq!(ranked.harness, HarnessId::ClaudeCode); + // The account the limit hit (the live login's slot id). + assert!(ranked.limited_account_id.is_some()); + let ids: Vec<&str> = ranked + .candidates + .iter() + .map(|row| row.id.as_str()) + .collect(); + assert_eq!( + ids, + vec!["slot-carol", "slot-bob", "slot-gina"], + "lowest usage first; a different org outranks the limited account's own pool" + ); + // The meters ride the rows: carol's Session window at 5%. + let carol = ranked.candidates.first().unwrap(); + assert_eq!(carol.usage_windows.len(), 1); + assert_eq!(carol.usage_windows[0].used_fraction, 0.05); + // The signed-out accounts (stories 47/50) ride their own list — dave + // is excluded as a candidate but offered for "Sign in again"; the + // still-limited erin and the API-key frank are neither (reviving them + // means waiting or a new API key, not a sign-in). + let signed_out: Vec<&str> = ranked + .signed_out + .iter() + .map(|row| row.id.as_str()) + .collect(); + assert_eq!(signed_out, vec!["slot-dave"]); + core.shutdown().await; +} + +/// The switch itself (ticket checklist 2): the chosen account is probed and +/// ACTIVATED (the CLI store now holds its login), the chat's warm runtime is +/// retired, and the continuation resumes the SAME harness session id with +/// the continue prompt — the budgeted history riding as the fallback. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_failover_switches_and_resumes_the_same_harness_session() { + let (core, harness, client) = setup(Continuation::Resumed, Store::Two).await; + create_chat(&core, LIMITED).await; + // A second chat with a PARKED warm runtime, stamped with the limited + // login's fingerprint: after the switch it must NOT be reused. + create_chat(&core, WARM).await; + queue_run(&core, WARM, "warm first turn", "m-warm-1"); + wait_for( + || harness.run_count(WARM) == 1, + "the warm chat's first run to start", + ) + .await; + wait_for_status(&core, WARM, SessionStatus::Idle).await; + + drive_into_limit(&core, &harness, LIMITED).await; + let outcome = resume_on_another_account(&client, LIMITED, None).await; + assert_eq!( + outcome, + ResumeOnAccountOutcome::Switched { + account_id: "slot-carol".into(), + account_email: Some("carol@example.com".into()), + } + ); + // The switch landed: the live login is carol's. + let active = active_row(&client).await.expect("an active account"); + assert_eq!(active.id, "slot-carol"); + assert_eq!(active.email.as_deref(), Some("carol@example.com")); + // The continuation resumes the chat's session with the continue prompt… + wait_for( + || harness.run_count(LIMITED) >= 2, + "the continuation to dispatch", + ) + .await; + let continuation = &harness.runs_for(LIMITED)[1]; + assert_eq!(continuation.prompt, roboco_proto::RESUME_CONTINUE_PROMPT); + assert_eq!(continuation.resume.as_deref(), Some("sess-0")); + // …and carries the chat's budgeted history as the fallback for a + // refused native resume. + let fallback = continuation.fallback.as_deref().expect("fallback history"); + assert!(fallback.contains("do the work"), "fallback: {fallback}"); + assert!(fallback.contains(""), "fallback: {fallback}"); + // The continuation completes: the chat is back to Idle. + wait_for_status(&core, LIMITED, SessionStatus::Idle).await; + + // The warm chat's NEXT message must replace its runtime (the login + // changed under it): a fresh run, never a steer into the parked one. + queue_run(&core, WARM, "warm second turn", "m-warm-2"); + wait_for( + || harness.run_count(WARM) >= 2, + "the warm chat's replaced runtime to start", + ) + .await; + let warm_runs = harness.runs_for(WARM); + assert_eq!(warm_runs.len(), 2, "a fresh runtime, not the parked one"); + assert_eq!(warm_runs[1].prompt, "warm second turn"); + core.shutdown().await; +} + +/// A refused native resume (ticket checklist 3): the harness starts a FRESH +/// session — its reported session id differs — and the budgeted history is +/// delivered in front of the continue prompt on that fresh session. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_refused_native_resume_falls_back_to_a_fresh_session_with_budgeted_history() { + let (core, harness, client) = setup(Continuation::Refused, Store::Two).await; + create_chat(&core, LIMITED).await; + drive_into_limit(&core, &harness, LIMITED).await; + + let outcome = resume_on_another_account(&client, LIMITED, None).await; + assert!(matches!(outcome, ResumeOnAccountOutcome::Switched { .. })); + wait_for( + || harness.run_count(LIMITED) >= 2, + "the continuation to dispatch", + ) + .await; + let continuation = &harness.runs_for(LIMITED)[1]; + // The engine asked for the native resume… + assert_eq!(continuation.resume.as_deref(), Some("sess-0")); + assert!(continuation.fallback.is_some()); + // …the harness refused it (the driver's part): a fresh session id, the + // budgeted history pasted in front of the prompt. + assert_ne!(continuation.session_id, "sess-0"); + assert_eq!(continuation.session_id, "fresh-9"); + assert!(continuation.delivered_prompt.contains("do the work")); + assert!(continuation.delivered_prompt.contains("")); + assert!(continuation + .delivered_prompt + .ends_with(roboco_proto::RESUME_CONTINUE_PROMPT)); + wait_for_status(&core, LIMITED, SessionStatus::Idle).await; + core.shutdown().await; +} + +/// A signed-out failure after the switch (ticket checklist 4): the account +/// is marked signed out (the run's own wiring) and the failover moves to +/// the NEXT candidate, whose continuation succeeds. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_signed_out_failure_marks_the_account_and_moves_to_the_next_candidate() { + let (core, harness, client) = setup(Continuation::SignedOut, Store::Two).await; + create_chat(&core, LIMITED).await; + drive_into_limit(&core, &harness, LIMITED).await; + + let outcome = resume_on_another_account(&client, LIMITED, None).await; + assert!(matches!( + outcome, + ResumeOnAccountOutcome::Switched { + account_id, .. + } if account_id == "slot-carol" + )); + // The carol continuation fails signed out; the chain retries on bob. + wait_for( + || harness.run_count(LIMITED) >= 3, + "the next candidate's continuation to dispatch", + ) + .await; + let runs = harness.runs_for(LIMITED); + assert_eq!(runs.len(), 3); + // Both continuations resume the same harness session. + assert_eq!(runs[1].resume.as_deref(), Some("sess-0")); + assert_eq!(runs[2].resume.as_deref(), Some("sess-0")); + assert_eq!(runs[2].prompt, roboco_proto::RESUME_CONTINUE_PROMPT); + // The chat settles Idle on bob's continuation. + wait_for_status(&core, LIMITED, SessionStatus::Idle).await; + // Carol is marked signed out (never offered again)… + let snapshot = client + .call( + methods::LIST_AGENT_ACCOUNTS, + serde_json::json!({ "forceUsage": false }), + ) + .await + .expect("ListAgentAccounts"); + let carol = snapshot["accounts"] + .as_array() + .expect("accounts") + .iter() + .find(|row| row["id"] == "slot-carol") + .expect("carol's row"); + assert_eq!(carol["signInState"], "signed-out"); + // …and the live login is bob's. + let active = active_row(&client).await.expect("an active account"); + assert_eq!(active.id, "slot-bob"); + core.shutdown().await; +} + +/// Any other failure (ticket checklist 4's other arm): the previously +/// active account is restored — a failed failover leaves the harness as it +/// found it — and no second candidate is tried. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_plain_failure_restores_the_previous_account() { + let (core, harness, client) = setup(Continuation::PlainFailure, Store::Two).await; + create_chat(&core, LIMITED).await; + drive_into_limit(&core, &harness, LIMITED).await; + + let outcome = resume_on_another_account(&client, LIMITED, None).await; + assert!(matches!(outcome, ResumeOnAccountOutcome::Switched { .. })); + wait_for( + || harness.run_count(LIMITED) >= 2, + "the continuation to dispatch", + ) + .await; + wait_for_status(&core, LIMITED, SessionStatus::Errored).await; + // The restore is a real account switch: the live login is alice's again. + wait_for_async( + || async { + active_row(&client) + .await + .is_some_and(|row| row.id != "slot-carol") + }, + "the previous account to be restored", + ) + .await; + let active = active_row(&client).await.expect("an active account"); + assert_eq!(active.email.as_deref(), Some("alice@example.com")); + // The failure is REPORTED (ticket 09): a System entry on the chat says + // the failover failed and the previous account is back — the banner + // (and the web client, which renders System entries) shows it. + wait_for( + || { + core.doc_host + .open(LIMITED) + .is_ok_and(|handle| { + handle + .doc() + .read_entries() + .map(|entries| { + entries.iter().any(|entry| { + entry.role == roboco_doc::MessageRole::System + && entry + .parts + .iter() + .any(|part| matches!(part, + roboco_doc::MessagePart::Text { text, .. } + if text.starts_with( + "Resume on another account failed: the resumed turn failed." + ) && text.contains("The harness is back on") + )) + }) + }) + .unwrap_or(false) + }) + }, + "the failed failover to be reported on the chat", + ) + .await; + // No second candidate: the chain ends with the restore. The settle + // beat lets a stray third run fail the count assertion. + tokio::time::sleep(Duration::from_millis(750)).await; + assert_eq!(harness.run_count(LIMITED), 2, "no third run on the chat"); + core.shutdown().await; +} + +/// No healthy candidate (ticket checklist 5): the reply is `noCandidates` — +/// the surface offers only Resume at reset — and nothing switched. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn with_no_healthy_candidate_only_resume_at_reset_remains() { + let (core, harness, client) = setup(Continuation::Resumed, Store::Bare).await; + create_chat(&core, LIMITED).await; + drive_into_limit(&core, &harness, LIMITED).await; + + let ranked = candidates(&client, LIMITED).await; + assert!(ranked.candidates.is_empty(), "no candidates offered"); + assert!( + ranked.signed_out.is_empty(), + "no signed-out rows either — nothing to sign back in" + ); + let outcome = resume_on_another_account(&client, LIMITED, None).await; + assert_eq!(outcome, ResumeOnAccountOutcome::NoCandidates); + // Nothing switched, nothing dispatched: the live login is still the + // limited account's, and the chat stays Limited. + let active = active_row(&client).await.expect("an active account"); + assert_eq!(active.email.as_deref(), Some("alice@example.com")); + assert_eq!(harness.run_count(LIMITED), 1); + assert!(core + .sessions + .session_status(LIMITED) + .is_some_and(|s| s.limit.is_some())); + core.shutdown().await; +} + +/// The user's explicit pick is honored when it is a candidate, and refused +/// when it is not (the limited account itself). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_explicit_pick_switches_to_it_and_a_non_candidate_is_refused() { + let (core, harness, client) = setup(Continuation::Resumed, Store::Two).await; + create_chat(&core, LIMITED).await; + drive_into_limit(&core, &harness, LIMITED).await; + + let outcome = resume_on_another_account(&client, LIMITED, Some("slot-bob")).await; + assert_eq!( + outcome, + ResumeOnAccountOutcome::Switched { + account_id: "slot-bob".into(), + account_email: Some("bob@example.com".into()), + } + ); + wait_for( + || harness.run_count(LIMITED) >= 2, + "bob's continuation to dispatch", + ) + .await; + wait_for_status(&core, LIMITED, SessionStatus::Idle).await; + core.shutdown().await; +} diff --git a/crates/engine/tests/account_sign_in_state.rs b/crates/engine/tests/account_sign_in_state.rs new file mode 100644 index 0000000000..b7a472ef10 --- /dev/null +++ b/crates/engine/tests/account_sign_in_state.rs @@ -0,0 +1,279 @@ +//! Harness-account sign-in state driven from RUNS (orchestration v2, +//! ticket 08). +//! +//! Ticket 05 gave `AgentEvent::Error` a structured kind; this is the wiring +//! that consumes it: a run whose error is classified signed-out marks the +//! harness's LIVE login's account signed out (visible through +//! `ListAgentAccounts`, cached meters and all), and a later successful +//! usage probe flips it back to signed-in. The scripted harness carries the +//! same classification ticket 05's Claude driver attaches to +//! `authentication_failed`, so the engine-side behavior is covered without +//! a real CLI. + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use futures::StreamExt; +use futures::stream::BoxStream; + +use roboco_engine::{AgentAccounts, AgentAccountsConfig, EngineCore, HarnessRegistry}; +use roboco_harness::{Harness, HarnessError, RunControls}; +use roboco_proto::{ + AgentErrorKind, AgentEvent, DoneStatus, HarnessId, Model, ReasoningLevel, RunRequest, + SandboxLevel, SessionStatus, SteeringMode, +}; +use roboco_rpc::methods; + +const CHAT: &str = "chat-signed-out"; + +/// A Claude-shaped harness whose scripted run fails with ticket 05's +/// structured signed-out kind. `nested` wraps the error in a subagent event +/// instead — a subagent runs on the same login, so its rejection is equally +/// hard evidence the login is dead. +struct SignedOutHarness { + nested: bool, +} + +#[async_trait] +impl Harness for SignedOutHarness { + fn id(&self) -> HarnessId { + HarnessId::ClaudeCode + } + fn display_name(&self) -> &str { + "Claude Code" + } + fn supports_steering(&self) -> bool { + false + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::TurnBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + _controls: RunControls, + ) -> Result>, HarnessError> { + let signed_out = AgentEvent::Error { + message: "Authentication failed — sign in to Claude again.".into(), + kind: Some(AgentErrorKind::SignedOut), + }; + let error = if self.nested { + AgentEvent::Subagent { + parent_tool_use_id: "sub-signout".into(), + event: Box::new(signed_out), + } + } else { + signed_out + }; + let events = vec![ + Ok(AgentEvent::SessionStarted { + harness: HarnessId::ClaudeCode, + model: "claude-test".into(), + tools: vec![], + cwd: request.cwd.clone(), + session_id: "sess-signout".into(), + assistant_message_id: "a-signout".into(), + }), + Ok(error), + Ok(AgentEvent::Done { + status: DoneStatus::Errored, + result: None, + error: Some("Authentication failed — sign in to Claude again.".into()), + session_id: Some("sess-signout".into()), + }), + ]; + Ok(futures::stream::iter(events).boxed()) + } +} + +/// A local stand-in for Anthropic's usage endpoint: every probe answered +/// with a green usage body, so a forced list re-establishes a marked +/// account as signed-in. +async fn usage_server() -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("bind mock usage server"); + let base = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + tokio::spawn(async move { + // Drain the request (a headers-only probe GET). + let mut request = vec![0u8; 16 * 1024]; + let _ = socket.read(&mut request).await.unwrap_or(0); + let body = + r#"{"five_hour":{"utilization":6.0,"resets_at":"2026-10-18T08:00:00Z"}}"#; + let reply = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: \ + {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = socket.write_all(reply.as_bytes()).await; + let _ = socket.shutdown().await; + }); + } + }); + base +} + +/// A Claude Code login (config + credentials) under the isolated `config`. +fn write_claude_login(config: &AgentAccountsConfig, uuid: &str, email: &str, token: &str) { + std::fs::create_dir_all(&config.claude_config_dir).expect("claude config dir"); + std::fs::write( + &config.claude_config_file, + serde_json::json!({ + "oauthAccount": { "accountUuid": uuid, "emailAddress": email }, + }) + .to_string(), + ) + .expect("claude config"); + std::fs::write( + config.claude_config_dir.join(".credentials.json"), + serde_json::json!({ + "claudeAiOauth": { + "accessToken": token, + "refreshToken": format!("refresh-{token}"), + // Far-future expiry: probes must never try to rotate it. + "expiresAt": 4_102_444_800_000i64, + } + }) + .to_string(), + ) + .expect("claude creds"); +} + +fn run_request(prompt: &str) -> RunRequest { + RunRequest { + mcp: None, + prompt: prompt.into(), + harness: Some(HarnessId::ClaudeCode), + model: None, + reasoning: None, + model_options: Default::default(), + cwd: "/tmp".into(), + sandbox: SandboxLevel::WorkspaceWrite, + auto_approve: false, + resume: None, + attachments: vec![], + worktree: None, + provider_fork: None, + resume_fallback_history: None, + } +} + +/// The engine under test: a Claude login live in an isolated account store +/// (whose usage endpoint is the local mock), and the signed-out harness +/// registered under `claude-code`. The SAME account store is wired into the +/// sessions engine (the run-error marking) and the RPC surface +/// (`ListAgentAccounts`). +async fn setup(nested: bool) -> (EngineCore, roboco_rpc::RpcClient) { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.keep(); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(SignedOutHarness { nested })); + let mut core = + EngineCore::assemble(&root, Arc::new(registry), HarnessId::ClaudeCode).expect("engine"); + + let config = AgentAccountsConfig::isolated(&root); + write_claude_login(&config, "uuid-alice", "alice@example.com", "alice-token"); + let base = usage_server().await; + let accounts = AgentAccounts::with_mocked_endpoints(config, &base); + // One store, two handles: the RPC lists from it, the marking writes to + // it — replacing the detected store `assemble` wired in. + core.agent_accounts = accounts.clone(); + core.sessions.set_agent_accounts(accounts); + let client = roboco_rpc::memory_client(core.rpc_service()); + (core, client) +} + +async fn wait_for(mut predicate: impl FnMut() -> bool, what: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while !predicate() { + assert!( + tokio::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(Duration::from_millis(25)).await; + } +} + +/// The signed-in state of the Claude row, as `ListAgentAccounts` reports it. +async fn claude_sign_in_state(client: &roboco_rpc::RpcClient, force: bool) -> String { + let snapshot = client + .call( + methods::LIST_AGENT_ACCOUNTS, + serde_json::json!({ "forceUsage": force }), + ) + .await + .expect("ListAgentAccounts"); + snapshot["accounts"] + .as_array() + .expect("accounts") + .iter() + .find(|account| account["harness"] == "claude-code") + .map(|account| account["signInState"].as_str().expect("signInState").into()) + .expect("a claude-code account row") +} + +async fn wait_for_status(core: &EngineCore, chat: &str, status: SessionStatus) { + wait_for( + || { + core.sessions + .session_status(chat) + .is_some_and(|s| s.status == status) + }, + "the chat to settle", + ) + .await; +} + +/// A run error of kind signed-out marks the harness's active account signed +/// out — the row says so through `ListAgentAccounts` even though its cached +/// meters never left — and a later successful usage probe marks it signed +/// in again. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_signed_out_run_error_marks_the_active_account_until_a_probe_clears_it() { + let (core, client) = setup(false).await; + core.doc_host.open(CHAT).expect("chat doc"); + + // Before any run: the live login is unknown (never probed). + assert_eq!(claude_sign_in_state(&client, false).await, "unknown"); + + core.sessions + .dispatch(CHAT, HarnessId::ClaudeCode, run_request("hello"), None, None, None) + .await + .expect("dispatch"); + wait_for_status(&core, CHAT, SessionStatus::Errored).await; + + // The run's error marked the LIVE account signed out — no probe ran + // (the mark is offline), so this is the run's evidence, not a probe's. + assert_eq!(claude_sign_in_state(&client, false).await, "signed-out"); + + // A later successful probe re-establishes the account: the forced list + // probes the mock usage endpoint, which answers green. + assert_eq!(claude_sign_in_state(&client, true).await, "signed-in"); +} + +/// A subagent's signed-out error marks the shared login just the same: the +/// child runs on the same credentials the parent does. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_subagents_signed_out_error_marks_the_shared_login_too() { + let (core, client) = setup(true).await; + core.doc_host.open(CHAT).expect("chat doc"); + + assert_eq!(claude_sign_in_state(&client, false).await, "unknown"); + core.sessions + .dispatch(CHAT, HarnessId::ClaudeCode, run_request("hello"), None, None, None) + .await + .expect("dispatch"); + wait_for_status(&core, CHAT, SessionStatus::Errored).await; + + assert_eq!(claude_sign_in_state(&client, false).await, "signed-out"); +} diff --git a/crates/engine/tests/acp_lifecycle.rs b/crates/engine/tests/acp_lifecycle.rs index 076f2ea756..3fc68a3ba5 100644 --- a/crates/engine/tests/acp_lifecycle.rs +++ b/crates/engine/tests/acp_lifecycle.rs @@ -40,6 +40,7 @@ async fn quiet_acp_prompt_stays_working_until_response() { let doc = handle.doc(); let queue = |id: &str, payload| { doc.queue_command(&SessionCommandEntry { + origin_chat_id: None, id: id.into(), payload, issued_by: "viewer".into(), @@ -68,6 +69,8 @@ async fn quiet_acp_prompt_stays_working_until_response() { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }, }, ); diff --git a/crates/engine/tests/adapter_chaos_live.rs b/crates/engine/tests/adapter_chaos_live.rs index 3b2eb24cd2..41560d5351 100644 --- a/crates/engine/tests/adapter_chaos_live.rs +++ b/crates/engine/tests/adapter_chaos_live.rs @@ -151,6 +151,8 @@ impl Suite { attachments: vec![], worktree: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } @@ -1215,6 +1217,8 @@ fn restart_request(name: &str, cwd: &str, prompt: &str) -> RunRequest { attachments: vec![], worktree: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } diff --git a/crates/engine/tests/agent_origin.rs b/crates/engine/tests/agent_origin.rs new file mode 100644 index 0000000000..e69c461d83 --- /dev/null +++ b/crates/engine/tests/agent_origin.rs @@ -0,0 +1,908 @@ +//! Message origin and agent-created side chats (orchestration v2, ticket 11). +//! +//! The groundwork completion notices need, all of it additive on the wire: +//! - a side chat created through the Roboco MCP surface (`Mutate createChat` +//! with `agentOrigin`) durably records the creating chat and its notify +//! flag, and survives engine restarts; +//! - user-made chats — forks, blank side chats, plain creates — carry no +//! agent origin and never notify; +//! - Run and Steer commands sent with an origin chat (`QueueCommand +//! originChatId`), and queued rows (`QueueMessage originChatId`), stamp +//! that origin on the user message entry they write, through every +//! delivery path: run, steer, hold-for-turn-end — and the Stop that +//! cancels a held send writes it unrun with the origin kept. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use futures::StreamExt; +use futures::stream::BoxStream; + +use roboco_doc::{MessagePart, MessageRole, SessionMessageEntry}; +use roboco_engine::{EngineCore, HarnessRegistry}; +use roboco_harness::{Harness, HarnessError, RunControls}; +use roboco_proto::{ + AgentEvent, DoneStatus, HarnessId, Model, ReasoningLevel, RunRequest, SandboxLevel, + SessionStatus, SteeringMode, +}; + +const CHAT: &str = "chat-origin"; +const PARENT: &str = "chat-parent-1"; + +/// A turn that runs until the test says so, so "busy" is a state the test +/// controls. Steers are accepted through the mailbox (the entry is written +/// before the harness ever sees the prompt) and confirmed with a `Steered` +/// boundary. +struct HeldHarness { + steering: SteeringMode, + finish: tokio::sync::broadcast::Sender<()>, + prompts: Arc>>, +} + +impl HeldHarness { + fn new(steering: SteeringMode) -> Arc { + let (finish, _) = tokio::sync::broadcast::channel(16); + Arc::new(Self { + steering, + finish, + prompts: Arc::new(Mutex::new(Vec::new())), + }) + } +} + +#[async_trait] +impl Harness for HeldHarness { + fn id(&self) -> HarnessId { + HarnessId::Mock + } + fn display_name(&self) -> &str { + "Held" + } + fn supports_steering(&self) -> bool { + true + } + fn steering_mode(&self) -> SteeringMode { + self.steering + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + controls: RunControls, + ) -> Result>, HarnessError> { + self.prompts.lock().unwrap().push(request.prompt.clone()); + let mut finish = self.finish.subscribe(); + let mut steering = controls.steering; + let (tx, mut rx) = tokio::sync::mpsc::channel(16); + let _ = tx.try_send(Ok(AgentEvent::SessionStarted { + harness: HarnessId::Mock, + model: "mock-1".into(), + tools: vec![], + cwd: request.cwd.clone(), + session_id: "sess-origin".into(), + assistant_message_id: format!("a-{}", request.prompt), + })); + tokio::spawn(async move { + let done = || { + Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some("sess-origin".into()), + }) + }; + loop { + tokio::select! { + _ = finish.recv() => { + let _ = tx.send(done()).await; + break; + } + steer = steering.recv() => { + if steer.is_none() { + let _ = tx.send(done()).await; + break; + } + // Confirm the steer's boundary like a real + // step-boundary agent: an accepted steer that never + // gets one is re-dispatched as a fresh turn when the + // run exits (a sent message must never silently not + // run), and that turn would hang past the test's + // one `finish`. + if tx + .send(Ok(AgentEvent::Steered { + assistant_message_id: None, + next_assistant_message_id: None, + })) + .await + .is_err() + { + break; + } + } + } + } + }); + Ok(futures::stream::poll_fn(move |cx| rx.poll_recv(cx)).boxed()) + } +} + +fn run_request(prompt: &str) -> RunRequest { + RunRequest { + mcp: None, + prompt: prompt.into(), + harness: Some(HarnessId::Mock), + model: None, + reasoning: None, + model_options: Default::default(), + cwd: "/tmp".into(), + sandbox: SandboxLevel::WorkspaceWrite, + auto_approve: false, + resume: None, + provider_fork: None, + resume_fallback_history: None, + attachments: vec![], + worktree: None, + } +} + +async fn wait_for(mut predicate: F, what: &str) +where + F: FnMut() -> bool, +{ + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while !predicate() { + assert!( + tokio::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(Duration::from_millis(15)).await; + } +} + +fn setup( + steering: SteeringMode, +) -> ( + EngineCore, + Arc, + Arc>>, + std::path::PathBuf, +) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.keep(); + let harness = HeldHarness::new(steering); + let prompts = harness.prompts.clone(); + let registry = HarnessRegistry::new(); + registry.register(harness.clone()); + let core = EngineCore::assemble(&path, Arc::new(registry), HarnessId::Mock) + .expect("engine core assembles"); + (core, harness, prompts, path) +} + +fn entries(core: &EngineCore, chat: &str) -> Vec { + core.doc_host + .open(chat) + .expect("chat doc") + .doc() + .read_entries() + .expect("transcript") +} + +/// The user entry whose text is exactly `text`, with its origin chat. +fn user_entry_origin(core: &EngineCore, chat: &str, text: &str) -> Option> { + entries(core, chat) + .into_iter() + .find(|entry| { + entry.role == MessageRole::User + && entry + .parts + .iter() + .any(|part| matches!(part, MessagePart::Text { text: body, .. } if body == text)) + }) + .map(|entry| entry.origin_chat_id) +} + +async fn wait_idle(core: &EngineCore, chat: &str) { + wait_for( + || { + core.sessions + .session_status(chat) + .is_some_and(|s| s.status == SessionStatus::Idle) + }, + "the chat to go idle", + ) + .await; +} + +/// An agent-made side chat records its creator and notify flag durably; the +/// row is the record completion notices will key on, so it must survive a +/// restart with the chat itself. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn agent_created_side_chats_record_their_creator_and_notify_durably() { + let (core, harness, _prompts, data_dir) = setup(SteeringMode::StepBoundary); + let client = roboco_rpc::memory_client(core.rpc_service()); + + for (chat, notify) in [("side-notify", None), ("side-quiet", Some(false))] { + let mut params = serde_json::json!({ + "op": "createChat", + "chatId": chat, + "deviceId": core.device_id, + "parentChatId": PARENT, + "agentOrigin": { "chatId": PARENT }, + }); + if let Some(notify) = notify { + params["agentOrigin"]["notify"] = serde_json::json!(notify); + } + client + .call(roboco_rpc::methods::MUTATE, params) + .await + .expect("createChat with agentOrigin"); + } + + let notify_row = core.workspace.chat("side-notify").unwrap().unwrap(); + assert_eq!( + notify_row.agent_origin, + Some(roboco_proto::AgentOrigin { + chat_id: PARENT.into(), + notify: true, + }), + "the notify flag defaults to on" + ); + assert_eq!(notify_row.parent_chat_id.as_deref(), Some(PARENT)); + let quiet_row = core.workspace.chat("side-quiet").unwrap().unwrap(); + assert_eq!( + quiet_row.agent_origin, + Some(roboco_proto::AgentOrigin { + chat_id: PARENT.into(), + notify: false, + }), + "notify: false is honoured" + ); + + let _ = harness.finish.send(()); + core.shutdown().await; + drop(client); + // The data-dir instance lock lives as long as the core: drop it before + // the restart, exactly like an app restart releases it. + drop(core); + + // The row is durable: the restart reads it back exactly. + let restarted = + EngineCore::assemble(&data_dir, Arc::new(HarnessRegistry::new()), HarnessId::Mock) + .unwrap(); + assert_eq!( + restarted.workspace.chat("side-notify").unwrap().unwrap().agent_origin, + Some(roboco_proto::AgentOrigin { + chat_id: PARENT.into(), + notify: true, + }) + ); + assert_eq!( + restarted.workspace.chat("side-quiet").unwrap().unwrap().agent_origin, + Some(roboco_proto::AgentOrigin { + chat_id: PARENT.into(), + notify: false, + }) + ); + restarted.shutdown().await; +} + +/// User-made chats are never marked: a fork, a blank side chat and a plain +/// create all carry no agent origin, so none of them can ever notify. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn user_created_chats_are_never_marked_agent_created() { + let (core, harness, _prompts, _data_dir) = setup(SteeringMode::StepBoundary); + let client = roboco_rpc::memory_client(core.rpc_service()); + + // A plain create. + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": "plain", + "deviceId": core.device_id, + }), + ) + .await + .expect("createChat"); + // A blank side chat (the UI's "+ New Side Chat"). + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": "blank-side", + "deviceId": core.device_id, + "parentChatId": "plain", + }), + ) + .await + .expect("createChat"); + // A createChat that carries a parent but no agentOrigin (a human at a + // terminal naming a parent). + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": "human-side", + "deviceId": core.device_id, + "parentChatId": "plain", + }), + ) + .await + .expect("createChat"); + + // A fork of a chat with history: the copy lists under the parent but is + // user-made. The fork button copies through the latest complete reply, + // so the source needs a settled turn, not just a user message. + settled_turn(&core, "plain"); + client + .call( + roboco_rpc::methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "fork", "sourceChatId": "plain" }), + ) + .await + .expect("fork"); + + for chat in ["plain", "blank-side", "human-side", "fork"] { + let row = core.workspace.chat(chat).unwrap().unwrap(); + assert_eq!( + row.agent_origin, None, + "{chat} is user-created and must never be marked" + ); + } + assert_eq!( + core.workspace.chat("fork").unwrap().unwrap().parent_chat_id, + Some("plain".into()) + ); + + let _ = harness.finish.send(()); + core.shutdown().await; +} + +/// A source chat with one settled turn, so the fork RPC has a boundary to +/// copy through. +fn settled_turn(core: &EngineCore, chat: &str) { + let doc = core.doc_host.open(chat).expect("chat doc"); + for (id, role, text) in [ + ("u1", MessageRole::User, "remember this"), + ("a1", MessageRole::Assistant, "noted"), + ] { + doc.doc() + .push_message(&SessionMessageEntry { + origin_chat_id: None, + id: id.into(), + role, + parts: vec![MessagePart::Text { + id: format!("t-{id}"), + text: text.into(), + }], + created_at: 1, + device_id: core.device_id.clone(), + status: Some(roboco_doc::MessageStatus::Complete), + continuation_of: None, + duration_ms: None, + provider_turn: None, + }) + .expect("push entry"); + } +} + +/// An agent-made fork (the `fork_chat` MCP tool, ticket 23) stamps the +/// creator and notify flag on the copy — the row completion notices key on +/// — while a fork RPC without an `agentOrigin` stays user-made and never +/// notifies, even when the source row itself was agent-created. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn agent_made_forks_notify_and_user_forks_stay_plain() { + let (core, harness, _prompts, _data_dir) = setup(SteeringMode::StepBoundary); + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": "fork-source", + "deviceId": core.device_id, + }), + ) + .await + .expect("createChat"); + settled_turn(&core, "fork-source"); + + // An agent fork: the agentOrigin param rides the RPC (notify omitted + // reads as on, notify: false opts out), and the copy hangs under the + // creator the tool names. + let fork = client + .call_as::( + roboco_rpc::methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": "agent-fork-on", + "sourceChatId": "fork-source", + "parentChatId": PARENT, + "agentOrigin": { "chatId": PARENT }, + }), + ) + .await + .expect("agent fork"); + assert_eq!( + fork.agent_origin, + Some(roboco_proto::AgentOrigin { + chat_id: PARENT.into(), + notify: true, + }) + ); + client + .call( + roboco_rpc::methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": "agent-fork-off", + "sourceChatId": "fork-source", + "parentChatId": PARENT, + "agentOrigin": { "chatId": PARENT, "notify": false }, + }), + ) + .await + .expect("agent fork"); + for (chat, notify) in [("agent-fork-on", true), ("agent-fork-off", false)] { + let row = core.workspace.chat(chat).unwrap().unwrap(); + assert_eq!( + row.agent_origin, + Some(roboco_proto::AgentOrigin { + chat_id: PARENT.into(), + notify, + }), + "{chat} records its creator and notify flag" + ); + assert_eq!(row.parent_chat_id.as_deref(), Some(PARENT)); + } + + // A fork without the param (the UI's fork button): user-made, hanging + // under the source as always. + client + .call( + roboco_rpc::methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "ui-fork", "sourceChatId": "fork-source" }), + ) + .await + .expect("ui fork"); + let row = core.workspace.chat("ui-fork").unwrap().unwrap(); + assert_eq!(row.agent_origin, None); + assert_eq!(row.parent_chat_id.as_deref(), Some("fork-source")); + + // The strip: a USER fork of an agent-created side chat is not marked, + // even though the source row carried an origin (ticket 11 — user forks + // never notify). + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": "agent-side", + "deviceId": core.device_id, + "parentChatId": PARENT, + "agentOrigin": { "chatId": PARENT }, + }), + ) + .await + .expect("createChat"); + settled_turn(&core, "agent-side"); + client + .call( + roboco_rpc::methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": "user-sibling", + "sourceChatId": "agent-side", + "parentChatId": PARENT, + }), + ) + .await + .expect("sibling fork"); + let row = core.workspace.chat("user-sibling").unwrap().unwrap(); + assert_eq!( + row.agent_origin, None, + "a user fork never inherits the source's agent origin" + ); + assert_eq!(row.parent_chat_id.as_deref(), Some(PARENT)); + + let _ = harness.finish.send(()); + core.shutdown().await; +} + +/// A Run command sent by a chat through the Roboco MCP surface carries its +/// origin on the ledger row AND on the user message entry it writes. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn run_commands_stamp_their_origin_on_the_user_entry() { + let (core, harness, prompts, _data_dir) = setup(SteeringMode::StepBoundary); + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": CHAT, + "deviceId": core.device_id, + }), + ) + .await + .expect("createChat"); + + let command = serde_json::to_value(roboco_doc::SessionCommandPayload::Run { + request: run_request("from the orchestrator"), + message_id: "m-run-1".into(), + }) + .unwrap(); + client + .call( + roboco_rpc::methods::QUEUE_COMMAND, + serde_json::json!({ + "chatId": CHAT, + "command": command, + "originChatId": PARENT, + }), + ) + .await + .expect("queue run"); + + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "from the orchestrator"), + "the run to dispatch", + ) + .await; + + // The ledger row keeps the origin… + let commands = core.doc_host.open(CHAT).unwrap().doc().read_commands().unwrap(); + let run = commands + .iter() + .find(|c| c.payload.kind() == roboco_doc::SessionCommandKind::Run) + .expect("run command in ledger"); + assert_eq!(run.origin_chat_id.as_deref(), Some(PARENT)); + // …and so does the user entry the run wrote. + assert_eq!( + user_entry_origin(&core, CHAT, "from the orchestrator"), + Some(Some(PARENT.into())), + "the user entry keeps the originating chat" + ); + + let _ = harness.finish.send(()); + wait_idle(&core, CHAT).await; + core.shutdown().await; +} + +/// A Steer command into a live steerable run writes its user entry with the +/// origin (the mid-run mailbox path). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn steer_commands_stamp_their_origin_on_the_user_entry() { + let (core, harness, prompts, _data_dir) = setup(SteeringMode::StepBoundary); + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": CHAT, + "deviceId": core.device_id, + }), + ) + .await + .expect("createChat"); + + // Start a turn and let it hang. + client + .call( + roboco_rpc::methods::QUEUE_COMMAND, + serde_json::json!({ + "chatId": CHAT, + "command": serde_json::to_value(roboco_doc::SessionCommandPayload::Run { + request: run_request("opening turn"), + message_id: "m-open".into(), + }).unwrap(), + }), + ) + .await + .expect("queue opening run"); + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "opening turn"), + "the opening turn to start", + ) + .await; + + let steer = serde_json::to_value(roboco_doc::SessionCommandPayload::Steer { + prompt: "mid-run course fix".into(), + message_id: Some("m-steer-1".into()), + }) + .unwrap(); + client + .call( + roboco_rpc::methods::QUEUE_COMMAND, + serde_json::json!({ + "chatId": CHAT, + "command": steer, + "originChatId": PARENT, + }), + ) + .await + .expect("queue steer"); + + wait_for( + || user_entry_origin(&core, CHAT, "mid-run course fix").is_some(), + "the steer's user entry to land", + ) + .await; + let commands = core.doc_host.open(CHAT).unwrap().doc().read_commands().unwrap(); + let steer = commands + .iter() + .find(|c| matches!(&c.payload, roboco_doc::SessionCommandPayload::Steer { message_id, .. } if message_id.as_deref() == Some("m-steer-1"))) + .expect("steer command in ledger"); + assert_eq!(steer.origin_chat_id.as_deref(), Some(PARENT)); + assert_eq!( + user_entry_origin(&core, CHAT, "mid-run course fix"), + Some(Some(PARENT.into())) + ); + + let _ = harness.finish.send(()); + wait_idle(&core, CHAT).await; + core.shutdown().await; +} + +/// A queued row sent by a chat keeps its origin through the hold and the +/// promotion at turn end (the `mode: "queue"` delivery). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn queued_rows_keep_their_origin_when_promoted() { + let (core, harness, prompts, _data_dir) = setup(SteeringMode::TurnBoundary); + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": CHAT, + "deviceId": core.device_id, + }), + ) + .await + .expect("createChat"); + + // Start a turn and let it hang: a turn-boundary agent holds the row. + client + .call( + roboco_rpc::methods::QUEUE_COMMAND, + serde_json::json!({ + "chatId": CHAT, + "command": serde_json::to_value(roboco_doc::SessionCommandPayload::Run { + request: run_request("opening turn"), + message_id: "m-open".into(), + }).unwrap(), + }), + ) + .await + .expect("queue opening run"); + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "opening turn"), + "the opening turn to start", + ) + .await; + + let queued = client + .call( + roboco_rpc::methods::QUEUE_MESSAGE, + serde_json::json!({ + "chatId": CHAT, + "text": "held from the orchestrator", + "originChatId": PARENT, + }), + ) + .await + .expect("queue message"); + let queued_id = queued["id"].as_str().unwrap().to_owned(); + + // The row itself carries the origin while it waits. + let row = core + .doc_host + .open(CHAT) + .unwrap() + .doc() + .read_queue() + .unwrap() + .into_iter() + .find(|row| row.id == queued_id) + .expect("held row"); + assert_eq!(row.origin_chat_id.as_deref(), Some(PARENT)); + + // End the turn: the row is promoted and its user entry keeps the origin. + let _ = harness.finish.send(()); + wait_for( + || user_entry_origin(&core, CHAT, "held from the orchestrator").is_some(), + "the held row to be promoted", + ) + .await; + wait_for( + || { + core.doc_host + .open(CHAT) + .unwrap() + .doc() + .read_queue() + .unwrap() + .is_empty() + }, + "the queue to empty", + ) + .await; + assert_eq!( + user_entry_origin(&core, CHAT, "held from the orchestrator"), + Some(Some(PARENT.into())) + ); + + // Let the promoted turn end too, then settle. + let _ = harness.finish.send(()); + wait_idle(&core, CHAT).await; + core.shutdown().await; +} + +/// A Stop drops the sends it cancels: the held steer leaves the queue and +/// lands in the transcript unrun — still carrying its origin chat. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_stop_cancels_a_held_send_but_keeps_its_origin() { + let (core, harness, prompts, _data_dir) = setup(SteeringMode::TurnBoundary); + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": CHAT, + "deviceId": core.device_id, + }), + ) + .await + .expect("createChat"); + + // Start a turn and let it hang: a turn-boundary agent holds the steer. + client + .call( + roboco_rpc::methods::QUEUE_COMMAND, + serde_json::json!({ + "chatId": CHAT, + "command": serde_json::to_value(roboco_doc::SessionCommandPayload::Run { + request: run_request("opening turn"), + message_id: "m-open".into(), + }).unwrap(), + }), + ) + .await + .expect("queue opening run"); + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "opening turn"), + "the opening turn to start", + ) + .await; + + // A steer sent by the orchestrator mid-turn: held for the turn's end. + client + .call( + roboco_rpc::methods::QUEUE_COMMAND, + serde_json::json!({ + "chatId": CHAT, + "command": serde_json::to_value(roboco_doc::SessionCommandPayload::Steer { + prompt: "stopped mid course".into(), + message_id: Some("m-stopped".into()), + }).unwrap(), + "originChatId": PARENT, + }), + ) + .await + .expect("queue steer"); + wait_for( + || { + core.doc_host + .open(CHAT) + .unwrap() + .doc() + .read_queue() + .unwrap() + .iter() + .any(|row| row.id == "m-stopped") + }, + "the steer to be held for the turn's end", + ) + .await; + + // Stop: the held send is cancelled — it lands in the transcript unrun, + // with the origin it was sent with. + client + .call( + roboco_rpc::methods::QUEUE_COMMAND, + serde_json::json!({ + "chatId": CHAT, + "command": serde_json::to_value(roboco_doc::SessionCommandPayload::Interrupt {}) .unwrap(), + }), + ) + .await + .expect("queue stop"); + wait_for( + || user_entry_origin(&core, CHAT, "stopped mid course").is_some(), + "the cancelled steer to land unrun in the transcript", + ) + .await; + assert_eq!( + user_entry_origin(&core, CHAT, "stopped mid course"), + Some(Some(PARENT.into())) + ); + wait_for( + || { + core.doc_host + .open(CHAT) + .unwrap() + .doc() + .read_queue() + .unwrap() + .is_empty() + }, + "the cancelled row to leave the queue", + ) + .await; + assert!( + !prompts + .lock() + .unwrap() + .iter() + .any(|p| p == "stopped mid course"), + "the cancelled steer never reaches the agent" + ); + + let _ = harness.finish.send(()); + wait_idle(&core, CHAT).await; + core.shutdown().await; +} + +/// Typed messages (the composer path) never carry an origin. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn typed_sends_carry_no_origin() { + let (core, harness, prompts, _data_dir) = setup(SteeringMode::StepBoundary); + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": CHAT, + "deviceId": core.device_id, + }), + ) + .await + .expect("createChat"); + + client + .call( + roboco_rpc::methods::QUEUE_COMMAND, + serde_json::json!({ + "chatId": CHAT, + "command": serde_json::to_value(roboco_doc::SessionCommandPayload::Run { + request: run_request("typed by the user"), + message_id: "m-typed".into(), + }).unwrap(), + }), + ) + .await + .expect("queue typed run"); + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "typed by the user"), + "the typed run to dispatch", + ) + .await; + + assert_eq!(user_entry_origin(&core, CHAT, "typed by the user"), Some(None)); + let commands = core.doc_host.open(CHAT).unwrap().doc().read_commands().unwrap(); + assert!(commands.iter().all(|c| c.origin_chat_id.is_none())); + + let _ = harness.finish.send(()); + wait_idle(&core, CHAT).await; + core.shutdown().await; +} diff --git a/crates/engine/tests/codex_subagents.rs b/crates/engine/tests/codex_subagents.rs index 83ecc5ef3b..8ad48626b3 100644 --- a/crates/engine/tests/codex_subagents.rs +++ b/crates/engine/tests/codex_subagents.rs @@ -65,6 +65,8 @@ async fn check_persistence( worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }; core.sessions .dispatch( @@ -72,7 +74,8 @@ async fn check_persistence( HarnessId::Codex, request.clone(), Some("user-prompt".into()), - ) + None, + None) .await .unwrap(); tokio::time::timeout(Duration::from_secs(10), async { @@ -177,7 +180,8 @@ async fn check_persistence( HarnessId::Codex, request, Some("user-followup".into()), - ) + None, + None) .await .unwrap(); tokio::time::timeout(Duration::from_secs(10), async { diff --git a/crates/engine/tests/completion_notices.rs b/crates/engine/tests/completion_notices.rs new file mode 100644 index 0000000000..7c6d698407 --- /dev/null +++ b/crates/engine/tests/completion_notices.rs @@ -0,0 +1,2446 @@ +//! Completion notices (orchestration v2, tickets 12 + 13). +//! +//! Ticket 12's tracer bullet for the spec's completion-notice stories +//! 13–17 and 19: +//! - a turn of an agent-created, NOTIFYING side chat is notifiable when its +//! starting message, or a steer within it, came from the parent; +//! - when such a turn ends, the engine records an outcome from its OWN +//! turn-end state — completed; errored with a one-line reason; stopped; +//! waiting for input with a one-line summary of the question — and wakes +//! an idle parent with one engine-authored notice message that starts a +//! parent turn; +//! - the notice is a SYSTEM entry (never a user message) carrying a +//! completion-notice marker for clients to render distinctly (ticket 15), +//! names the side chat by title and short id with its outcome, and tells +//! the agent to use `read_chat` for the full result; +//! - a user-started turn in the child, a user-created side chat, and a +//! `notify: false` child never wake the parent. +//! +//! Ticket 13's rules on top: +//! - a BUSY parent is never steered: the notice waits for the parent's +//! turn to end, then starts its next turn; +//! - several children finishing while the parent works merge into ONE +//! notice; +//! - pending outcomes survive an engine restart and are delivered after +//! boot; +//! - a notifiable run lost to a restart without auto-resume yields an +//! INTERRUPTED outcome; with auto-resume, the notice follows the resumed +//! turn; +//! - archived parents receive nothing and archived side chats send +//! nothing. + +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use futures::StreamExt; +use futures::stream::BoxStream; + +use roboco_doc::{ + MessagePart, MessageRole, MessageStatus, NoticeOutcome, NoticeSide, SessionCommandPayload, + SessionDoc, SessionMessageEntry, +}; +use roboco_engine::{EngineCore, HarnessRegistry, RunJournal}; +use roboco_harness::{Harness, HarnessError, RunControls}; +use roboco_proto::{ + AgentEvent, DoneStatus, HarnessId, Model, ReasoningLevel, RunRequest, SandboxLevel, + SessionStatus, SteeringMode, UserInputQuestion, +}; +use roboco_sync::DocsStore; + +const PARENT: &str = "chat-parent"; +const SIDE: &str = "chat-side"; +const SIDE2: &str = "chat-side-2"; + +/// What one `run()` call does — the tests script turns FIFO, one entry per +/// run the engine starts (a side chat's turn, then the parent's notice +/// turn, …). +enum Turn { + /// Stream one line of text, then Done with the status and error. + End { + status: DoneStatus, + error: Option, + }, + /// Ask a question through the engine's input bridge, then park awaiting + /// the answer (the turn stays open, the chat awaits input). + AskInput { question: &'static str }, + /// Hang until a steer arrives: consume it, emit the `Steered` boundary, + /// then settle the turn Completed (what a steered run does when its + /// work finishes). A Stop (the engine's interrupt token) ends the + /// parked turn as Interrupted, the way a real harness does. + Hang, +} + +/// A steerable mock whose turns the test scripts. Every run's prompt is +/// recorded, so a test can prove the parent was — or was never — woken by +/// a notice. +struct Scripted { + turns: Arc>>, + prompts: Arc>>, + /// The prompts of steers the mock consumed (delivered mid-run). + steered: Arc>>, +} + +impl Scripted { + fn new(turns: Vec) -> Arc { + Arc::new(Self { + turns: Arc::new(Mutex::new(turns.into())), + prompts: Arc::new(Mutex::new(Vec::new())), + steered: Arc::new(Mutex::new(Vec::new())), + }) + } + + fn run_count(&self) -> usize { + self.prompts.lock().unwrap().len() + } + + fn prompts_containing(&self, needle: &str) -> bool { + self.prompts + .lock() + .unwrap() + .iter() + .any(|p| p.contains(needle)) + } + + /// How many runs so far carried `needle` in their prompt. + fn prompt_count(&self, needle: &str) -> usize { + self.prompts + .lock() + .unwrap() + .iter() + .filter(|p| p.contains(needle)) + .count() + } +} + +#[async_trait] +impl Harness for Scripted { + fn id(&self) -> HarnessId { + HarnessId::Mock + } + fn display_name(&self) -> &str { + "Scripted" + } + fn supports_steering(&self) -> bool { + true + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::StepBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + controls: RunControls, + ) -> Result>, HarnessError> { + // Take the turn before recording the prompt: a test that sees this + // prompt may start the next run at once, and that run must never + // take the turn scripted for this one. + let turn = self + .turns + .lock() + .unwrap() + .pop_front() + .unwrap_or(Turn::Hang); + self.prompts.lock().unwrap().push(request.prompt.clone()); + let session = format!("sess-{}", self.run_count()); + let (tx, rx) = tokio::sync::mpsc::channel::>(32); + let steered = self.steered.clone(); + let mut steering = controls.steering; + let request_input = controls.request_input; + let interrupt = controls.interrupt; + let started_session = session.clone(); + let cwd = request.cwd.clone(); + tokio::spawn(async move { + let _ = tx + .send(Ok(AgentEvent::SessionStarted { + harness: HarnessId::Mock, + model: "mock-1".into(), + tools: vec![], + cwd, + session_id: started_session, + assistant_message_id: "a0".into(), + })) + .await; + match turn { + Turn::End { status, error } => { + let _ = tx + .send(Ok(AgentEvent::TextDelta { + text: "side answer".into(), + })) + .await; + let _ = tx + .send(Ok(AgentEvent::Done { + status, + result: None, + error, + session_id: Some(session), + })) + .await; + } + Turn::AskInput { question } => { + // The engine's input bridge mints the id and parks the + // resolver before the event reaches the run loop, so a + // legitimate question is always known here. + let questions = vec![UserInputQuestion { + id: "q1".into(), + header: "Approval".into(), + question: question.into(), + options: vec!["Yes".into(), "No".into()], + multi_select: false, + }]; + // Awaiting the answer parks the turn: the chat goes to + // awaiting input and stays there. + let _ = (request_input)(questions).await; + let _ = tx + .send(Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some(session), + })) + .await; + } + Turn::Hang => loop { + tokio::select! { + steer = steering.recv() => match steer { + Some(message) => { + steered.lock().unwrap().push(message.prompt.clone()); + // The steer's boundary, then the steered turn + // settles: the engine retires the accepted + // steer at the boundary and parks the turn at + // Done. + if tx + .send(Ok(AgentEvent::Steered { + assistant_message_id: None, + next_assistant_message_id: None, + })) + .await + .is_err() + { + break; + } + let _ = tx + .send(Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some(session), + })) + .await; + break; + } + None => break, + }, + _ = interrupt.cancelled() => { + // A Stop: the turn ends interrupted, the runtime + // goes with it. + let _ = tx + .send(Ok(AgentEvent::Done { + status: DoneStatus::Interrupted, + result: None, + error: None, + session_id: Some(session), + })) + .await; + break; + }, + _ = tx.closed() => break, + } + }, + } + }); + Ok(futures::stream::unfold(rx, |mut rx| async move { + rx.recv().await.map(|event| (event, rx)) + }) + .boxed()) + } +} + +fn run_request(prompt: &str) -> RunRequest { + RunRequest { + mcp: None, + prompt: prompt.into(), + harness: Some(HarnessId::Mock), + model: None, + reasoning: None, + model_options: Default::default(), + cwd: "/tmp".into(), + sandbox: SandboxLevel::WorkspaceWrite, + auto_approve: false, + resume: None, + provider_fork: None, + resume_fallback_history: None, + attachments: vec![], + worktree: None, + } +} + +async fn wait_for(mut predicate: F, what: &str) +where + F: FnMut() -> bool, +{ + // 30 s: the suite boots ~25 engines in parallel on a small CI runner. + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while !predicate() { + assert!( + tokio::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(Duration::from_millis(15)).await; + } +} + +fn entries(core: &EngineCore, chat: &str) -> Vec { + core.doc_host + .open(chat) + .expect("chat doc") + .doc() + .read_entries() + .expect("transcript") +} + +/// The completion-notice entries in a chat: SYSTEM entries carrying the +/// marker — never user entries. +fn notice_entries(core: &EngineCore, chat: &str) -> Vec { + entries(core, chat) + .into_iter() + .filter(|entry| entry.role == MessageRole::System) + .filter(|entry| { + entry + .parts + .iter() + .any(|part| matches!(part, MessagePart::CompletionNotice { .. })) + }) + .collect() +} + +fn notice_text(entry: &SessionMessageEntry) -> String { + entry + .parts + .iter() + .find_map(|part| match part { + MessagePart::Text { text, .. } => Some(text.clone()), + _ => None, + }) + .unwrap_or_default() +} + +fn notice_sides(entry: &SessionMessageEntry) -> Vec { + entry + .parts + .iter() + .find_map(|part| match part { + MessagePart::CompletionNotice { sides, .. } => Some(sides.clone()), + _ => None, + }) + .unwrap_or_default() +} + +async fn wait_idle(core: &EngineCore, chat: &str) { + wait_for( + || { + core.sessions + .session_status(chat) + .is_some_and(|s| s.status == SessionStatus::Idle) + }, + "the chat to go idle", + ) + .await; +} + +/// Queue a Run for `chat` and wait for THAT run to start and settle. A bare +/// `wait_idle` right after the queue can pass on the chat's previous Idle +/// row before this run even dispatched; the run's own prompt reaching the +/// harness means its Working status already published (dispatch sets it +/// before the run task starts), so the idle wait after it is this run's. +async fn run_to_idle( + core: &EngineCore, + client: &roboco_rpc::RpcClient, + scripted: &Scripted, + chat: &str, + prompt: &str, + message_id: &str, + origin_chat_id: Option<&str>, +) { + let before = scripted.prompt_count(prompt); + queue_run(client, chat, prompt, message_id, origin_chat_id).await; + wait_for( + || scripted.prompt_count(prompt) > before, + &format!("the run of {prompt:?} to start"), + ) + .await; + wait_idle(core, chat).await; +} + +/// Create the orchestrating parent: a cwd'd, titled chat a notice turn can +/// run on. +async fn create_parent(core: &EngineCore, client: &roboco_rpc::RpcClient) { + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": PARENT, + "deviceId": core.device_id, + "cwd": "/tmp", + }), + ) + .await + .expect("createChat parent"); + core.workspace.rename_chat(PARENT, "Orchestrator").unwrap(); +} + +/// Create an agent-created, notifying side chat under the parent: titled, +/// with a cwd (a boot auto-resume re-derives its run config from the row) +/// and the `notify` opt in/out. +async fn create_side( + core: &EngineCore, + client: &roboco_rpc::RpcClient, + chat_id: &str, + title: &str, + notify: Option, +) { + let mut agent_origin = serde_json::json!({ "chatId": PARENT }); + if let Some(notify) = notify { + agent_origin["notify"] = serde_json::json!(notify); + } + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": chat_id, + "deviceId": core.device_id, + "parentChatId": PARENT, + "cwd": "/tmp", + "agentOrigin": agent_origin, + }), + ) + .await + .expect("createChat side"); + core.workspace.rename_chat(chat_id, title).unwrap(); +} + +/// The ticket-12 shape: one parent and one titled side chat. +async fn create_parent_and_side( + core: &EngineCore, + client: &roboco_rpc::RpcClient, + notify: Option, +) { + create_parent(core, client).await; + create_side(core, client, SIDE, "Implement login", notify).await; +} + +/// Queue a Run command for `chat`, optionally sent by the parent through the +/// MCP surface (`origin_chat_id`). +async fn queue_run( + client: &roboco_rpc::RpcClient, + chat: &str, + prompt: &str, + message_id: &str, + origin_chat_id: Option<&str>, +) { + let mut params = serde_json::json!({ + "chatId": chat, + "command": serde_json::to_value(roboco_doc::SessionCommandPayload::Run { + request: run_request(prompt), + message_id: message_id.into(), + }) + .unwrap(), + }); + if let Some(origin) = origin_chat_id { + params["originChatId"] = serde_json::json!(origin); + } + client + .call(roboco_rpc::methods::QUEUE_COMMAND, params) + .await + .expect("queue run"); +} + +/// Assemble an engine over `dir` whose runs the Scripted mock serves. +fn assemble(dir: &std::path::Path, scripted: Arc) -> EngineCore { + let registry = HarnessRegistry::new(); + registry.register(scripted); + EngineCore::assemble(dir, Arc::new(registry), HarnessId::Mock).expect("engine core assembles") +} + +/// A user's steer (no origin chat): the queue routes it into a working +/// chat's mailbox, and the mock's Hang turn consumes it and settles — the +/// way a working parent's turn ends. +async fn queue_user_steer( + client: &roboco_rpc::RpcClient, + chat: &str, + prompt: &str, + message_id: &str, +) { + client + .call( + roboco_rpc::methods::QUEUE_COMMAND, + serde_json::json!({ + "chatId": chat, + "command": serde_json::to_value(SessionCommandPayload::Steer { + prompt: prompt.into(), + message_id: Some(message_id.into()), + }) + .unwrap(), + }), + ) + .await + .expect("queue steer"); +} + +/// Rows the durable completion-notice ledger under this engine's data dir +/// currently holds — the engine-local surface restart round-trips assert +/// on, like the run journals. Ticket 14 wrapped the ledger as +/// `{ pending, loops }`; ticket 13's was a bare parent → rows map. Both +/// shapes read. +fn ledger_rows(data_dir: &std::path::Path) -> usize { + std::fs::read(data_dir.join("orgs/dev-org/dev-user/completion-notices.json")) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .and_then(|ledger| { + let parents = ledger.get("pending").cloned().unwrap_or(ledger); + parents.as_object().map(|parents| { + parents + .values() + .map(|rows| rows.as_array().map_or(0, Vec::len)) + .sum() + }) + }) + .unwrap_or(0) +} + +/// An idle parent is woken by a notice when its side chat's parent-started +/// turn completes: the notice is a system entry (never a user message) +/// carrying the marker, names the child by title and short id, points at +/// `read_chat`, and starts a parent turn on the parent's own harness. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_idle_parent_is_woken_when_its_side_chat_turn_completes() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![ + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + // The parent's notice turn. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let registry = HarnessRegistry::new(); + registry.register(scripted.clone()); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + // The parent starts the child's turn through the MCP surface. + queue_run(&client, SIDE, "please implement login", "m-run-1", Some(PARENT)).await; + wait_for( + || scripted.prompts_containing("please implement login"), + "the side chat's turn to start", + ) + .await; + + // The child's turn completes; the idle parent is woken by the notice. + wait_for( + || scripted.prompts_containing("finished"), + "the notice to start a parent turn", + ) + .await; + // The notice turn is the parent's SECOND run, on its own harness. + assert_eq!(scripted.run_count(), 2); + let notice_prompt = scripted.prompts.lock().unwrap()[1].clone(); + assert!(notice_prompt.contains("Side chat \"Implement login\"")); + assert!(notice_prompt.contains("read_chat")); + + let notices = notice_entries(&core, PARENT); + assert_eq!(notices.len(), 1, "exactly one notice entry"); + let notice = ¬ices[0]; + // Engine/system message, not a user message, and engine-authored (no + // originating chat). + assert_eq!(notice.role, MessageRole::System); + assert_eq!(notice.origin_chat_id, None); + assert_eq!(notice.status, Some(roboco_doc::MessageStatus::Complete)); + // The marker clients render distinctly (ticket 15). + assert_eq!( + notice_sides(notice), + vec![NoticeSide { + chat_id: SIDE.into(), + title: "Implement login".into(), + outcome: NoticeOutcome::Completed, + detail: None, + }] + ); + // The plain-text sibling names the child and points at read_chat. + let text = notice_text(notice); + let short = &SIDE[..8]; + assert!(text.contains(&format!("\"Implement login\" ({short}) finished"))); + assert!(text.contains("read_chat")); + assert!(text.contains(short)); + + // The parent's notice turn ran and settled. + wait_idle(&core, PARENT).await; + // The child's own transcript keeps its answer, unrun by the parent. + assert!(entries(&core, SIDE) + .iter() + .any(|e| e.parts.iter().any(|p| matches!(p, MessagePart::Text { text, .. } if text == "side answer")))); + + core.shutdown().await; +} + +/// A parent-started turn that errors notifies with the one-line reason — +/// the first line of the engine's recorded error, capped. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_errored_side_chat_turn_notifies_the_parent_with_the_reason() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![ + Turn::End { + status: DoneStatus::Errored, + error: Some( + "provider 502: bad gateway\nstream failed mid-turn\nretry later".into(), + ), + }, + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let registry = HarnessRegistry::new(); + registry.register(scripted.clone()); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + queue_run(&client, SIDE, "please implement login", "m-run-1", Some(PARENT)).await; + wait_for( + || scripted.prompts_containing("failed"), + "the failure notice to start a parent turn", + ) + .await; + + let notices = notice_entries(&core, PARENT); + assert_eq!(notices.len(), 1); + let sides = notice_sides(¬ices[0]); + assert_eq!( + sides, + vec![NoticeSide { + chat_id: SIDE.into(), + title: "Implement login".into(), + outcome: NoticeOutcome::Errored, + // One line: the first line of the recorded error only. + detail: Some("provider 502: bad gateway".into()), + }] + ); + let text = notice_text(¬ices[0]); + assert!(text.contains("failed: provider 502: bad gateway")); + assert!(!text.contains("retry later")); + assert!(text.contains("read_chat")); + + core.shutdown().await; +} + +/// A parent-started turn that is stopped or interrupted notifies as +/// stopped. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_stopped_side_chat_turn_notifies_the_parent() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![ + Turn::End { + status: DoneStatus::Interrupted, + error: None, + }, + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let registry = HarnessRegistry::new(); + registry.register(scripted.clone()); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + queue_run(&client, SIDE, "please implement login", "m-run-1", Some(PARENT)).await; + wait_for( + || scripted.prompts_containing("was stopped"), + "the stopped notice to start a parent turn", + ) + .await; + + let notices = notice_entries(&core, PARENT); + assert_eq!(notices.len(), 1); + let sides = notice_sides(¬ices[0]); + assert_eq!( + sides, + vec![NoticeSide { + chat_id: SIDE.into(), + title: "Implement login".into(), + outcome: NoticeOutcome::Stopped, + detail: None, + }] + ); + assert!(notice_text(¬ices[0]).contains("was stopped")); + + core.shutdown().await; +} + +/// A parent-started turn that reaches awaiting input notifies with a +/// one-line summary of the question it is waiting on. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_awaiting_input_side_chat_turn_notifies_with_the_question() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![ + Turn::AskInput { + question: "Should we bump the major version?", + }, + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let registry = HarnessRegistry::new(); + registry.register(scripted.clone()); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + queue_run(&client, SIDE, "please review the API", "m-run-1", Some(PARENT)).await; + // The child parks on its question; the parent is woken by the notice. + wait_for( + || { + core.sessions + .session_status(SIDE) + .is_some_and(|s| s.status == SessionStatus::AwaitingInput) + }, + "the side chat to await input", + ) + .await; + wait_for( + || scripted.prompts_containing("waiting for input"), + "the waiting-for-input notice to start a parent turn", + ) + .await; + + let notices = notice_entries(&core, PARENT); + assert_eq!(notices.len(), 1); + let sides = notice_sides(¬ices[0]); + assert_eq!( + sides, + vec![NoticeSide { + chat_id: SIDE.into(), + title: "Implement login".into(), + outcome: NoticeOutcome::WaitingForInput, + detail: Some("Should we bump the major version?".into()), + }] + ); + let text = notice_text(¬ices[0]); + assert!(text.contains("is waiting for input: Should we bump the major version?")); + assert!(text.contains("read_chat")); + + core.shutdown().await; +} + +/// A turn the USER started in the child (no parent-originated message) +/// produces no notice: the parent is never woken. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_user_started_turn_in_the_child_produces_no_notice() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![Turn::End { + status: DoneStatus::Completed, + error: None, + }]); + let registry = HarnessRegistry::new(); + registry.register(scripted.clone()); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + // A typed send: no origin chat on the command or the entry. + run_to_idle(&core, &client, &scripted, SIDE, "typed by the user", "m-run-1", None).await; + // Let any (wrongly recorded) notice delivery settle before asserting + // absence. + tokio::time::sleep(Duration::from_millis(300)).await; + + assert!( + notice_entries(&core, PARENT).is_empty(), + "a user-started turn must not wake the parent" + ); + assert!(entries(&core, PARENT).is_empty()); + assert_eq!( + scripted.run_count(), + 1, + "only the side chat's turn ran — no parent turn" + ); + // The typed send carries no origin on its user entry. + assert!(entries(&core, SIDE) + .iter() + .all(|e| e.origin_chat_id.is_none())); + + core.shutdown().await; +} + +/// A user-created side chat (no agent origin on its row) produces no notice +/// even when a chat-originated message reaches it: the durable creator row +/// is what makes turns notifiable. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_user_created_side_chat_produces_no_notice() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![Turn::End { + status: DoneStatus::Completed, + error: None, + }]); + let registry = HarnessRegistry::new(); + registry.register(scripted.clone()); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + // A side chat with a parent but NO agent origin: the UI's blank side + // chat shape. + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": PARENT, + "deviceId": core.device_id, + "cwd": "/tmp", + }), + ) + .await + .expect("createChat parent"); + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": SIDE, + "deviceId": core.device_id, + "parentChatId": PARENT, + }), + ) + .await + .expect("createChat blank side chat"); + + // Even a parent-originated message does not make its turns notifiable: + // there is no durable creator row to key on. + run_to_idle(&core, &client, &scripted, SIDE, "sent by a chat anyway", "m-run-1", Some(PARENT)).await; + tokio::time::sleep(Duration::from_millis(300)).await; + + assert!( + notice_entries(&core, PARENT).is_empty(), + "a user-created side chat must never notify" + ); + assert_eq!(scripted.run_count(), 1, "no parent turn was started"); + + core.shutdown().await; +} + +/// A `notify: false` side chat (fire-and-forget) records no outcomes for its +/// creator: the opt-out at creation is durable. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_notify_false_side_chat_produces_no_notice() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![Turn::End { + status: DoneStatus::Completed, + error: None, + }]); + let registry = HarnessRegistry::new(); + registry.register(scripted.clone()); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, Some(false)).await; + assert_eq!( + core.workspace.chat(SIDE).unwrap().unwrap().agent_origin, + Some(roboco_proto::AgentOrigin { + chat_id: PARENT.into(), + notify: false, + }) + ); + + run_to_idle(&core, &client, &scripted, SIDE, "fire and forget", "m-run-1", Some(PARENT)).await; + tokio::time::sleep(Duration::from_millis(300)).await; + + assert!( + notice_entries(&core, PARENT).is_empty(), + "a notify: false child must never wake its creator" + ); + assert_eq!(scripted.run_count(), 1, "no parent turn was started"); + + core.shutdown().await; +} + +/// A steer the parent sent WITHIN a user-started turn makes that turn +/// notifiable: the notice fires when the steered turn completes. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_parent_steer_within_the_turn_makes_it_notifiable() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![ + // The user starts the child's turn; it hangs. + Turn::Hang, + // The parent's notice turn, after the steered turn completes. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let registry = HarnessRegistry::new(); + registry.register(scripted.clone()); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + // The user's opening turn — no origin, not notifiable by itself. + queue_run(&client, SIDE, "typed by the user", "m-open", None).await; + wait_for( + || scripted.prompts_containing("typed by the user"), + "the user's turn to start", + ) + .await; + wait_for( + || { + core.sessions + .session_status(SIDE) + .is_some_and(|s| s.status == SessionStatus::Working) + }, + "the child to be working", + ) + .await; + + // The parent steers mid-run: the steer joins the turn in flight. + client + .call( + roboco_rpc::methods::QUEUE_COMMAND, + serde_json::json!({ + "chatId": SIDE, + "command": serde_json::to_value(roboco_doc::SessionCommandPayload::Steer { + prompt: "use the other approach".into(), + message_id: Some("m-steer-1".into()), + }) + .unwrap(), + "originChatId": PARENT, + }), + ) + .await + .expect("queue steer"); + // The mock consumes the steer and emits its Steered boundary. + wait_for( + || { + scripted + .steered + .lock() + .unwrap() + .iter() + .any(|p| p.contains("use the other approach")) + }, + "the steer to be consumed", + ) + .await; + // The user entry the steer wrote carries the parent's origin. + wait_for( + || { + entries(&core, SIDE) + .iter() + .any(|e| e.origin_chat_id.as_deref() == Some(PARENT)) + }, + "the steer's user entry to land", + ) + .await; + + // The steered turn completes at the mock's Done: the parent is woken by + // the completion notice. + wait_for( + || scripted.prompts_containing("finished"), + "the completion notice to start a parent turn", + ) + .await; + let notices = notice_entries(&core, PARENT); + assert_eq!(notices.len(), 1); + assert_eq!( + notice_sides(¬ices[0]), + vec![NoticeSide { + chat_id: SIDE.into(), + title: "Implement login".into(), + outcome: NoticeOutcome::Completed, + detail: None, + }] + ); + assert_eq!(scripted.run_count(), 2); + + core.shutdown().await; +} + +/// A busy parent is never steered: the child's outcome waits for the +/// parent's turn to end, and the notice then starts the parent's NEXT turn +/// (ticket 13). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_working_parent_receives_the_notice_after_its_turn_as_its_next_turn() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![ + // The parent's working turn: hangs until the user steers it. + Turn::Hang, + // The side chat's turn. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + // The parent's notice turn, after its turn ends. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let core = assemble(dir.path(), scripted.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + // The user starts the parent's turn; it hangs (the parent is working). + queue_run(&client, PARENT, "orchestrate the release", "m-parent-1", None).await; + wait_for( + || scripted.prompts_containing("orchestrate the release"), + "the parent's turn to start", + ) + .await; + + // The parent starts the child's turn through the MCP surface; the + // child completes while the parent still works. + run_to_idle(&core, &client, &scripted, SIDE, "please implement login", "m-run-1", Some(PARENT)).await; + + // No notice while the parent is busy: nothing steered in, no parent + // turn started, no entry written. The outcome is recorded durably + // (one ledger row) before the parent's turn ends. + assert!( + core.sessions.turn_in_flight(PARENT), + "the parent is still working" + ); + assert!(notice_entries(&core, PARENT).is_empty()); + assert_eq!(scripted.run_count(), 2); + assert!( + scripted.steered.lock().unwrap().is_empty(), + "nothing was steered into the parent mid-turn" + ); + wait_for( + || ledger_rows(dir.path()) == 1, + "the child's outcome to be recorded durably", + ) + .await; + + // The user's steer ends the parent's turn. + queue_user_steer(&client, PARENT, "please wrap up", "m-steer-1").await; + wait_for( + || scripted.prompts_containing("finished"), + "the notice to start the parent's next turn", + ) + .await; + + // ONE notice, delivered as the parent's next turn — never steered in. + let notices = notice_entries(&core, PARENT); + assert_eq!(notices.len(), 1); + assert_eq!( + notice_sides(¬ices[0]), + vec![NoticeSide { + chat_id: SIDE.into(), + title: "Implement login".into(), + outcome: NoticeOutcome::Completed, + detail: None, + }] + ); + assert_eq!(scripted.run_count(), 3); + assert_eq!( + scripted.steered.lock().unwrap().clone(), + vec!["please wrap up".to_string()], + "the only consumed steer is the user's" + ); + wait_idle(&core, PARENT).await; + + core.shutdown().await; +} + +/// Three children finishing while the parent works produce ONE merged +/// notice naming each of them (ticket 13). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn three_children_finishing_while_the_parent_works_merge_into_one_notice() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![ + // The parent's working turn. + Turn::Hang, + // The three children's turns. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + // The parent's merged notice turn. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let core = assemble(dir.path(), scripted.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent(&core, &client).await; + let sides = [ + ("chat-side-1", "Implement login"), + ("chat-side-2", "Write the docs"), + ("chat-side-3", "Run the tests"), + ]; + for (chat_id, title) in sides { + create_side(&core, &client, chat_id, title, None).await; + } + + queue_run(&client, PARENT, "orchestrate the release", "m-parent-1", None).await; + wait_for( + || scripted.prompts_containing("orchestrate the release"), + "the parent's turn to start", + ) + .await; + for (index, (chat_id, _)) in sides.iter().enumerate() { + queue_run( + &client, + chat_id, + "please do your part", + &format!("m-run-{index}"), + Some(PARENT), + ) + .await; + } + // Every child's run dispatched (its Working row published) before the + // idle waits read the rows. + wait_for( + || scripted.prompt_count("please do your part") == sides.len(), + "every child's turn to start", + ) + .await; + for (chat_id, _) in sides { + wait_idle(&core, chat_id).await; + } + + // All three outcomes are owed while the parent works: no notice yet, + // and all three are recorded durably before the turn ends — the merged + // notice is what the ledger holds at the boundary. + assert!(core.sessions.turn_in_flight(PARENT)); + assert!(notice_entries(&core, PARENT).is_empty()); + assert_eq!(scripted.run_count(), 4); + wait_for( + || ledger_rows(dir.path()) == 3, + "the three children's outcomes to be recorded durably", + ) + .await; + + // The user's steer ends the parent's turn: the merged notice is the + // next one. + queue_user_steer(&client, PARENT, "please wrap up", "m-steer-1").await; + wait_for( + || scripted.prompts_containing("3 side chats reached outcomes"), + "the merged notice to start the parent's next turn", + ) + .await; + + let notices = notice_entries(&core, PARENT); + assert_eq!(notices.len(), 1, "one merged notice, not one per child"); + let mut rows = notice_sides(¬ices[0]); + rows.sort_by(|a, b| a.chat_id.cmp(&b.chat_id)); + let mut expected: Vec = sides + .iter() + .map(|(chat_id, title)| NoticeSide { + chat_id: (*chat_id).into(), + title: (*title).into(), + outcome: NoticeOutcome::Completed, + detail: None, + }) + .collect(); + expected.sort_by(|a, b| a.chat_id.cmp(&b.chat_id)); + assert_eq!(rows, expected); + let text = notice_text(¬ices[0]); + assert!(text.starts_with("3 side chats reached outcomes:")); + for (chat_id, title) in sides { + assert!(text.contains(&format!("\"{title}\" ({}) finished", &chat_id[..8]))); + } + assert!(text.contains("Read each chat's full result")); + assert_eq!( + scripted.run_count(), + 5, + "the parent ran its own turn and one notice turn" + ); + wait_idle(&core, PARENT).await; + + core.shutdown().await; +} + +/// A pending notice survives an engine restart: the outcome a busy parent +/// never received is loaded from the durable ledger and delivered after +/// boot (restart-resume prior art: shutdown → relaunch over one dir). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_pending_notice_survives_an_engine_restart_and_is_delivered_after_boot() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("data"); + std::fs::create_dir_all(&dir).unwrap(); + + // Engine 1: the child completes while the parent works, so the outcome + // is recorded into the durable ledger and stays pending. + let first = Scripted::new(vec![ + Turn::Hang, + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let core = assemble(&dir, first.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + queue_run(&client, PARENT, "orchestrate the release", "m-parent-1", None).await; + wait_for( + || first.prompts_containing("orchestrate the release"), + "the parent's turn to start", + ) + .await; + run_to_idle(&core, &client, &first, SIDE, "please implement login", "m-run-1", Some(PARENT)).await; + assert!(core.sessions.turn_in_flight(PARENT)); + assert!(notice_entries(&core, PARENT).is_empty()); + // The outcome is recorded durably before the engine dies — that is the + // whole point of the ledger. + wait_for( + || ledger_rows(&dir) == 1, + "the child's outcome to be recorded durably", + ) + .await; + + // The engine dies: shutdown interrupts the parent's turn, and an + // interrupted turn wakes nobody, so the notice stays pending. + core.shutdown().await; + drop(core); + + // Engine 2 over the same data dir: the ledger survives, and the notice + // is delivered after boot. + let second = Scripted::new(vec![Turn::End { + status: DoneStatus::Completed, + error: None, + }]); + let core = assemble(&dir, second.clone()); + wait_for( + || second.prompts_containing("finished"), + "the pending notice to be delivered after boot", + ) + .await; + let notices = notice_entries(&core, PARENT); + assert_eq!(notices.len(), 1); + assert_eq!( + notice_sides(¬ices[0]), + vec![NoticeSide { + chat_id: SIDE.into(), + title: "Implement login".into(), + outcome: NoticeOutcome::Completed, + detail: None, + }] + ); + assert_eq!( + second.run_count(), + 1, + "the notice turn is the relaunch's only run" + ); + + core.shutdown().await; +} + +/// The kill -9 state for a notifiable side-chat run: rows from a live +/// engine, the doc and journal written by hand (restart-resume prior art). +/// `age_ms` backs the crash, deciding whether boot recovery auto-resumes +/// the run; the side chat's last user message carries the parent origin. +fn manufacture_lost_side_chat_run(dir: &std::path::Path, device_id: &str, age_ms: i64) { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + let store = DocsStore::open(dir.join("orgs/dev-org/dev-user")).unwrap(); + let doc = SessionDoc::init(SIDE).unwrap(); + doc.push_message(&SessionMessageEntry { + origin_chat_id: Some(PARENT.into()), + id: "m-run-1".into(), + role: MessageRole::User, + parts: vec![MessagePart::Text { + id: "t0".into(), + text: "please implement login".into(), + }], + created_at: now - age_ms, + device_id: device_id.into(), + status: Some(MessageStatus::Complete), + continuation_of: None, + duration_ms: None, + provider_turn: None, + }) + .unwrap(); + store + .save_snapshot(SIDE, &doc.export_snapshot().unwrap()) + .unwrap(); + let journal = RunJournal::open(dir.join("orgs/dev-org/dev-user/journals")).unwrap(); + journal + .append( + SIDE, + &AgentEvent::SessionStarted { + harness: HarnessId::Mock, + model: "mock-1".into(), + tools: vec![], + cwd: "/tmp".into(), + session_id: "hs-side".into(), + assistant_message_id: "a-1".into(), + }, + ) + .unwrap(); +} + +/// A child run lost to a restart with NO auto-resume (the crashed turn is +/// older than the revival window) yields an INTERRUPTED outcome at boot: +/// the parent is told the work never finished, and it may restart it. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_run_lost_to_a_restart_without_auto_resume_notifies_interrupted() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("data"); + std::fs::create_dir_all(&dir).unwrap(); + + // Engine 1 creates the rows only. + { + let scripted = Scripted::new(vec![]); + let core = assemble(&dir, scripted); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + let device_id = core.device_id.clone(); + core.shutdown().await; + // The run died 13h ago — outside the 12h auto-resume window. + manufacture_lost_side_chat_run(&dir, &device_id, 13 * 60 * 60 * 1000); + } + + // Engine 2: recovery records the interrupted outcome, and the boot + // delivery wakes the parent with it. + let scripted = Scripted::new(vec![Turn::End { + status: DoneStatus::Completed, + error: None, + }]); + let core = assemble(&dir, scripted.clone()); + wait_for( + || scripted.prompts_containing("was interrupted"), + "the interrupted notice to be delivered after boot", + ) + .await; + let notices = notice_entries(&core, PARENT); + assert_eq!(notices.len(), 1); + assert_eq!( + notice_sides(¬ices[0]), + vec![NoticeSide { + chat_id: SIDE.into(), + title: "Implement login".into(), + outcome: NoticeOutcome::Interrupted, + detail: Some("engine restart".into()), + }] + ); + let text = notice_text(¬ices[0]); + assert!(text.contains("was interrupted: engine restart")); + assert!(text.contains("read_chat")); + // The lost run was not revived: the notice turn is the only run. + assert_eq!(scripted.run_count(), 1); + + core.shutdown().await; +} + +/// A child run lost to a restart WITH auto-resume (a fresh crash) owes no +/// interrupted outcome: the revival re-dispatch keeps the starting +/// message's parent origin, so the notice follows the RESUMED turn's end. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_auto_resumed_run_notifies_when_the_resumed_turn_ends() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("data"); + std::fs::create_dir_all(&dir).unwrap(); + + // Engine 1 creates the rows only. + { + let scripted = Scripted::new(vec![]); + let core = assemble(&dir, scripted); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + let device_id = core.device_id.clone(); + core.shutdown().await; + // The run died a minute ago — inside the revival window. + manufacture_lost_side_chat_run(&dir, &device_id, 60_000); + } + + // Engine 2: the crashed run is picked back up (staggered start) under + // the same user message, and the notice follows its completion. + let scripted = Scripted::new(vec![ + // The revived side-chat turn. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + // The parent's notice turn. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let core = assemble(&dir, scripted.clone()); + wait_for( + || scripted.prompts_containing("please implement login"), + "the crashed turn to be auto-resumed", + ) + .await; + wait_for( + || scripted.prompts_containing("finished"), + "the completion notice to reach the parent", + ) + .await; + let notices = notice_entries(&core, PARENT); + assert_eq!(notices.len(), 1); + assert_eq!( + notice_sides(¬ices[0]), + vec![NoticeSide { + chat_id: SIDE.into(), + title: "Implement login".into(), + // The notice reports the RESUMED turn's completion, not an + // interruption. + outcome: NoticeOutcome::Completed, + detail: None, + }] + ); + let text = notice_text(¬ices[0]); + assert!(!text.contains("interrupted")); + // The revival ran the crashed prompt; the notice is the second turn. + assert_eq!(scripted.run_count(), 2); + assert_eq!( + scripted.prompts.lock().unwrap()[0], + "please implement login", + "the revival re-dispatched the crashed turn's own message" + ); + + core.shutdown().await; +} + +/// An archived parent receives nothing: new outcomes are never recorded +/// for it (ticket 13). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_archived_parent_receives_no_notices() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![Turn::End { + status: DoneStatus::Completed, + error: None, + }]); + let core = assemble(dir.path(), scripted.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + core.workspace.set_chat_archived(PARENT, true).unwrap(); + + run_to_idle(&core, &client, &scripted, SIDE, "please implement login", "m-run-1", Some(PARENT)).await; + // Let a wrongly recorded outcome settle before asserting absence. + tokio::time::sleep(Duration::from_millis(300)).await; + + assert!( + notice_entries(&core, PARENT).is_empty(), + "an archived parent must receive no notices" + ); + assert_eq!(scripted.run_count(), 1, "no parent turn was started"); + + core.shutdown().await; +} + +/// A deleted parent is owed nothing (ticket 13): the delete path drops the +/// parent's pending outcomes — the durable ledger forgets them, and a +/// delivery attempt for a chat whose row is gone stays quiet instead of +/// waking a chat that no longer exists. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_deleted_parent_is_owed_nothing() { + let dir = tempfile::tempdir().unwrap(); + // The child's turn, the parent's working turn (Hang), then nothing for + // the parent — the parked notice's turn never happens. + let scripted = Scripted::new(vec![Turn::Hang, Turn::End { + status: DoneStatus::Completed, + error: None, + }]); + let core = assemble(dir.path(), scripted.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + // The parent works; the child's outcome is recorded and stays pending. + queue_run(&client, PARENT, "orchestrate the release", "m-parent-1", None).await; + wait_for( + || scripted.prompts_containing("orchestrate the release"), + "the parent's turn to start", + ) + .await; + run_to_idle(&core, &client, &scripted, SIDE, "please implement login", "m-run-1", Some(PARENT)).await; + wait_for( + || ledger_rows(dir.path()) == 1, + "the child's outcome to be recorded", + ) + .await; + + // The parent is deleted through the RPC (the delete path purges the + // chat, dropping what it is owed). + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ "op": "deleteChat", "chatId": PARENT }), + ) + .await + .expect("deleteChat"); + wait_for( + || ledger_rows(dir.path()) == 0, + "the deleted parent's pending outcomes to go with the chat", + ) + .await; + + // Nothing wakes the gone chat: no notice turn was started for it. + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + scripted.run_count(), + 2, + "a deleted parent receives nothing: no notice turn started" + ); + core.shutdown().await; +} + +/// An archived side chat sends nothing: archiving the child mid-run +/// suppresses the outcome its stopped turn would otherwise owe (an +/// interrupt is not a message, so it never revives the archived chat). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_archived_side_chat_sends_no_notices() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![Turn::Hang]); + let core = assemble(dir.path(), scripted.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + queue_run(&client, SIDE, "please implement login", "m-run-1", Some(PARENT)).await; + wait_for( + || { + core.sessions + .session_status(SIDE) + .is_some_and(|s| s.status == SessionStatus::Working) + }, + "the side chat's turn to start", + ) + .await; + // Archive the child while its turn runs; a Stop ends the turn. + core.workspace.set_chat_archived(SIDE, true).unwrap(); + core.doc_host + .queue_command(SIDE, SessionCommandPayload::Interrupt {}) + .unwrap(); + wait_idle(&core, SIDE).await; + tokio::time::sleep(Duration::from_millis(300)).await; + + assert!( + notice_entries(&core, PARENT).is_empty(), + "an archived side chat must send no notices" + ); + assert_eq!(scripted.run_count(), 1, "no parent turn was started"); + + core.shutdown().await; +} + +/// Outcomes already owed to a parent when it is ARCHIVED are dropped, not +/// held: after a restart the boot delivery finds the parent archived and +/// the ledger forgets its rows — an archived parent receives nothing even +/// from the pending set (ticket 13's delivery-path rule, beside the +/// record-time one). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn notices_owed_to_a_parent_archived_before_delivery_are_dropped_at_boot() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("data"); + std::fs::create_dir_all(&dir).unwrap(); + + // Engine 1: the child completes while the parent works, so the outcome + // is recorded — the parent is owed it — and then the parent is + // archived before it could ever receive the notice. + let first = Scripted::new(vec![ + Turn::Hang, + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let core = assemble(&dir, first.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + queue_run(&client, PARENT, "orchestrate the release", "m-parent-1", None).await; + wait_for( + || first.prompts_containing("orchestrate the release"), + "the parent's turn to start", + ) + .await; + run_to_idle(&core, &client, &first, SIDE, "please implement login", "m-run-1", Some(PARENT)).await; + wait_for( + || ledger_rows(&dir) == 1, + "the child's outcome to be recorded durably", + ) + .await; + core.workspace.set_chat_archived(PARENT, true).unwrap(); + core.shutdown().await; + drop(core); + + // Engine 2: the boot delivery finds the archived parent and drops the + // owed outcome — no notice turn ever starts, and the ledger is empty. + let second = Scripted::new(vec![Turn::End { + status: DoneStatus::Completed, + error: None, + }]); + let core = assemble(&dir, second.clone()); + wait_for( + || ledger_rows(&dir) == 0, + "the archived parent's owed outcomes to be dropped", + ) + .await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert!( + notice_entries(&core, PARENT).is_empty(), + "an archived parent receives nothing, even outcomes it was already owed" + ); + assert_eq!( + second.run_count(), + 0, + "no notice turn was started for the archived parent" + ); + + core.shutdown().await; +} + +// ---- ticket 14: acknowledgement, stop, loop cap --------------------------- + +/// Report an observation the way the Roboco MCP server does (ticket 14): +/// the parent — the server's calling chat — observed `side` after its last +/// outcome happened, through `wait_for_turn`, a send with wait, `read_chat` +/// or `get_chat`. All four arrive as this one RPC. +async fn acknowledge(client: &roboco_rpc::RpcClient, side: &str, parent: &str) { + acknowledge_at(client, side, parent, None).await; +} + +/// The observation report with the moment the observation STARTED (the +/// shape the Roboco MCP server sends for `read_chat`/`get_chat`/waits): +/// the engine's watermark covers only the settles that had happened by +/// then, so an outcome landing between the read and this report — the +/// parent's unseen work — survives it. +async fn acknowledge_at( + client: &roboco_rpc::RpcClient, + side: &str, + parent: &str, + observed_at: Option, +) { + let mut params = serde_json::json!({ "chatId": side, "originChatId": parent }); + if let Some(at) = observed_at { + params["observedAt"] = serde_json::json!(at); + } + client + .call(roboco_rpc::methods::ACK_SIDE_CHAT, params) + .await + .expect("report observation"); +} + +/// The user's Stop on a chat (the composer's and the MCP `interrupt_chat` +/// tool's Interrupt command). +fn stop(core: &EngineCore, chat: &str) { + core.doc_host + .queue_command(chat, SessionCommandPayload::Interrupt {}) + .expect("queue interrupt"); +} + +/// The paused queue rows a chat currently shows (WatchQueue's surface). +fn queue_rows(core: &EngineCore, chat: &str) -> Vec<(String, String)> { + queue_rows_with_sides(core, chat) + .into_iter() + .map(|(id, text, _)| (id, text)) + .collect() +} + +/// The queue rows with the completion-notice sides a parked notice row +/// carries (ticket 14): the row is the notice's durable carrier while it +/// waits, and its release writes the entry those sides name. +fn queue_rows_with_sides( + core: &EngineCore, + chat: &str, +) -> Vec<(String, String, Option>)> { + core.doc_host + .open(chat) + .expect("chat doc") + .doc() + .read_queue() + .expect("queue") + .into_iter() + .map(|row| (row.id, row.text, row.completion_notice)) + .collect() +} + +/// A parent that waited with `wait_for_turn` — and got the child's +/// completed result — receives no notice for it: the outcome it observed +/// after it happened is dropped, so the parent's next turn end delivers +/// nothing (the same report covers `read_chat` and `get_chat`: all three +/// arrive as the observation RPC). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_observed_outcome_owes_no_notice() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![ + // The parent's working turn: hangs until the user steers it. + Turn::Hang, + // The side chat's turn. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let core = assemble(dir.path(), scripted.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + // The parent works; the child's parent-started turn completes while it + // does, so the outcome is recorded and stays pending. + queue_run(&client, PARENT, "orchestrate the release", "m-parent-1", None).await; + wait_for( + || scripted.prompts_containing("orchestrate the release"), + "the parent's turn to start", + ) + .await; + run_to_idle(&core, &client, &scripted, SIDE, "please implement login", "m-run-1", Some(PARENT)).await; + wait_for( + || ledger_rows(dir.path()) == 1, + "the child's outcome to be recorded durably", + ) + .await; + + // The parent waits (or reads) — the observation is reported for it. + acknowledge(&client, SIDE, PARENT).await; + + // The parent's turn ends: the pending outcome is gone, so no notice + // turn starts. + queue_user_steer(&client, PARENT, "please wrap up", "m-steer-1").await; + wait_idle(&core, PARENT).await; + tokio::time::sleep(Duration::from_millis(300)).await; + + assert!( + notice_entries(&core, PARENT).is_empty(), + "an outcome the parent observed owes no notice" + ); + assert_eq!( + ledger_rows(dir.path()), + 0, + "the observed outcome is dropped from the ledger" + ); + assert_eq!( + scripted.run_count(), + 2, + "only the parent's and the child's turns ran — no notice turn" + ); + + core.shutdown().await; +} + +/// The observation covers exactly the settles it saw: a LATER turn of the +/// same side chat still owes its notice. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_outcome_after_the_observation_still_notifies() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![ + // The parent's working turn. + Turn::Hang, + // The child's first turn. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + // The child's second turn. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + // The parent's notice turn, for the second outcome only. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let core = assemble(dir.path(), scripted.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + queue_run(&client, PARENT, "orchestrate the release", "m-parent-1", None).await; + wait_for( + || scripted.prompts_containing("orchestrate the release"), + "the parent's turn to start", + ) + .await; + run_to_idle(&core, &client, &scripted, SIDE, "first task", "m-run-1", Some(PARENT)).await; + wait_for( + || ledger_rows(dir.path()) == 1, + "the first outcome to be recorded", + ) + .await; + + // The parent observed the chat after the first turn ended. + acknowledge(&client, SIDE, PARENT).await; + assert_eq!(ledger_rows(dir.path()), 0); + + // A later turn of the same child settles while the parent still works: + // its outcome is past the observation's watermark, so it is owed. + run_to_idle(&core, &client, &scripted, SIDE, "second task", "m-run-2", Some(PARENT)).await; + wait_for( + || ledger_rows(dir.path()) == 1, + "the second outcome to be recorded", + ) + .await; + + // The parent's turn ends: the second outcome reaches it. + queue_user_steer(&client, PARENT, "please wrap up", "m-steer-1").await; + wait_for( + || scripted.prompts_containing("finished"), + "the second outcome's notice to start a parent turn", + ) + .await; + let notices = notice_entries(&core, PARENT); + assert_eq!(notices.len(), 1, "one notice, for the second outcome only"); + assert_eq!( + notice_sides(¬ices[0]), + vec![NoticeSide { + chat_id: SIDE.into(), + title: "Implement login".into(), + outcome: NoticeOutcome::Completed, + detail: None, + }] + ); + assert_eq!(scripted.run_count(), 4); + + core.shutdown().await; +} + +/// The report carries WHEN the observation happened (ticket 14): a settle +/// that lands between the read and the report is the parent's UNSEEN +/// outcome — the ack must not drop it, or a busy child's report races away +/// its own completion notice. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_late_observation_report_never_drops_an_unseen_outcome() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![ + // The parent's working turn. + Turn::Hang, + // The child's first turn (seen by the observation). + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + // The child's second turn (settles between the read and the report). + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + // The parent's notice turn, for the second outcome only. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let core = assemble(dir.path(), scripted.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + queue_run(&client, PARENT, "orchestrate the release", "m-parent-1", None).await; + wait_for( + || scripted.prompts_containing("orchestrate the release"), + "the parent's turn to start", + ) + .await; + run_to_idle(&core, &client, &scripted, SIDE, "first task", "m-run-1", Some(PARENT)).await; + wait_for( + || ledger_rows(dir.path()) == 1, + "the first outcome to be recorded", + ) + .await; + + // The parent's read of the chat happens HERE. Millisecond separation is + // forced on both sides so the settle marks order unambiguously. + tokio::time::sleep(Duration::from_millis(5)).await; + let observed_at = chrono::Utc::now().timestamp_millis(); + tokio::time::sleep(Duration::from_millis(5)).await; + + // The child's second turn settles after the read: unseen by it. + run_to_idle(&core, &client, &scripted, SIDE, "second task", "m-run-2", Some(PARENT)).await; + wait_for( + || ledger_rows(dir.path()) == 2, + "the second outcome to be recorded", + ) + .await; + + // The report arrives now — long after the read it speaks for. Only the + // settle the read saw is covered; the second outcome is owed. + acknowledge_at(&client, SIDE, PARENT, Some(observed_at)).await; + assert_eq!( + ledger_rows(dir.path()), + 1, + "the unseen outcome survives the late report; the seen one drops" + ); + + // The parent's turn ends: the notice covers the second outcome. + queue_user_steer(&client, PARENT, "please wrap up", "m-steer-1").await; + wait_for( + || scripted.prompts_containing("finished"), + "the unseen outcome's notice to start a parent turn", + ) + .await; + let notices = notice_entries(&core, PARENT); + assert_eq!(notices.len(), 1); + assert_eq!( + notice_sides(¬ices[0]), + vec![NoticeSide { + chat_id: SIDE.into(), + title: "Implement login".into(), + outcome: NoticeOutcome::Completed, + detail: None, + }], + "the notice reports the unseen outcome only" + ); + assert_eq!(scripted.run_count(), 4); + + core.shutdown().await; +} + +/// Stopping the parent drops the notices owed to it and suppresses +/// outcomes from side chats whose notifiable turns were running at that +/// moment: the children keep running (Stop does not cascade), and their +/// later completions do not wake the stopped parent. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stopping_the_parent_drops_pending_notices_and_suppresses_running_children() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![ + // The parent's working turn. + Turn::Hang, + // Child A: completes while the parent works (a pending outcome). + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + // Child B: still running when the parent is stopped, steered to its + // completion afterwards. + Turn::Hang, + ]); + let core = assemble(dir.path(), scripted.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent(&core, &client).await; + create_side(&core, &client, "chat-side-a", "Implement login", None).await; + create_side(&core, &client, "chat-side-b", "Write the docs", None).await; + + queue_run(&client, PARENT, "orchestrate the release", "m-parent-1", None).await; + wait_for( + || scripted.prompts_containing("orchestrate the release"), + "the parent's turn to start", + ) + .await; + run_to_idle(&core, &client, &scripted, "chat-side-a", "please implement login", "m-run-a", Some(PARENT)).await; + wait_for( + || ledger_rows(dir.path()) == 1, + "child A's outcome to be recorded durably", + ) + .await; + queue_run(&client, "chat-side-b", "please write the docs", "m-run-b", Some(PARENT)).await; + wait_for( + || { + core.sessions + .session_status("chat-side-b") + .is_some_and(|s| s.status == SessionStatus::Working) + }, + "child B's turn to be running", + ) + .await; + + // The user stops the parent. + stop(&core, PARENT); + wait_idle(&core, PARENT).await; + wait_for( + || ledger_rows(dir.path()) == 0, + "the parent's pending notices to be dropped", + ) + .await; + // Child B keeps running — Stop did not cascade. + assert_eq!( + core.sessions + .session_status("chat-side-b") + .map(|s| s.status), + Some(SessionStatus::Working), + "the side chat keeps running across the parent's Stop" + ); + + // Child B's turn completes later: its outcome was suppressed at the + // Stop, so the stopped parent is never woken. + queue_user_steer(&client, "chat-side-b", "please wrap up", "m-steer-b").await; + wait_idle(&core, "chat-side-b").await; + tokio::time::sleep(Duration::from_millis(300)).await; + + assert!( + notice_entries(&core, PARENT).is_empty(), + "a stopped parent receives no notices, and a running child's later \ + completion does not wake it" + ); + assert_eq!(ledger_rows(dir.path()), 0); + assert_eq!( + scripted.run_count(), + 3, + "the parent's and the two children's turns ran — no notice turn" + ); + + core.shutdown().await; +} + +/// A later message from the parent to that child re-arms notices for its +/// next turn (ticket 14's re-arming rule): the steer joins the running +/// child, arms the turn again, and its completion wakes the parent. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_later_parent_message_re_arms_the_child() { + let dir = tempfile::tempdir().unwrap(); + let scripted = Scripted::new(vec![ + // The parent's working turn. + Turn::Hang, + // The child's turn: hangs across the parent's Stop, then steered by + // the re-engaging parent and completed. + Turn::Hang, + // The parent's notice turn. + Turn::End { + status: DoneStatus::Completed, + error: None, + }, + ]); + let core = assemble(dir.path(), scripted.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + queue_run(&client, PARENT, "orchestrate the release", "m-parent-1", None).await; + wait_for( + || scripted.prompts_containing("orchestrate the release"), + "the parent's turn to start", + ) + .await; + queue_run(&client, SIDE, "please implement login", "m-run-1", Some(PARENT)).await; + wait_for( + || { + core.sessions + .session_status(SIDE) + .is_some_and(|s| s.status == SessionStatus::Working) + }, + "the child's turn to be running", + ) + .await; + + // The user stops the parent; the child's turn keeps running, its + // outcome suppressed. + stop(&core, PARENT); + wait_idle(&core, PARENT).await; + assert_eq!( + core.sessions.session_status(SIDE).map(|s| s.status), + Some(SessionStatus::Working) + ); + + // The parent resumes orchestrating: its message steers the running + // child, re-arming the turn's notifiability. + queue_run(&client, SIDE, "round two, please", "m-run-2", Some(PARENT)).await; + wait_for( + || scripted.prompts_containing("finished"), + "the re-armed turn's completion notice to wake the parent", + ) + .await; + + let notices = notice_entries(&core, PARENT); + assert_eq!(notices.len(), 1, "the re-armed turn notifies again"); + assert_eq!( + notice_sides(¬ices[0]), + vec![NoticeSide { + chat_id: SIDE.into(), + title: "Implement login".into(), + outcome: NoticeOutcome::Completed, + detail: None, + }] + ); + // The parent's turn, the child's turn (steered in place) and the notice + // turn: the child never restarted. + assert_eq!(scripted.run_count(), 3); + assert!( + scripted + .steered + .lock() + .unwrap() + .iter() + .any(|p| p.contains("round two, please")), + "the parent's message steered the running child" + ); + + core.shutdown().await; +} + +/// The 26th consecutive notice-started turn does not start: the notice +/// waits visibly in the parent's queue, paused, until the user acts. A user +/// message resets the count and thaws the queue, and the parked row then +/// runs as the parent's next turn (counting as notice-started itself). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_loop_cap_parks_the_26th_notice_until_the_user_acts() { + let dir = tempfile::tempdir().unwrap(); + // 27 child turns (25 rounds + the capped 26th + the post-reset 27th) + // + 26 notice turns + an agent's message to the parent + the user's + // message + the parked row's own turn — every run pops one scripted + // turn. + let turns: Vec = (0..27 + 26 + 1 + 1 + 1) + .map(|_| Turn::End { + status: DoneStatus::Completed, + error: None, + }) + .collect(); + let scripted = Scripted::new(turns); + let core = assemble(dir.path(), scripted.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + // 25 rounds: the parent orchestrates, the child completes, the notice + // starts the parent's next turn. The parent never sees a user message. + for round in 0..25 { + run_to_idle( + &core, + &client, + &scripted, + SIDE, + &format!("round {round}"), + &format!("m-run-{round}"), + Some(PARENT), + ) + .await; + wait_for( + || notice_entries(&core, PARENT).len() == round + 1, + &format!("round {round}'s notice to be written"), + ) + .await; + // The notice turn itself dispatched (child + notice per round) + // before the idle wait reads the parent's row. + wait_for( + || scripted.run_count() == 2 * (round + 1), + &format!("round {round}'s notice turn to start"), + ) + .await; + wait_idle(&core, PARENT).await; + } + assert_eq!(notice_entries(&core, PARENT).len(), 25); + assert_eq!(scripted.run_count(), 50); + + // Round 26: the child completes, but the 26th notice-started turn does + // not start — the notice parks as a paused queue row. + run_to_idle(&core, &client, &scripted, SIDE, "round 26", "m-run-26", Some(PARENT)).await; + wait_for( + || queue_rows(&core, PARENT).len() == 1, + "the capped notice to park in the parent's queue", + ) + .await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + notice_entries(&core, PARENT).len(), + 25, + "no 26th notice entry — the notice did not start a turn" + ); + assert_eq!( + scripted.run_count(), + 51, + "the 26th notice turn never ran (25 child + 25 notice + the 26th child)" + ); + let rows = queue_rows_with_sides(&core, PARENT); + assert_eq!(rows.len(), 1, "exactly one parked notice row"); + assert!( + rows[0].0.starts_with("notice:"), + "the parked row carries the engine's notice prefix: {}", + rows[0].0 + ); + assert!( + rows[0].1.contains("Implement login"), + "the parked row shows the notice itself: {}", + rows[0].1 + ); + assert!( + rows[0].2.is_some(), + "the parked row carries the notice's sides, so its release writes the + same System completion-notice entry a direct delivery writes" + ); + + // An agent's message to the parent (a chat-origin Run, the MCP + // `send_message` surface) is NOT the user acting: it must not thaw the + // cap's pause, or a busy child lifts the brake that waits for the user + // (ticket 14's stopped-parent cap rule). The message still runs as its + // own turn — the queue, and the parked notice with it, stays held. + queue_run(&client, PARENT, "an agent nudges the parent", "m-agent", Some(SIDE)).await; + wait_for( + || scripted.prompts_containing("an agent nudges the parent"), + "the agent's turn to start", + ) + .await; + wait_idle(&core, PARENT).await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + queue_rows(&core, PARENT).len(), + 1, + "an agent-origin message does not release the cap's parked notice" + ); + assert_eq!( + notice_entries(&core, PARENT).len(), + 25, + "the parked notice did not run on an agent's message" + ); + + // The user acts: their message resets the count, thaws the queue, and + // runs as the parent's turn; the parked notice row follows it out. + queue_run(&client, PARENT, "user speaks now", "m-user", None).await; + wait_for( + || scripted.prompts_containing("user speaks now"), + "the user's turn to start", + ) + .await; + wait_idle(&core, PARENT).await; + wait_for( + || scripted.run_count() == 54, + "the parked notice row to run as the parent's next turn", + ) + .await; + // The row's turn is the 54th run, carrying the notice text as its + // prompt — and it lands as the SAME System completion-notice entry a + // direct delivery writes (its own sides ride the row), never a user + // message: notices are engine artifacts, not the user's words. + wait_for( + || notice_entries(&core, PARENT).len() == 26, + "the released row's turn to write the 26th system notice entry", + ) + .await; + let released = notice_entries(&core, PARENT) + .pop() + .expect("the released notice entry"); + assert!( + released.id.starts_with("notice:"), + "the released entry keeps the parked row's id: {}", + released.id + ); + let prompts = scripted.prompts.lock().unwrap(); + assert!( + prompts[53].contains("Implement login") && prompts[53].contains("read_chat"), + "the released row ran the notice's own text: {}", + prompts[53] + ); + drop(prompts); + assert!( + queue_rows(&core, PARENT).is_empty(), + "the parked row was consumed by its turn" + ); + // A new outcome after the user's message can start a notice turn again: + // the count reset with the user's message (plus the row's own turn). + run_to_idle(&core, &client, &scripted, SIDE, "round 27", "m-run-27", Some(PARENT)).await; + wait_for( + || notice_entries(&core, PARENT).len() == 27, + "the count reset lets the next notice start a turn again", + ) + .await; + + core.shutdown().await; +} + +/// Outcomes that arrive while the cap holds MERGE into the parked row +/// (ticket 13's merging rule, extended to the cap's waiting form): one +/// parked notice, not a row per capped delivery, and its release is ONE +/// system notice covering every side chat that reached an outcome. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn capped_outcomes_merge_into_the_one_parked_notice_row() { + let dir = tempfile::tempdir().unwrap(); + // 26 side turns (25 rounds + the capped 26th) + 25 notice turns + the + // second side chat's turn + the user's message + the released row's + // turn — every run pops one scripted turn. + let turns: Vec = (0..26 + 25 + 1 + 1 + 1) + .map(|_| Turn::End { + status: DoneStatus::Completed, + error: None, + }) + .collect(); + let scripted = Scripted::new(turns); + let core = assemble(dir.path(), scripted.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + create_side(&core, &client, SIDE2, "Flaky test", None).await; + + // Fill the cap, then park the 26th notice. + for round in 0..25 { + run_to_idle( + &core, + &client, + &scripted, + SIDE, + &format!("round {round}"), + &format!("m-run-{round}"), + Some(PARENT), + ) + .await; + wait_for( + || notice_entries(&core, PARENT).len() == round + 1, + &format!("round {round}'s notice to be written"), + ) + .await; + // The notice turn itself dispatched (child + notice per round) + // before the idle wait reads the parent's row. + wait_for( + || scripted.run_count() == 2 * (round + 1), + &format!("round {round}'s notice turn to start"), + ) + .await; + wait_idle(&core, PARENT).await; + } + run_to_idle(&core, &client, &scripted, SIDE, "round 26", "m-run-26", Some(PARENT)).await; + wait_for( + || queue_rows(&core, PARENT).len() == 1, + "the capped notice to park in the parent's queue", + ) + .await; + + // A second capped outcome parks INTO the same row: one row, its text + // and sides now covering both side chats. + run_to_idle(&core, &client, &scripted, SIDE2, "the second child fails", "m-run-2", Some(PARENT)).await; + wait_for( + || queue_rows_with_sides(&core, PARENT) + .into_iter() + .any(|(_, text, sides)| text.contains("Flaky test") && sides.is_some_and(|s| s.len() == 2)), + "the second capped outcome to merge into the parked row", + ) + .await; + let rows = queue_rows_with_sides(&core, PARENT); + assert_eq!(rows.len(), 1, "still exactly one parked notice row"); + assert!( + rows[0].1.contains("2 side chats reached outcomes"), + "the merged row's text is one notice: {}", + rows[0].1 + ); + assert_eq!( + scripted.run_count(), + 52, + "no notice turn started while the cap held (25 child + 25 notice + 2 capped children)" + ); + + // The user acts: the released row is ONE system notice naming both. + queue_run(&client, PARENT, "user speaks now", "m-user", None).await; + wait_for( + || scripted.prompts_containing("user speaks now"), + "the user's turn to start", + ) + .await; + wait_idle(&core, PARENT).await; + wait_for( + || notice_entries(&core, PARENT).len() == 26, + "the released row to write its merged system notice entry", + ) + .await; + let released = notice_entries(&core, PARENT).pop().expect("the released entry"); + let text = notice_text(&released); + assert!(text.contains("Implement login"), "{text}"); + assert!(text.contains("Flaky test"), "{text}"); + assert!( + text.starts_with("2 side chats reached outcomes:"), + "the released notice is one merged notice: {text}" + ); + // The entry lands before the released row's dispatch: wait for its turn. + wait_for( + || scripted.run_count() == 54, + "the released row's notice turn to start", + ) + .await; + wait_idle(&core, PARENT).await; + assert!(queue_rows(&core, PARENT).is_empty()); + + core.shutdown().await; +} + +/// A Stop also drops the loop cap's parked notice row (ticket 14: it is a +/// pending notice, the visible form the cap left it waiting in): after the +/// Stop the user's next message runs alone, and nothing restarts the +/// parent on the parked notice's behalf. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stopping_the_parent_drops_the_parked_notice_row() { + let dir = tempfile::tempdir().unwrap(); + // 26 child turns + 25 notice turns + the user's message — the parked + // row's turn never happens. + let turns: Vec = (0..26 + 25 + 1) + .map(|_| Turn::End { + status: DoneStatus::Completed, + error: None, + }) + .collect(); + let scripted = Scripted::new(turns); + let core = assemble(dir.path(), scripted.clone()); + let client = roboco_rpc::memory_client(core.rpc_service()); + create_parent_and_side(&core, &client, None).await; + + // Fill the cap: 25 notice-started turns, then the 26th child completion + // parks its notice in the parent's queue. + for round in 0..25 { + run_to_idle( + &core, + &client, + &scripted, + SIDE, + &format!("round {round}"), + &format!("m-run-{round}"), + Some(PARENT), + ) + .await; + wait_for( + || notice_entries(&core, PARENT).len() == round + 1, + &format!("round {round}'s notice to be written"), + ) + .await; + // The notice turn itself dispatched (child + notice per round) + // before the idle wait reads the parent's row. + wait_for( + || scripted.run_count() == 2 * (round + 1), + &format!("round {round}'s notice turn to start"), + ) + .await; + wait_idle(&core, PARENT).await; + } + run_to_idle(&core, &client, &scripted, SIDE, "round 26", "m-run-26", Some(PARENT)).await; + wait_for( + || queue_rows(&core, PARENT).len() == 1, + "the capped notice to park in the parent's queue", + ) + .await; + assert_eq!(scripted.run_count(), 51); + + // The user stops the (idle) parent: the parked row goes with the rest + // of its pending notices. + stop(&core, PARENT); + wait_for( + || queue_rows(&core, PARENT).is_empty(), + "the parked notice row to be dropped by the Stop", + ) + .await; + + // The user's next message runs alone: the parked notice's turn never + // starts, and no later outcome wakes the parent unprompted. + queue_run(&client, PARENT, "user speaks now", "m-user", None).await; + wait_for( + || scripted.prompts_containing("user speaks now"), + "the user's turn to start", + ) + .await; + wait_idle(&core, PARENT).await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + scripted.run_count(), + 52, + "the user's turn is the last run — the parked row never ran" + ); + assert!(queue_rows(&core, PARENT).is_empty()); + assert_eq!( + notice_entries(&core, PARENT).len(), + 25, + "no new notice entry appeared" + ); + + core.shutdown().await; +} diff --git a/crates/engine/tests/continue_on_harness.rs b/crates/engine/tests/continue_on_harness.rs new file mode 100644 index 0000000000..2465393974 --- /dev/null +++ b/crates/engine/tests/continue_on_harness.rs @@ -0,0 +1,677 @@ +//! A Limited chat continues on another harness (t3-orchestration-v2 ticket +//! 25): the composition of ticket 06's Limited record and ticket 24's +//! one-way harness switch. +//! +//! "Continue on another harness" performs the switch first — the seam, the +//! dropped session, the saved config — and then sends the continue prompt as +//! a normal run (ticket 06's Resume path, the request carrying the new +//! harness the way the composer's optimistic row stamp routes it), so: +//! - the continuation dispatches on the NEW harness with the budgeted +//! transcript of everything above the seam, never resuming the limited +//! harness's session; +//! - the limit record survives the switch alone (the chat stays Limited +//! until a turn actually starts) and clears when the continuation runs; +//! - messages queued while the chat was Limited wait behind the +//! continuation and then follow on the new harness. +//! +//! The harness seam is scripted: a `Limit` stand-in whose first run ends on +//! a classified usage-limit error exactly the way the Claude/Codex drivers +//! do (`Error { kind }` then an errored `Done`), and a `Capture` stand-in +//! that records every request it ran and holds its turns until the test +//! releases them — one instance per harness id, so the continuation shows +//! up as the requests moving from one capture to the other. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use futures::StreamExt; +use futures::stream::BoxStream; + +use roboco_doc::{MessagePart, MessageRole, SessionCommandPayload}; +use roboco_engine::{AgentAccounts, AgentAccountsConfig, EngineCore, HarnessRegistry}; +use roboco_harness::{Harness, HarnessError, RunControls}; +use roboco_proto::{ + AgentErrorKind, AgentEvent, ChatConfig, DoneStatus, HarnessId, Model, ReasoningLevel, + RunRequest, SandboxLevel, SessionLimit, SessionStatus, SteeringMode, +}; +use roboco_rpc::methods; + +const CHAT: &str = "chat-continue"; +const CONTINUE_PROMPT: &str = roboco_proto::RESUME_CONTINUE_PROMPT; + +/// A scripted harness: the FIRST run ends on a classified usage-limit error +/// (`Error { kind }` then an errored `Done` — the driver shape ticket 05 +/// landed); every run waits for a `finish` signal before its terminal event, +/// so the test owns when the limited turn ends. Later runs never happen in +/// these tests — the continuation always switches to the `Capture` harness. +struct Limit { + finish: tokio::sync::broadcast::Sender<()>, + prompts: Arc>>, +} + +impl Limit { + fn new() -> (Arc, Arc>>) { + let (finish, _) = tokio::sync::broadcast::channel(16); + let prompts = Arc::new(Mutex::new(Vec::new())); + ( + Arc::new(Self { + finish, + prompts: prompts.clone(), + }), + prompts, + ) + } +} + +#[async_trait] +impl Harness for Limit { + fn id(&self) -> HarnessId { + HarnessId::ClaudeCode + } + fn display_name(&self) -> &str { + "Limit" + } + // A turn-boundary agent like the Claude driver it stands in for: a Run + // sent mid-turn on the same runtime is held for the turn's end, not + // dispatched over the live turn. + fn supports_steering(&self) -> bool { + true + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::TurnBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[ReasoningLevel::Medium] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + _controls: RunControls, + ) -> Result>, HarnessError> { + // Subscribe before recording the prompt: a test that sees the prompt + // may send `finish` at once. + let mut finish = self.finish.subscribe(); + self.prompts.lock().unwrap().push(request.prompt.clone()); + Ok(futures::stream::once(async move { + // The turn runs until the test says it ends — then it dies on + // the usage limit exactly the way the driver reports one. + let _ = finish.recv().await; + vec![ + Ok(AgentEvent::Error { + message: "Claude 5-hour limit reached — try again after the limit resets." + .into(), + kind: Some(AgentErrorKind::UsageLimit { + window: "5-hour".into(), + resets_at: Some(1_762_675_400_000), + }), + }), + Ok(AgentEvent::Done { + status: DoneStatus::Errored, + result: None, + error: Some("the turn failed".into()), + session_id: Some("claude-session".into()), + }), + ] + }) + .flat_map(futures::stream::iter) + .boxed()) + } +} + +/// A scripted harness that records every request it ran and holds each +/// turn until the test releases it (the same control ticket 06's harness +/// has), so queue-order assertions stay deterministic. The continuation +/// switches onto it, so the switch shows up as the requests moving from +/// the `Limit` capture to this one. +struct Capture { + session: &'static str, + reply: &'static str, + requests: Arc>>, + finish: tokio::sync::broadcast::Sender<()>, +} + +impl Capture { + fn build( + session: &'static str, + reply: &'static str, + ) -> (Arc, Arc>>) { + let requests = Arc::new(Mutex::new(Vec::new())); + let (finish, _) = tokio::sync::broadcast::channel(16); + ( + Arc::new(Self { + session, + reply, + requests: requests.clone(), + finish, + }), + requests, + ) + } +} + +#[async_trait] +impl Harness for Capture { + fn id(&self) -> HarnessId { + HarnessId::Pi + } + fn display_name(&self) -> &str { + "Capture" + } + fn supports_steering(&self) -> bool { + false + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::TurnBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + _: RunControls, + ) -> Result>, HarnessError> { + let mut finish = self.finish.subscribe(); + self.requests.lock().unwrap().push(request.clone()); + let reply = self.reply.to_string(); + let session = self.session.to_string(); + Ok(futures::stream::once(async move { + // The turn runs until the test says it ends. + let _ = finish.recv().await; + vec![ + Ok(AgentEvent::TextDelta { text: reply }), + Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some(session), + }), + ] + }) + .flat_map(futures::stream::iter) + .boxed()) + } +} + +async fn wait_for(mut predicate: F, what: &str) +where + F: FnMut() -> bool, +{ + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while !predicate() { + assert!( + tokio::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(Duration::from_millis(15)).await; + } +} + +fn claude_config() -> ChatConfig { + ChatConfig { + harness: HarnessId::ClaudeCode, + model: Some("claude-default".into()), + reasoning: None, + model_options: Default::default(), + sandbox: SandboxLevel::WorkspaceWrite, + } +} + +/// Queue a Run command the way the composer's Resume / normal send does (the +/// durable command plane, not the visible message queue). `model` is the +/// pick the dialog landed on for the new harness; `None` leaves the request +/// to the harness's own default. +fn queue_run( + core: &EngineCore, + prompt: &str, + harness: HarnessId, + model: Option<&str>, + message_id: &str, +) { + let request = RunRequest { + prompt: prompt.into(), + harness: Some(harness), + model: model.map(str::to_string), + reasoning: None, + model_options: Default::default(), + // The chat row's cwd: queued rows dispatch from the row, and a + // harness session only resumes from the cwd it was opened in. + cwd: "/tmp".into(), + sandbox: SandboxLevel::WorkspaceWrite, + auto_approve: true, + attachments: Vec::new(), + worktree: None, + mcp: None, + resume: None, + provider_fork: None, + resume_fallback_history: None, + }; + core.doc_host + .queue_command( + CHAT, + SessionCommandPayload::Run { + request, + message_id: message_id.into(), + }, + ) + .expect("queue run command"); +} + +async fn switch( + core: &EngineCore, + harness: HarnessId, + model: Option<&str>, +) -> Result { + let client = roboco_rpc::memory_client(core.rpc_service()); + let mut params = serde_json::json!({ "chatId": CHAT, "harness": harness }); + if let Some(model) = model { + params["model"] = serde_json::Value::String(model.into()); + } + client.call(methods::SWITCH_HARNESS, params).await +} + +fn queue_texts(core: &EngineCore) -> Vec { + core.doc_host + .open(CHAT) + .ok() + .and_then(|h| h.doc().read_queue().ok()) + .unwrap_or_default() + .into_iter() + .map(|item| item.text) + .collect() +} + +fn user_texts(core: &EngineCore) -> Vec { + core.doc_host + .open(CHAT) + .ok() + .and_then(|h| h.doc().read_entries().ok()) + .unwrap_or_default() + .iter() + .filter(|e| e.role == MessageRole::User) + .map(|e| { + e.parts + .iter() + .filter_map(|p| match p { + MessagePart::Text { text, .. } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("") + }) + .collect() +} + +fn limit_row(core: &EngineCore) -> Option { + core.sessions + .session_status(CHAT) + .and_then(|s| s.limit.clone()) +} + +/// The chat row as the workspace doc sees it. +fn chat_row(core: &EngineCore) -> roboco_proto::Chat { + core.workspace.chat(CHAT).expect("chat row").expect("chat exists") +} + +/// The transcript's harness-switch seams naming `harness`: `count` of them, +/// each a complete system row. +fn expect_seams(core: &EngineCore, harness: HarnessId, count: usize) { + let seams: Vec = core + .doc_host + .open(CHAT) + .expect("open chat") + .doc() + .read_entries() + .expect("read entries") + .into_iter() + .filter(|entry| { + entry.role == MessageRole::System + && entry + .parts + .iter() + .any(|part| matches!(part, MessagePart::HarnessSwitch { harness: to, .. } if *to == harness)) + }) + .collect(); + assert_eq!(seams.len(), count, "seams for {harness:?}"); + for seam in &seams { + assert_eq!(seam.status, Some(roboco_doc::MessageStatus::Complete)); + } +} + +/// Assemble a core around the two scripted harnesses, pre-titled so the +/// auto-titler never dispatches a request of its own, and mount an ISOLATED +/// accounts store (empty) so limit records never read the machine's real +/// CLI logins. +fn setup() -> ( + EngineCore, + Arc, + Arc>>, + Arc, + Arc>>, + tempfile::TempDir, +) { + let tmp = tempfile::tempdir().unwrap(); + let (limited, limited_prompts) = Limit::new(); + let (pi, pi_requests) = Capture::build("pi-session", "Picked the work back up."); + let registry = HarnessRegistry::new(); + registry.register(limited.clone()); + registry.register(pi.clone()); + let core = EngineCore::assemble(&tmp.path().join("data"), Arc::new(registry), HarnessId::Mock) + .expect("engine core assembles"); + core.sessions.set_agent_accounts(AgentAccounts::new( + AgentAccountsConfig::isolated(&tmp.path().join("accounts")), + )); + core.workspace + .create_chat( + CHAT, + None, + Some(&core.device_id), + Some(claude_config()), + Some("/tmp".into()), + ) + .expect("create chat"); + // Pre-title so the auto-titler never dispatches a harness request of its own. + core.workspace + .rename_chat(CHAT, "Pre-titled") + .expect("rename chat"); + (core, limited, limited_prompts, pi, pi_requests, tmp) +} + +/// Start the turn that hits the limit and wait for the run to be working. +async fn start_limited_turn(prompts: &Arc>>) { + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "Remember PINEAPPLE"), + "the limited run to start", + ) + .await; +} + +/// Release the limited turn and wait for the chat to settle Limited. +async fn settle_limited(core: &EngineCore) { + wait_for(|| limit_row(core).is_some(), "the chat to go Limited").await; +} + +/// Choose "Continue on another harness": the switch (ticket 24), then the +/// continue prompt (ticket 06's Resume path) — the new agent picks the work +/// up from the budgeted history, the limited harness's session never +/// resumes, the limit record clears when the continuation runs, and the +/// messages queued while the chat was Limited follow behind the +/// continuation on the new harness. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn continuing_on_another_harness_switches_then_sends_the_continue_prompt() { + let (core, limited, limited_prompts, pi, pi_requests, _tmp) = setup(); + queue_run(&core, "Remember PINEAPPLE", HarnessId::ClaudeCode, None, "m-open"); + start_limited_turn(&limited_prompts).await; + // A message the user sent while the doomed run was working. + core.doc_host + .queue_message(CHAT, "queued one", Vec::new()) + .expect("queue message"); + let _ = limited.finish.send(()); + settle_limited(&core).await; + // Limited: the queued row WAITS instead of dispatching into the limit. + assert_eq!(queue_texts(&core), vec!["queued one"]); + assert_eq!( + limited_prompts.lock().unwrap().clone(), + vec!["Remember PINEAPPLE"], + "no queued message dispatches into the limited chat" + ); + + // The switch is allowed while the chat is Limited (it is errored, not + // working) — ticket 24's one-way switch runs first. + switch(&core, HarnessId::Pi, Some("pi-default")) + .await + .expect("switch while limited"); + expect_seams(&core, HarnessId::Pi, 1); + let row = chat_row(&core); + assert_eq!(row.config.as_ref().map(|c| c.harness), Some(HarnessId::Pi)); + assert_eq!(row.config.as_ref().and_then(|c| c.model.as_deref()), Some("pi-default")); + assert_eq!( + row.harness_session_id.as_deref(), + Some(""), + "the limited harness's session is tombstoned, never resumed" + ); + // Still holding the queue — the switch thaws nothing by itself. + assert_eq!(queue_texts(&core), vec!["queued one"]); + + // The continue prompt — the composer's normal-send path with the fixed + // prompt, carrying the new harness (the optimistic row stamp routes it) + // and the model the dialog picked. + queue_run(&core, CONTINUE_PROMPT, HarnessId::Pi, Some("pi-default"), "m-resume"); + wait_for( + || pi_requests + .lock() + .unwrap() + .iter() + .any(|r| r.prompt.ends_with(CONTINUE_PROMPT)), + "the continue prompt to run on the new harness", + ) + .await; + let request = pi_requests.lock().unwrap().last().cloned().unwrap(); + assert_eq!(request.harness, Some(HarnessId::Pi)); + assert_eq!(request.model.as_deref(), Some("pi-default")); + assert_eq!( + request.resume, None, + "the limited harness's session must never resume" + ); + assert!(request.prompt.contains("Remember PINEAPPLE"), "{}", request.prompt); + assert!(request.prompt.ends_with(CONTINUE_PROMPT), "{}", request.prompt); + assert!(request.prompt.contains(""), "{}", request.prompt); + // The continuation owns the turn (it holds until the test releases it); + // the queued row still waits behind it. + assert_eq!(queue_texts(&core), vec!["queued one"]); + // The continuation ran: the limit record is gone. + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.limit.is_none()) + }, + "the continuation to clear the limit record", + ) + .await; + + // The continuation ends → the queue drains behind it, onto the new + // harness, continuing the session the continuation opened. + let _ = pi.finish.send(()); + wait_for( + || chat_row(&core).harness_session_id.as_deref() == Some("pi-session"), + "the new harness's session to be recorded", + ) + .await; + wait_for( + || pi_requests.lock().unwrap().iter().any(|r| r.prompt.contains("queued one")), + "the queued message to follow the continuation on the new harness", + ) + .await; + let follow_up = pi_requests.lock().unwrap().last().cloned().unwrap(); + assert_eq!(follow_up.harness, Some(HarnessId::Pi)); + assert_eq!(follow_up.resume.as_deref(), Some("pi-session")); + assert!(!follow_up.prompt.contains("PINEAPPLE"), "{}", follow_up.prompt); + + let _ = pi.finish.send(()); + wait_for( + || { + queue_texts(&core).is_empty() + && core + .sessions + .session_status(CHAT) + .is_none_or(|s| s.status != SessionStatus::Working) + }, + "the last turn to settle and the queue to drain", + ) + .await; + // The transcript carries the continue prompt as the run's user entry. + assert!(user_texts(&core).contains(&CONTINUE_PROMPT.to_string())); + assert!(queue_texts(&core).is_empty(), "the queue drained"); + assert_eq!(limit_row(&core), None, "the limit record stayed cleared"); + assert_eq!( + limited_prompts.lock().unwrap().clone(), + vec!["Remember PINEAPPLE"], + "the limited harness never ran again" + ); + core.shutdown().await; +} + +/// A message held for the turn end carries its FULL RunRequest — the +/// harness, model, options it was sent with — and the drain prefers that +/// stored request over the chat row. The switch rewrites the stored request +/// onto its config (ticket 24: "queued messages after a switch go to the new +/// harness"), or the held message would run on the OLD harness (its model +/// ids in the new harness's vocabulary) after the switch. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_full_request_held_across_the_switch_runs_on_the_new_harness() { + let (core, limited, limited_prompts, pi, pi_requests, _tmp) = setup(); + // The opening turn runs the same claude model the held Run is sent + // with: only a Run the live runtime would carry is held — any other + // config replaces the runtime instead. + queue_run( + &core, + "Remember PINEAPPLE", + HarnessId::ClaudeCode, + Some("claude-model"), + "m-open", + ); + start_limited_turn(&limited_prompts).await; + // A full Run command sent while the turn-boundary run is working: it is + // HELD for the turn's end with the exact request it was sent with + // (claude, a claude model) — not a text-only queue row. + queue_run( + &core, + "queued as a full request", + HarnessId::ClaudeCode, + Some("claude-model"), + "m-held", + ); + wait_for( + || queue_texts(&core) == vec!["queued as a full request"], + "the full request to be held for the turn's end", + ) + .await; + let _ = limited.finish.send(()); + settle_limited(&core).await; + assert_eq!(queue_texts(&core), vec!["queued as a full request"]); + + // The switch, while the held row waits behind the Limited pause. + switch(&core, HarnessId::Pi, Some("pi-default")) + .await + .expect("switch while limited"); + expect_seams(&core, HarnessId::Pi, 1); + + // The continuation runs on the new harness; when it settles, the held + // row follows — rewritten onto the switch's config. + queue_run(&core, CONTINUE_PROMPT, HarnessId::Pi, Some("pi-default"), "m-resume"); + wait_for( + || pi_requests + .lock() + .unwrap() + .iter() + .any(|r| r.prompt.ends_with(CONTINUE_PROMPT)), + "the continue prompt to run on the new harness", + ) + .await; + let _ = pi.finish.send(()); + wait_for( + || pi_requests + .lock() + .unwrap() + .iter() + .any(|r| r.prompt.contains("queued as a full request")), + "the held row to follow the continuation", + ) + .await; + let held = pi_requests + .lock() + .unwrap() + .iter() + .find(|r| r.prompt.contains("queued as a full request")) + .cloned() + .expect("the held request"); + assert_eq!( + held.harness, Some(HarnessId::Pi), + "the stored request runs on the NEW harness after the switch" + ); + assert_eq!( + held.model.as_deref(), + Some("pi-default"), + "the stored request carries the switch's model, not the claude one" + ); + assert!(held.prompt.contains("queued as a full request"), "{}", held.prompt); + let _ = pi.finish.send(()); + wait_for( + || queue_texts(&core).is_empty() + && core + .sessions + .session_status(CHAT) + .is_none_or(|s| s.status != SessionStatus::Working), + "the last turn to settle and the queue to drain", + ) + .await; + assert_eq!( + limited_prompts.lock().unwrap().clone(), + vec!["Remember PINEAPPLE"], + "the limited harness never ran the held message" + ); + core.shutdown().await; +} + +/// The switch alone does NOT clear the Limited state: the record lives as +/// long as the errored status it rode, so the banner stays up (and the +/// Resume / continue actions stay offered) until a turn actually starts. A +/// crash between the switch and the send leaves an honest Limited chat on +/// the new harness, ready to resume. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_switch_alone_keeps_the_chat_limited_until_a_turn_runs() { + let (core, limited, limited_prompts, pi, pi_requests, _tmp) = setup(); + queue_run(&core, "Remember PINEAPPLE", HarnessId::ClaudeCode, None, "m-open"); + start_limited_turn(&limited_prompts).await; + let _ = limited.finish.send(()); + settle_limited(&core).await; + + switch(&core, HarnessId::Pi, None).await.expect("switch"); + expect_seams(&core, HarnessId::Pi, 1); + assert_eq!( + chat_row(&core).config.as_ref().map(|c| c.harness), + Some(HarnessId::Pi) + ); + // Still Limited: errored with the record — no turn has started. + assert!(limit_row(&core).is_some()); + assert_eq!( + core.sessions.session_status(CHAT).map(|s| s.status), + Some(SessionStatus::Errored) + ); + + // The user's own next message (a normal send, not the resume prompt) + // clears it just the same. + queue_run(&core, "try a different angle", HarnessId::Pi, None, "m-next"); + // Wait for the harness to hold the run, not just for Working: the + // status publishes before run() subscribes to `finish`, so a send in + // that window would be lost and the turn would never settle. + wait_for( + || pi_requests + .lock() + .unwrap() + .iter() + .any(|r| r.prompt.contains("try a different angle")), + "the new turn to start", + ) + .await; + let _ = pi.finish.send(()); + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == SessionStatus::Idle) + }, + "the new turn to settle", + ) + .await; + assert_eq!(limit_row(&core), None, "the new turn cleared the limit record"); + core.shutdown().await; +} diff --git a/crates/engine/tests/cursor_live.rs b/crates/engine/tests/cursor_live.rs index 3c9010145b..fd7027af59 100644 --- a/crates/engine/tests/cursor_live.rs +++ b/crates/engine/tests/cursor_live.rs @@ -84,6 +84,8 @@ fn start(core: &EngineCore, cwd: &std::path::Path, prompt: String) { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }, }, ) @@ -130,6 +132,7 @@ async fn remote_steer_batch_reaches_real_muse_with_every_message() { .as_millis() as i64; for (i, token) in tokens.iter().enumerate().skip(1) { handle.doc().queue_command(&roboco_doc::SessionCommandEntry { + origin_chat_id: None, id: format!("remote-{i}"), payload: SessionCommandPayload::Steer {prompt: format!("Remember {token}. Reply with the immediately previous user token and this token. Do not use tools."), message_id: Some(format!("message-{i}"))}, issued_by: "remote-viewer".into(), issued_at: now + i as i64, diff --git a/crates/engine/tests/e2e.rs b/crates/engine/tests/e2e.rs index 2ca2aa62aa..ad3d7cb8fa 100644 --- a/crates/engine/tests/e2e.rs +++ b/crates/engine/tests/e2e.rs @@ -39,6 +39,8 @@ fn run_request(prompt: &str) -> RunRequest { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } @@ -284,7 +286,8 @@ async fn update_deferred_steering_preserves_the_active_turn_and_queued_prompt() HarnessId::Mock, run_request("opening"), Some("opening".into()), - ) + None, + None) .await .unwrap(); tx.send(mock_script()[0].clone()).unwrap(); @@ -420,7 +423,8 @@ async fn pending_update_does_not_block_dispatch_or_other_harnesses() { HarnessId::ClaudeCode, run_request("waiting"), None, - ), + None, + None), ) .await .expect("dispatch must not wait for the update") @@ -434,13 +438,14 @@ async fn pending_update_does_not_block_dispatch_or_other_harnesses() { HarnessId::ClaudeCode, run_request("cancelled"), None, - ), + None, + None), ) .await .expect("dispatch must not wait for installation") .unwrap(); core.sessions - .dispatch("other", HarnessId::Codex, run_request("other"), None) + .dispatch("other", HarnessId::Codex, run_request("other"), None, None, None) .await .unwrap(); wait_for( @@ -477,6 +482,7 @@ fn queue_as_viewer(doc: &SessionDoc, id: &str, payload: SessionCommandPayload) { frontier: None, }); doc.queue_command(&SessionCommandEntry { + origin_chat_id: None, id: id.into(), payload, issued_by: VIEWER.into(), @@ -1227,6 +1233,7 @@ async fn recover_stale_journal_stamps_aborted_on_boot() { let doc = SessionDoc::init(CHAT).unwrap(); doc.push_message(&SessionMessageEntry { + origin_chat_id: None, id: "m-user".into(), role: MessageRole::User, parts: vec![MessagePart::Text { @@ -1237,6 +1244,7 @@ async fn recover_stale_journal_stamps_aborted_on_boot() { device_id: device_id.into(), status: Some(MessageStatus::Complete), continuation_of: None, + provider_turn: None, duration_ms: None, }) .unwrap(); @@ -1329,6 +1337,7 @@ async fn recover_stale_journal_settles_chips_in_completed_local_entries() { let doc = SessionDoc::init(CHAT).unwrap(); for (id, device) in [("local", device_id), ("remote", "other-device")] { doc.push_message(&SessionMessageEntry { + origin_chat_id: None, id: id.into(), role: MessageRole::Assistant, parts: vec![ @@ -1339,6 +1348,7 @@ async fn recover_stale_journal_settles_chips_in_completed_local_entries() { device_id: device.into(), status: Some(MessageStatus::Complete), continuation_of: None, + provider_turn: None, duration_ms: None, }) .unwrap(); @@ -1400,6 +1410,7 @@ async fn subagent_done_without_a_live_sink_updates_a_persisted_chip() { .unwrap() .doc() .push_message(&SessionMessageEntry { + origin_chat_id: None, id: "old-assistant".into(), role: MessageRole::Assistant, parts: vec![spawn_chip("old-spawn", SubagentStatus::Running)], @@ -1407,11 +1418,12 @@ async fn subagent_done_without_a_live_sink_updates_a_persisted_chip() { device_id: core.device_id.clone(), status: Some(MessageStatus::Complete), continuation_of: None, + provider_turn: None, duration_ms: None, }) .unwrap(); core.sessions - .dispatch(CHAT, HarnessId::Mock, run_request("resume"), None) + .dispatch(CHAT, HarnessId::Mock, run_request("resume"), None, None, None) .await .unwrap(); wait_for( @@ -2439,6 +2451,8 @@ async fn real_claude_sees_uploaded_image_inline() { auto_approve: false, attachments: vec![path], resume: None, + provider_fork: None, + resume_fallback_history: None, worktree: None, mcp: None, }; @@ -2963,13 +2977,14 @@ async fn pending_steer_handoff_does_not_publish_a_completion() { HarnessId::Mock, run_request("redirect"), Some("user-steer".into()), - ) + None, + None) .await .unwrap(); } else { assert_eq!( core.sessions - .steer(CHAT, "redirect", Some("user-steer".into())) + .steer(CHAT, "redirect", Some("user-steer".into()), None) .await .unwrap(), roboco_engine::sessions::SteerOutcome::Accepted @@ -3080,7 +3095,7 @@ async fn generated_image_is_materialized_before_publication_and_survives_reopen( ); sessions.set_doc_host(host.clone()); sessions - .dispatch(CHAT, HarnessId::Mock, run_request("image"), None) + .dispatch(CHAT, HarnessId::Mock, run_request("image"), None, None, None) .await .unwrap(); wait_for( @@ -3128,7 +3143,7 @@ async fn generated_image_is_materialized_before_publication_and_survives_reopen( ); // Resume echoes the successful completed item after Codex removed its source. sessions - .dispatch(CHAT, HarnessId::Mock, run_request("resume"), None) + .dispatch(CHAT, HarnessId::Mock, run_request("resume"), None, None, None) .await .unwrap(); wait_for( @@ -3180,7 +3195,8 @@ async fn real_image_generation_profile_smoke() { HarnessId::Codex, run_request("Generate an image of a small green goblin using image generation."), None, - ) + None, + None) .await .unwrap(); tokio::time::timeout(Duration::from_secs(300), async { diff --git a/crates/engine/tests/engine_local.rs b/crates/engine/tests/engine_local.rs index 5e69500d84..6873ec150f 100644 --- a/crates/engine/tests/engine_local.rs +++ b/crates/engine/tests/engine_local.rs @@ -68,7 +68,7 @@ async fn local_listener_feeds_exclude_legacy_cloud_rows_and_keep_transcripts() { core.doc_host .open("local-chat") .unwrap() - .write_user_message("m1", "saved locally", 123) + .write_user_message("m1", "saved locally", 123, None) .unwrap(); // Completed subagent/tool sidecars use the local database too. core.doc_host.upload_tool_sidecar( diff --git a/crates/engine/tests/harness_switch.rs b/crates/engine/tests/harness_switch.rs new file mode 100644 index 0000000000..55ab5aad45 --- /dev/null +++ b/crates/engine/tests/harness_switch.rs @@ -0,0 +1,607 @@ +//! One-way harness switch between turns (t3-orchestration-v2 ticket 24). +//! +//! Engine-side machinery under test: +//! - the `SwitchHarness` RPC writes the "Switched to " seam as a +//! system entry, clears the chat's harness session (the empty-string +//! tombstone — which also stops the journal scan from resurrecting the old +//! harness's session ids), retires any stale native-fork intent, and saves +//! the new harness and model; +//! - it is refused while a turn runs, on a switch onto the harness the chat +//! already runs, for an unknown chat, and for a harness that is not +//! registered; +//! - the next send dispatches on the NEW harness with no resume, carrying +//! the budgeted transcript (ticket 01) of everything above the seam — for +//! a queued message too (the queue rides the chat row's config) — exactly +//! once per provider session; +//! - switching back is just another one-way switch: a second seam, another +//! fresh session, the whole conversation (old-harness turns included) in +//! the budgeted history; +//! - the cleared session stays cleared across an engine restart: the +//! journal's old-harness session ids never ride a resume on the new +//! harness. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use futures::StreamExt; +use futures::stream::BoxStream; + +use roboco_doc::{MessagePart, MessageRole, MessageStatus, SessionMessageEntry}; +use roboco_engine::{EngineCore, HarnessRegistry}; +use roboco_harness::{Harness, HarnessError, RunControls}; +use roboco_proto::{ + AgentEvent, ChatConfig, DoneStatus, HarnessId, Model, ReasoningLevel, RunRequest, + SandboxLevel, SteeringMode, +}; +use roboco_rpc::methods; + +const CHAT: &str = "chat-switch"; + +/// A scripted harness that answers at once and records every request it ran. +/// One instance per harness id, so a switch shows up as the requests moving +/// from one capture to the other. +struct Capture { + id: HarnessId, + name: &'static str, + /// The provider session id every Done reports (distinct per harness, so + /// assertions can tell which session the engine resumed). + session: &'static str, + reply: &'static str, + requests: Arc>>, +} + +impl Capture { + fn build( + id: HarnessId, + name: &'static str, + session: &'static str, + reply: &'static str, + ) -> (Arc, Arc>>) { + let requests = Arc::new(Mutex::new(Vec::new())); + ( + Arc::new(Self { + id, + name, + session, + reply, + requests: requests.clone(), + }), + requests, + ) + } +} + +#[async_trait] +impl Harness for Capture { + fn id(&self) -> HarnessId { + self.id + } + fn display_name(&self) -> &str { + self.name + } + fn supports_steering(&self) -> bool { + false + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::TurnBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + _: RunControls, + ) -> Result>, HarnessError> { + self.requests.lock().unwrap().push(request.clone()); + let reply = self.reply.to_string(); + let session = self.session.to_string(); + Ok(futures::stream::iter(vec![ + Ok(AgentEvent::TextDelta { text: reply }), + Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some(session), + }), + ]) + .boxed()) + } +} + +/// A harness whose turn runs until the test says so — "the chat is working" +/// as a state the test controls rather than races. +struct Held { + finish: tokio::sync::broadcast::Sender<()>, +} + +impl Held { + fn new() -> (Arc, tokio::sync::broadcast::Sender<()>) { + let (finish, _) = tokio::sync::broadcast::channel(16); + (Arc::new(Self { finish: finish.clone() }), finish) + } +} + +#[async_trait] +impl Harness for Held { + fn id(&self) -> HarnessId { + HarnessId::Mock + } + fn display_name(&self) -> &str { + "Held" + } + fn supports_steering(&self) -> bool { + false + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::TurnBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + _: RunRequest, + _: RunControls, + ) -> Result>, HarnessError> { + let mut finish = self.finish.subscribe(); + Ok(futures::stream::once(async move { + let _ = finish.recv().await; + Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some("held-session".into()), + }) + }) + .boxed()) + } +} + +async fn wait_for(mut predicate: F, what: &str) +where + F: FnMut() -> bool, +{ + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while !predicate() { + assert!( + tokio::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(Duration::from_millis(15)).await; + } +} + +fn create_chat(core: &EngineCore, config: ChatConfig) { + core.workspace + .create_chat(CHAT, None, Some(&core.device_id), Some(config), Some("/tmp".into())) + .expect("create chat"); + // Pre-title so the auto-titler never dispatches a harness request of its own. + core.workspace.rename_chat(CHAT, "Pre-titled").expect("rename chat"); +} + +fn entries(core: &EngineCore) -> Vec { + core.doc_host + .open(CHAT) + .expect("open chat") + .doc() + .read_entries() + .expect("read entries") +} + +/// The chat row as the workspace doc sees it. +fn chat_row(core: &EngineCore) -> roboco_proto::Chat { + core.workspace.chat(CHAT).expect("chat row").expect("chat exists") +} + +fn mock_config() -> ChatConfig { + ChatConfig { + harness: HarnessId::Mock, + model: Some("mock-model".into()), + reasoning: None, + model_options: Default::default(), + sandbox: SandboxLevel::WorkspaceWrite, + } +} + +/// Queue one message on the (idle) chat — the composer's send path; the +/// drain dispatches it on the chat row's harness. +fn send(core: &EngineCore, text: &str) { + core.doc_host.queue_message(CHAT, text, Vec::new()).expect("queue message"); +} + +async fn switch( + core: &EngineCore, + harness: HarnessId, + model: Option<&str>, +) -> Result { + let client = roboco_rpc::memory_client(core.rpc_service()); + let mut params = serde_json::json!({ "chatId": CHAT, "harness": harness }); + if let Some(model) = model { + params["model"] = serde_json::Value::String(model.into()); + } + client.call(methods::SWITCH_HARNESS, params).await +} + +/// The transcript's harness-switch seams naming `harness`: `count` of them, +/// each a complete system row. +fn expect_seams(core: &EngineCore, harness: HarnessId, count: usize) { + let seams: Vec = entries(core) + .into_iter() + .filter(|entry| { + entry.role == MessageRole::System + && entry.parts.iter().any(|part| { + matches!( + part, + MessagePart::HarnessSwitch { harness: to, .. } if *to == harness + ) + }) + }) + .collect(); + assert_eq!(seams.len(), count, "seams for {harness:?}"); + for seam in &seams { + assert_eq!(seam.status, Some(MessageStatus::Complete)); + } +} + +/// One mock exchange leaves the transcript and the row session behind. +async fn run_mock_turn( + core: &EngineCore, + mock_requests: &Arc>>, + prompt: &str, +) { + send(core, prompt); + wait_for( + || mock_requests.lock().unwrap().iter().any(|r| r.prompt.contains(prompt)), + "the mock turn to run", + ) + .await; + wait_for( + || chat_row(core).harness_session_id.as_deref() == Some("mock-session"), + "the mock session to be recorded", + ) + .await; + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_none_or(|s| s.status != roboco_proto::SessionStatus::Working) + }, + "the mock turn to settle", + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn switch_writes_the_seam_clears_the_session_and_saves_the_harness() { + let tmp = tempfile::tempdir().unwrap(); + let (mock, mock_requests) = + Capture::build(HarnessId::Mock, "Mock", "mock-session", "I remember PINEAPPLE"); + let (pi, _) = Capture::build(HarnessId::Pi, "Pi", "pi-session", "pi answer"); + let registry = HarnessRegistry::new(); + registry.register(mock); + registry.register(pi); + let core = EngineCore::assemble(tmp.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + create_chat(&core, mock_config()); + + run_mock_turn(&core, &mock_requests, "Remember PINEAPPLE").await; + let before = entries(&core).len(); + assert_eq!(chat_row(&core).harness_session_id.as_deref(), Some("mock-session")); + + switch(&core, HarnessId::Pi, Some("pi-default")).await.expect("switch"); + + // The seam: a system entry after the transcript, complete, naming the + // new harness. + expect_seams(&core, HarnessId::Pi, 1); + assert_eq!(entries(&core).len(), before + 1); + // The row: new harness and model, and no session left to resume (how + // the clear is spelled is the engine's business; that the old session + // never rides a resume — not even after a restart's journal scan — is + // asserted by the post-switch send tests below). + let row = chat_row(&core); + assert_eq!(row.config.as_ref().map(|c| c.harness), Some(HarnessId::Pi)); + assert_eq!(row.config.as_ref().and_then(|c| c.model.as_deref()), Some("pi-default")); + assert!(row.harness_session_id.as_deref().is_none_or(str::is_empty)); + assert!(row.harness_session_cwd.as_deref().is_none_or(|cwd| cwd.is_empty())); + + // Refusals: the same harness, an unregistered one, an unknown chat. + let again = switch(&core, HarnessId::Pi, None).await.unwrap_err(); + assert!(again.to_string().contains("already on"), "{again}"); + let unregistered = switch(&core, HarnessId::Codex, None).await.unwrap_err(); + assert!(unregistered.to_string().contains("not found"), "{unregistered}"); + let client = roboco_rpc::memory_client(core.rpc_service()); + let no_chat = client + .call( + methods::SWITCH_HARNESS, + serde_json::json!({ "chatId": "nope", "harness": "pi" }), + ) + .await + .unwrap_err(); + assert!(no_chat.to_string().contains("no longer exists"), "{no_chat}"); + // The refusals wrote nothing: one seam, the pi row. + expect_seams(&core, HarnessId::Pi, 1); + assert_eq!(chat_row(&core).config.as_ref().map(|c| c.harness), Some(HarnessId::Pi)); + + core.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_switch_is_refused_while_the_chat_works_and_allowed_once_idle() { + let tmp = tempfile::tempdir().unwrap(); + let (held, finish) = Held::new(); + let (pi, _) = Capture::build(HarnessId::Pi, "Pi", "pi-session", "pi answer"); + let registry = HarnessRegistry::new(); + registry.register(held); + registry.register(pi); + let core = EngineCore::assemble(tmp.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + create_chat(&core, mock_config()); + + send(&core, "hold this turn"); + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == roboco_proto::SessionStatus::Working) + }, + "the turn to start", + ) + .await; + + let refused = switch(&core, HarnessId::Pi, None).await.unwrap_err(); + assert!(refused.to_string().contains("between turns"), "{refused}"); + // Nothing was written: no seam, no row change. + assert!(entries(&core).iter().all(|entry| entry.role != MessageRole::System)); + assert_eq!(chat_row(&core).config.as_ref().map(|c| c.harness), Some(HarnessId::Mock)); + + // The turn ends → the chat is idle → the same switch goes through. + let _ = finish.send(()); + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_none_or(|s| s.status != roboco_proto::SessionStatus::Working) + }, + "the turn to settle", + ) + .await; + switch(&core, HarnessId::Pi, None).await.expect("switch once idle"); + expect_seams(&core, HarnessId::Pi, 1); + + core.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_next_send_carries_the_budgeted_history_exactly_once_on_the_new_harness() { + let tmp = tempfile::tempdir().unwrap(); + let (mock, mock_requests) = + Capture::build(HarnessId::Mock, "Mock", "mock-session", "I remember PINEAPPLE"); + let (pi, pi_requests) = Capture::build(HarnessId::Pi, "Pi", "pi-session", "pi answer"); + let registry = HarnessRegistry::new(); + registry.register(mock); + registry.register(pi); + let core = EngineCore::assemble(tmp.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + create_chat(&core, mock_config()); + + run_mock_turn(&core, &mock_requests, "Remember PINEAPPLE").await; + switch(&core, HarnessId::Pi, Some("pi-default")).await.expect("switch"); + + // A queued message after the switch: the chat row routes it to the new + // harness, the old session never rides a resume, and the budgeted + // transcript of everything above the seam rides in front of it. + send(&core, "What did I ask you to remember?"); + wait_for( + || { + pi_requests + .lock() + .unwrap() + .iter() + .any(|r| r.prompt.contains("What did I ask you to remember?")) + }, + "the post-switch send to reach pi", + ) + .await; + let request = pi_requests.lock().unwrap().last().cloned().unwrap(); + assert_eq!(request.harness, Some(HarnessId::Pi)); + assert_eq!(request.resume, None, "the old harness's session must not resume"); + assert!(request.prompt.contains("Remember PINEAPPLE"), "{}", request.prompt); + assert!(request.prompt.contains("I remember PINEAPPLE"), "{}", request.prompt); + assert!(request.prompt.contains(""), "{}", request.prompt); + // The seam itself is structure, never agent-visible text. + assert!(!request.prompt.contains("switch:"), "{}", request.prompt); + wait_for( + || chat_row(&core).harness_session_id.as_deref() == Some("pi-session"), + "the new harness's session to be recorded", + ) + .await; + + // The send after that continues the pi session as-is: the history went + // out exactly once, to the session that now holds it. + send(&core, "and now?"); + wait_for( + || pi_requests.lock().unwrap().len() >= 2, + "the second post-switch send to reach pi", + ) + .await; + let request = pi_requests.lock().unwrap().last().cloned().unwrap(); + assert_eq!(request.resume.as_deref(), Some("pi-session")); + assert!( + !request.prompt.contains("PINEAPPLE"), + "the history must not ride twice: {}", + request.prompt + ); + assert!(request.prompt.contains("and now?"), "{}", request.prompt); + + core.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_native_command_first_turn_defers_the_history_to_the_next_ordinary_send() { + let tmp = tempfile::tempdir().unwrap(); + let (mock, mock_requests) = + Capture::build(HarnessId::Mock, "Mock", "mock-session", "I remember PINEAPPLE"); + let (pi, pi_requests) = Capture::build(HarnessId::Pi, "Pi", "pi-session", "pi answer"); + let registry = HarnessRegistry::new(); + registry.register(mock); + registry.register(pi); + let core = EngineCore::assemble(tmp.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + create_chat(&core, mock_config()); + + run_mock_turn(&core, &mock_requests, "Remember PINEAPPLE").await; + switch(&core, HarnessId::Pi, None).await.expect("switch"); + + // The first post-switch send is a native command: it goes out bare (no + // wrapper in front of a leading `/name`), so the history it owes the + // fresh session has to ride the first ORDINARY send instead. + send(&core, "/review"); + wait_for( + || pi_requests.lock().unwrap().iter().any(|r| r.prompt.contains("/review")), + "the native command to reach pi", + ) + .await; + let command = pi_requests.lock().unwrap().last().cloned().unwrap(); + assert_eq!(command.prompt, "/review"); + assert!(!command.prompt.contains("PINEAPPLE"), "{}", command.prompt); + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_none_or(|s| s.status != roboco_proto::SessionStatus::Working) + }, + "the command turn to settle", + ) + .await; + + send(&core, "What did I ask you to remember?"); + wait_for( + || pi_requests.lock().unwrap().len() >= 2, + "the first ordinary send to reach pi", + ) + .await; + let request = pi_requests.lock().unwrap().last().cloned().unwrap(); + assert_eq!(request.resume.as_deref(), Some("pi-session")); + assert!(request.prompt.contains("PINEAPPLE"), "{}", request.prompt); + assert!(request.prompt.contains("What did I ask you to remember?"), "{}", request.prompt); + // The command's own turn is below the seam: it already rode its run, + // and is never pasted as conversation text. + assert!(!request.prompt.contains("/review"), "{}", request.prompt); + assert!(!request.prompt.contains("switch:"), "{}", request.prompt); + + core.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn switching_back_is_another_one_way_switch_with_the_whole_conversation() { + let tmp = tempfile::tempdir().unwrap(); + let (mock, mock_requests) = + Capture::build(HarnessId::Mock, "Mock", "mock-session", "I remember PINEAPPLE"); + let (pi, pi_requests) = Capture::build(HarnessId::Pi, "Pi", "pi-session", "the pi exchange"); + let registry = HarnessRegistry::new(); + registry.register(mock); + registry.register(pi); + let core = EngineCore::assemble(tmp.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + create_chat(&core, mock_config()); + + run_mock_turn(&core, &mock_requests, "Remember PINEAPPLE").await; + switch(&core, HarnessId::Pi, None).await.expect("switch to pi"); + send(&core, "continue on pi"); + wait_for( + || pi_requests.lock().unwrap().iter().any(|r| r.prompt.contains("continue on pi")), + "the pi turn to run", + ) + .await; + wait_for( + || chat_row(&core).harness_session_id.as_deref() == Some("pi-session"), + "the pi session to be recorded", + ) + .await; + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_none_or(|s| s.status != roboco_proto::SessionStatus::Working) + }, + "the pi turn to settle", + ) + .await; + + // Switching back: a second seam, the pi session dropped the same way. + switch(&core, HarnessId::Mock, Some("mock-model")).await.expect("switch back"); + expect_seams(&core, HarnessId::Pi, 1); + expect_seams(&core, HarnessId::Mock, 1); + assert_eq!(chat_row(&core).harness_session_id.as_deref(), Some("")); + let mock_turns = mock_requests.lock().unwrap().len(); + assert_eq!(mock_turns, 1, "the mock harness ran exactly its one turn"); + + send(&core, "back on mock"); + wait_for( + || mock_requests.lock().unwrap().len() >= 2, + "the post-switch send to reach the mock harness", + ) + .await; + let request = mock_requests.lock().unwrap().last().cloned().unwrap(); + assert_eq!(request.harness, Some(HarnessId::Mock)); + assert_eq!(request.resume, None, "neither session may resume after the switch"); + // The budgeted history now covers BOTH sides of the earlier seam: the + // original mock exchange and the pi exchange above the switch back. + assert!(request.prompt.contains("Remember PINEAPPLE"), "{}", request.prompt); + assert!(request.prompt.contains("the pi exchange"), "{}", request.prompt); + assert!(request.prompt.contains("back on mock"), "{}", request.prompt); + assert!(!request.prompt.contains("switch:"), "{}", request.prompt); + + core.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_cleared_session_stays_cleared_across_an_engine_restart() { + let tmp = tempfile::tempdir().unwrap(); + let (mock, mock_requests) = + Capture::build(HarnessId::Mock, "Mock", "mock-session", "I remember PINEAPPLE"); + let (pi, _) = Capture::build(HarnessId::Pi, "Pi", "pi-session", "pi answer"); + let registry = HarnessRegistry::new(); + registry.register(mock); + registry.register(pi); + let core = EngineCore::assemble(tmp.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + create_chat(&core, mock_config()); + run_mock_turn(&core, &mock_requests, "Remember PINEAPPLE").await; + switch(&core, HarnessId::Pi, None).await.expect("switch"); + + // Restart: the journal still holds the mock session ids; the tombstone + // on the row must keep them from riding a resume on the new harness. + core.shutdown().await; + drop(core); + let (pi, pi_requests) = Capture::build(HarnessId::Pi, "Pi", "pi-session", "pi answer"); + let registry = HarnessRegistry::new(); + registry.register(pi); + let restarted = + EngineCore::assemble(tmp.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + assert_eq!(chat_row(&restarted).harness_session_id.as_deref(), Some("")); + + send(&restarted, "What did I ask you to remember?"); + wait_for( + || { + pi_requests + .lock() + .unwrap() + .iter() + .any(|r| r.prompt.contains("What did I ask you to remember?")) + }, + "the post-restart send to reach pi", + ) + .await; + let request = pi_requests.lock().unwrap().last().cloned().unwrap(); + assert_eq!(request.harness, Some(HarnessId::Pi)); + assert_eq!( + request.resume, None, + "the journal must not resurrect the old harness's session" + ); + assert!(request.prompt.contains("PINEAPPLE"), "{}", request.prompt); + + restarted.shutdown().await; +} diff --git a/crates/engine/tests/limited_chats.rs b/crates/engine/tests/limited_chats.rs new file mode 100644 index 0000000000..595e5a5dfd --- /dev/null +++ b/crates/engine/tests/limited_chats.rs @@ -0,0 +1,737 @@ +//! Limited chats (orchestration v2, ticket 06): a run that ends on a +//! usage-limit error leaves the additive limit record on the chat's status — +//! harness, window, reset time (filled from the harness account's cached +//! windows when the harness's own error carried none) and the account it +//! happened on. Queued messages wait behind the continuation; Resume sends +//! the continue prompt as a normal run; a new turn clears the record. +//! +//! The harness seam is a scripted stand-in (`LimitHarness`): its first run +//! ends on a classified usage-limit error exactly the way the Claude/Codex +//! drivers do (`Error { kind }` then an errored `Done`), later runs hold +//! until the test releases them so queue-order assertions are deterministic. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use futures::StreamExt; +use futures::stream::BoxStream; + +use roboco_doc::{MessagePart, MessageRole, SessionCommandPayload}; +use roboco_engine::{AgentAccounts, AgentAccountsConfig, EngineCore, HarnessRegistry}; +use roboco_harness::{Harness, HarnessError, RunControls}; +use roboco_proto::{ + AgentErrorKind, AgentEvent, DoneStatus, HarnessId, Model, ReasoningLevel, RunRequest, + SandboxLevel, SessionLimit, SessionStatus, SteeringMode, +}; + +const CHAT: &str = "chat-limited"; + +/// A scripted harness: the FIRST run ends on an error carrying `kind` +/// (a usage-limit kind for the Limited tests, `None` for plain failures); +/// every run waits for a `finish` signal before its terminal event, so the +/// test owns when turns end; later runs complete normally. +struct LimitHarness { + id: HarnessId, + kind: Option, + /// A second classified error the doomed run emits AFTER the first — + /// Claude's live shape (a detailed `rate_limit_event`, then the CLI's + /// terse `rate_limit` assistant code), which the engine's limit merge + /// must survive. + second_kind: Option, + finish: tokio::sync::broadcast::Sender<()>, + prompts: Arc>>, + runs: std::sync::atomic::AtomicUsize, +} + +impl LimitHarness { + fn new(id: HarnessId, kind: Option) -> (Arc, Arc>>) { + Self::with_second_kind(id, kind, None) + } + + fn with_second_kind( + id: HarnessId, + kind: Option, + second_kind: Option, + ) -> (Arc, Arc>>) { + let (finish, _) = tokio::sync::broadcast::channel(16); + let prompts = Arc::new(Mutex::new(Vec::new())); + ( + Arc::new(Self { + id, + kind, + second_kind, + finish, + prompts: prompts.clone(), + runs: std::sync::atomic::AtomicUsize::new(0), + }), + prompts, + ) + } +} + +#[async_trait] +impl Harness for LimitHarness { + fn id(&self) -> HarnessId { + self.id + } + fn display_name(&self) -> &str { + "Limit" + } + fn supports_steering(&self) -> bool { + false + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::TurnBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[ReasoningLevel::Medium] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + _controls: RunControls, + ) -> Result>, HarnessError> { + // Subscribe before recording the prompt: a test that sees the prompt + // may send `finish` at once. + let mut finish = self.finish.subscribe(); + self.prompts.lock().unwrap().push(request.prompt.clone()); + let run = self.runs.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let kind = self.kind.clone(); + let second_kind = self.second_kind.clone(); + let session = format!("sess-{run}"); + let started = futures::stream::iter(vec![Ok(AgentEvent::SessionStarted { + harness: self.id, + model: "limit-1".into(), + tools: vec![], + cwd: request.cwd.clone(), + session_id: session.clone(), + assistant_message_id: format!("a-{run}"), + })]); + let tail = futures::stream::once(async move { + // The turn runs until the test says it ends. + let _ = finish.recv().await; + if run == 0 { + // The classified limit error (then an optional second one — + // `second_kind`, Claude's live shape), then the errored Done + // — the driver shape ticket 05 landed. `None` kind = a + // plain, unclassified failure (the not-Limited control + // case). + let (message, kind) = match kind { + Some(kind @ AgentErrorKind::UsageLimit { .. }) => ( + "Claude 5-hour limit reached — the turn was blocked.", + Some(kind), + ), + Some(kind) => ("The agent failed its login.", Some(kind)), + None => ("The agent child crashed.", None), + }; + let mut events = vec![ + Ok(AgentEvent::Error { + message: message.into(), + kind, + }), + ]; + if let Some(kind) = second_kind { + events.push(Ok(AgentEvent::Error { + message: "Claude usage limit reached — try again after the limit resets." + .into(), + kind: Some(kind), + })); + } + events.push(Ok(AgentEvent::Done { + status: DoneStatus::Errored, + result: None, + error: Some("the turn failed".into()), + session_id: Some(session), + })); + events + } else { + vec![ + Ok(AgentEvent::TextDelta { + text: "Picked the work back up.".into(), + }), + Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some(session), + }), + ] + } + }) + .flat_map(futures::stream::iter); + Ok(started.chain(tail).boxed()) + } +} + +async fn wait_for(mut predicate: F, what: &str) +where + F: FnMut() -> bool, +{ + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while !predicate() { + assert!( + tokio::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(Duration::from_millis(15)).await; + } +} + +/// The continue prompt Resume sends — asserted as the exact engine-side run +/// prompt, spelled by the shared proto constant. +fn run_request(prompt: &str, harness: HarnessId) -> RunRequest { + RunRequest { + prompt: prompt.into(), + harness: Some(harness), + model: None, + reasoning: None, + model_options: Default::default(), + cwd: ".".into(), + sandbox: SandboxLevel::WorkspaceWrite, + auto_approve: true, + attachments: Vec::new(), + worktree: None, + mcp: None, + resume: None, + provider_fork: None, + resume_fallback_history: None, + } +} + +/// Queue a Run command the way the composer's Resume / normal send does +/// (the durable command plane, not the visible message queue). +fn queue_run(core: &EngineCore, prompt: &str, harness: HarnessId, message_id: &str) { + core.doc_host + .queue_command( + CHAT, + SessionCommandPayload::Run { + request: run_request(prompt, harness), + message_id: message_id.into(), + }, + ) + .expect("queue run command"); +} + +fn queue_texts(core: &EngineCore) -> Vec { + core.doc_host + .open(CHAT) + .ok() + .and_then(|h| h.doc().read_queue().ok()) + .unwrap_or_default() + .into_iter() + .map(|item| item.text) + .collect() +} + +fn user_texts(core: &EngineCore) -> Vec { + core.doc_host + .open(CHAT) + .ok() + .and_then(|h| h.doc().read_entries().ok()) + .unwrap_or_default() + .iter() + .filter(|e| e.role == MessageRole::User) + .map(|e| { + e.parts + .iter() + .filter_map(|p| match p { + MessagePart::Text { text, .. } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("") + }) + .collect() +} + +/// Assemble a core around the scripted harness, pre-titled so the +/// auto-titler never dispatches a request of its own, and mount an ISOLATED +/// accounts store (empty by default) so limit records never read the +/// machine's real CLI logins. +async fn setup( + built: (Arc, Arc>>), +) -> ( + EngineCore, + Arc, + Arc>>, +) { + let tmp = tempfile::tempdir().unwrap(); + // Leak the guard: the engine outlives this helper; uniqueness is all the + // test needs. + let path = tmp.keep(); + let (harness, prompts) = built; + let registry = HarnessRegistry::new(); + registry.register(harness.clone()); + let core = EngineCore::assemble(&path.join("data"), Arc::new(registry), HarnessId::Mock) + .expect("engine core assembles"); + core.sessions + .set_agent_accounts(AgentAccounts::new(AgentAccountsConfig::isolated( + &path.join("accounts"), + ))); + create_chat(&core).await; + (core, harness, prompts) +} + +async fn create_chat(core: &EngineCore) { + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": CHAT, + "deviceId": core.device_id, + }), + ) + .await + .expect("createChat"); + // Pre-title so the auto-titler never dispatches a harness request of its own. + core.workspace + .rename_chat(CHAT, "Pre-titled") + .expect("rename chat"); +} + +fn limit_row(core: &EngineCore) -> Option { + core.sessions + .session_status(CHAT) + .and_then(|s| s.limit.clone()) +} + +/// **The terse code never overwrites the detailed event** (Claude's live +/// order): a turn that first reports a rejected window WITH its reset, then +/// the CLI's terse `rate_limit` assistant code (window unknown, no reset) +/// ends Limited on the DETAILED classification — the later, windowless +/// limit merges into the earlier one instead of replacing it. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_terse_limit_code_does_not_overwrite_the_detailed_event() { + let (core, harness, prompts) = setup(LimitHarness::with_second_kind( + HarnessId::ClaudeCode, + Some(AgentErrorKind::UsageLimit { + window: "5-hour".into(), + resets_at: Some(1_762_675_400_000), + }), + // The terse code: window unknown ("usage"), no reset. + Some(AgentErrorKind::UsageLimit { + window: "usage".into(), + resets_at: None, + }), + )) + .await; + + queue_run(&core, "do the work", HarnessId::ClaudeCode, "m-open"); + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "do the work"), + "the limited run to start", + ) + .await; + let _ = harness.finish.send(()); + + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == SessionStatus::Errored && s.limit.is_some()) + }, + "the chat to go Limited", + ) + .await; + let limit = limit_row(&core).expect("the limit record"); + assert_eq!(limit.harness, HarnessId::ClaudeCode); + assert_eq!(limit.window, "5-hour", "the detailed window survives the terse code"); + assert_eq!( + limit.resets_at, + Some(1_762_675_400_000), + "the reported reset survives the terse code" + ); + core.shutdown().await; +} + +/// A run ending on a classified usage-limit error leaves the additive limit +/// record on the errored chat — harness, window and reset time exactly as +/// the harness's own error reported them — and mirrors it onto the workspace +/// status row so remote sidebars can show it too. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_limit_errored_run_records_the_limit_on_the_chat_status() { + let (core, harness, prompts) = setup(LimitHarness::new( + HarnessId::ClaudeCode, + Some(AgentErrorKind::UsageLimit { + window: "5-hour".into(), + resets_at: Some(1_762_675_400_000), + }), + )) + .await; + + queue_run(&core, "do the work", HarnessId::ClaudeCode, "m-open"); + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "do the work"), + "the limited run to start", + ) + .await; + let _ = harness.finish.send(()); + + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == SessionStatus::Errored) + }, + "the chat to settle errored", + ) + .await; + // The chat stays ERRORED on the wire (ADR 0005); the record is the + // additive Limited detail. + assert_eq!( + limit_row(&core), + Some(SessionLimit { + harness: HarnessId::ClaudeCode, + window: "5-hour".into(), + resets_at: Some(1_762_675_400_000), + account_id: None, + }) + ); + // No accounts on the engine's isolated store → no account on the record. + // The row rides the workspace doc too (remote devices' markers). + wait_for( + || { + core.workspace + .read_sessions() + .map(|rows| { + rows.iter().any(|row| { + row.chat_id == CHAT && row.status == SessionStatus::Errored + && row.limit.as_ref().is_some_and(|l| l.window == "5-hour") + }) + }) + .unwrap_or(false) + }, + "the limit row to mirror into the workspace doc", + ) + .await; + core.shutdown().await; +} + +/// A run that ends on an UNCLASSIFIED error leaves the chat errored with no +/// limit record — and its queued messages still flush at turn end (the +/// queue-hold is a Limited behavior, not a general error behavior). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_plain_error_records_no_limit_and_flushes_the_queue_as_before() { + let (core, harness, prompts) = + setup(LimitHarness::new(HarnessId::ClaudeCode, None)).await; + + queue_run(&core, "do the work", HarnessId::ClaudeCode, "m-open"); + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "do the work"), + "the failing run to start", + ) + .await; + core.doc_host + .queue_message(CHAT, "follow up", Vec::new()) + .expect("queue message"); + let _ = harness.finish.send(()); + + // Today's behavior, unchanged: the turn ended errored and the queued row + // goes out at once. The Errored status is only a blip before that next + // run starts, so the flush itself is the proof the errored turn ended. + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "follow up"), + "the queued message to flush after a plain error", + ) + .await; + assert_eq!(limit_row(&core), None, "unclassified errors are not Limited"); + let _ = harness.finish.send(()); + core.shutdown().await; +} + +/// The reset time the harness did not carry is filled from the harness +/// account's cached usage windows, and the record names that account: the +/// active login at run time, by its engine slot id. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_missing_reset_time_is_filled_from_the_accounts_cached_windows() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("accounts"); + let config = AgentAccountsConfig { + data_dir: root.join("data"), + claude_config_dir: root.join("claude"), + claude_config_file: root.join("claude.json"), + codex_home: root.join("codex"), + cursor_sdk_auth_file: root.join("cursor-sdk").join("auth.json"), + // File-only: a temp config must never reach the real Keychain login. + claude_keychain_service: None, + antigravity_home: Some(root.join("gemini")), + antigravity_keychain: false, + ..AgentAccountsConfig::isolated(&root) + }; + // A live Claude login (accountUuid uuid-limit-1 → usage key + // "claude-code:uuid-limit-1") with cached meters: the Session window is + // EXHAUSTED (1.0) so the account reads limited-until its reset; the Week + // window has usage left and a later reset — it must contribute nothing + // to the fill. + std::fs::create_dir_all(&config.claude_config_dir).unwrap(); + std::fs::write( + &config.claude_config_file, + serde_json::json!({ + "oauthAccount": { + "accountUuid": "uuid-limit-1", + "emailAddress": "dev@roboco.test", + "displayName": "Dev Tester", + "organizationName": "Test Org", + }, + "userID": "user-uuid-limit-1", + }) + .to_string(), + ) + .unwrap(); + std::fs::write( + config.claude_config_dir.join(".credentials.json"), + serde_json::json!({ + "claudeAiOauth": { + "accessToken": "token-limit-1", + "refreshToken": "refresh-limit-1", + "expiresAt": 4_102_444_800_000i64, + } + }) + .to_string(), + ) + .unwrap(); + let session_reset = chrono::DateTime::parse_from_rfc3339("2026-11-09T10:30:00Z") + .unwrap() + .to_utc(); + let cache = root.join("data").join("agent-accounts"); + std::fs::create_dir_all(&cache).unwrap(); + std::fs::write( + cache.join("usage-cache.json"), + serde_json::json!({ + "entries": { + "claude-code:uuid-limit-1": { + "usage": { + "windows": [ + { "label": "Session", "usedFraction": 1.0, "resetsAt": "2026-11-09T10:30:00Z" }, + { "label": "Week", "usedFraction": 0.4, "resetsAt": "2026-11-12T09:00:00Z" }, + ], + }, + "fetchedAt": 1_762_670_000_000i64, + "checkedAt": 1_762_670_000_000i64, + }, + }, + }) + .to_string(), + ) + .unwrap(); + let accounts = AgentAccounts::new(config); + // The account the limit "happened on": the active login's slot id. + let expected_account = accounts + .active_account(HarnessId::ClaudeCode) + .await + .expect("the fake login is the active account"); + + let (core, harness, prompts) = setup(LimitHarness::new( + HarnessId::ClaudeCode, + Some(AgentErrorKind::UsageLimit { + window: "usage".into(), + resets_at: None, + }), + )) + .await; + // Mount the SEEDED store the engine should read the fill from. + core.sessions.set_agent_accounts(accounts); + + queue_run(&core, "do the work", HarnessId::ClaudeCode, "m-open"); + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "do the work"), + "the limited run to start", + ) + .await; + let _ = harness.finish.send(()); + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == SessionStatus::Errored) + }, + "the chat to settle errored", + ) + .await; + assert_eq!( + limit_row(&core), + Some(SessionLimit { + harness: HarnessId::ClaudeCode, + window: "usage".into(), + // Filled from the account's limited_until: the exhausted + // Session window's reset, not the unexhausted Week's later one. + resets_at: Some(session_reset.timestamp_millis()), + account_id: Some(expected_account.id), + }), + "the reset time and account come from the harness account" + ); + core.shutdown().await; +} + +/// Resume sends the continue prompt as a NORMAL run — the composer's Run +/// command — which dispatches ahead of the queue the limited chat is +/// holding; the queued messages are delivered after the continuation, in +/// order; a new turn clears the limit record. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resume_runs_the_continue_prompt_with_the_queue_behind_it() { + let (core, harness, prompts) = setup(LimitHarness::new( + HarnessId::ClaudeCode, + Some(AgentErrorKind::UsageLimit { + window: "5-hour".into(), + resets_at: None, + }), + )) + .await; + + queue_run(&core, "do the work", HarnessId::ClaudeCode, "m-open"); + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "do the work"), + "the limited run to start", + ) + .await; + // Messages the user sent while the doomed run was working. + core.doc_host + .queue_message(CHAT, "queued one", Vec::new()) + .expect("queue one"); + core.doc_host + .queue_message(CHAT, "queued two", Vec::new()) + .expect("queue two"); + let _ = harness.finish.send(()); + + wait_for(|| limit_row(&core).is_some(), "the chat to go Limited").await; + // Limited: the queued rows WAIT instead of dispatching into the limit. + assert_eq!(queue_texts(&core), vec!["queued one", "queued two"]); + assert_eq!( + prompts.lock().unwrap().clone(), + vec!["do the work"], + "no queued message dispatches into the limited chat" + ); + + // Resume — the composer's normal-send path with the fixed prompt. + queue_run( + &core, + roboco_proto::RESUME_CONTINUE_PROMPT, + HarnessId::ClaudeCode, + "m-resume", + ); + wait_for( + || { + prompts + .lock() + .unwrap() + .iter() + .any(|p| p == roboco_proto::RESUME_CONTINUE_PROMPT) + }, + "the continue prompt to run ahead of the queue", + ) + .await; + // The continuation owns the turn; the queued rows still wait behind it. + assert_eq!(queue_texts(&core), vec!["queued one", "queued two"]); + // A new turn started: the limit record is gone. + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.limit.is_none()) + }, + "the new turn to clear the limit record", + ) + .await; + + // The continuation ends → the queue drains behind it, head first. + let _ = harness.finish.send(()); + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "queued one"), + "the first queued message to follow the continuation", + ) + .await; + let _ = harness.finish.send(()); + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "queued two"), + "the second queued message to follow in order", + ) + .await; + let _ = harness.finish.send(()); + + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == SessionStatus::Idle) + }, + "the last turn to settle", + ) + .await; + let sent: Vec = prompts.lock().unwrap().clone(); + assert_eq!( + sent, + vec![ + "do the work", + roboco_proto::RESUME_CONTINUE_PROMPT, + "queued one", + "queued two", + ], + "resume runs first, queued messages follow in order" + ); + assert!(queue_texts(&core).is_empty(), "the queue drained"); + // The transcript carries the continue prompt as the run's user entry. + assert!(user_texts(&core).contains(&roboco_proto::RESUME_CONTINUE_PROMPT.to_string())); + assert_eq!( + limit_row(&core), + None, + "the limit record stayed cleared after the resumed turns" + ); + core.shutdown().await; +} + +/// ANY new turn clears the record, not just a Resume: an ordinary follow-up +/// message starts a fresh turn and the chat stops reading as Limited. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_plain_new_turn_clears_the_limit_record() { + let (core, harness, prompts) = setup(LimitHarness::new( + HarnessId::ClaudeCode, + Some(AgentErrorKind::UsageLimit { + window: "5-hour".into(), + resets_at: None, + }), + )) + .await; + + queue_run(&core, "do the work", HarnessId::ClaudeCode, "m-open"); + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "do the work"), + "the limited run to start", + ) + .await; + let _ = harness.finish.send(()); + wait_for(|| limit_row(&core).is_some(), "the chat to go Limited").await; + + // The user's own next message (a normal send, not the resume prompt). + queue_run(&core, "try a different angle", HarnessId::ClaudeCode, "m-next"); + // Wait for the harness to hold the run, not just for Working: the + // status publishes before run() subscribes to `finish`, so a send in + // that window would be lost and the turn would never settle. + wait_for( + || prompts.lock().unwrap().iter().any(|p| p == "try a different angle"), + "the new turn to start", + ) + .await; + assert_eq!( + limit_row(&core), + None, + "the limit record clears when the new turn starts" + ); + let _ = harness.finish.send(()); + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == SessionStatus::Idle) + }, + "the new turn to settle", + ) + .await; + assert_eq!(limit_row(&core), None); + core.shutdown().await; +} diff --git a/crates/engine/tests/m5c_accounts_uploads_titles.rs b/crates/engine/tests/m5c_accounts_uploads_titles.rs index 4781895067..6fa738c273 100644 --- a/crates/engine/tests/m5c_accounts_uploads_titles.rs +++ b/crates/engine/tests/m5c_accounts_uploads_titles.rs @@ -967,9 +967,11 @@ async fn titling_e2e_names_chat_and_renames_worktree_branch() { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }; core.sessions - .dispatch(chat_id, HarnessId::Mock, request, None) + .dispatch(chat_id, HarnessId::Mock, request, None, None, None) .await .expect("dispatch"); @@ -1013,9 +1015,11 @@ async fn titling_e2e_names_chat_and_renames_worktree_branch() { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }; core.sessions - .dispatch(chat_id, HarnessId::Mock, request, None) + .dispatch(chat_id, HarnessId::Mock, request, None, None, None) .await .expect("second dispatch"); tokio::time::sleep(Duration::from_millis(400)).await; diff --git a/crates/engine/tests/message_queue.rs b/crates/engine/tests/message_queue.rs index dee0ae691a..bc67df5ef0 100644 --- a/crates/engine/tests/message_queue.rs +++ b/crates/engine/tests/message_queue.rs @@ -592,6 +592,7 @@ async fn batched_remote_steers_preserve_every_message_in_order_exactly_once() { handle .doc() .queue_command(&roboco_doc::SessionCommandEntry { + origin_chat_id: None, id: format!("remote-command-{i}"), payload: SessionCommandPayload::Steer { prompt: prompt.clone(), @@ -655,7 +656,7 @@ async fn queued_text_waits_for_a_steerable_turn_even_with_legacy_policy() { .queue_message(CHAT, "first queued", Vec::new()) .unwrap(); core.doc_host - .queue_message_with_behavior(CHAT, "second queued", Vec::new(), false) + .queue_message_with_behavior(CHAT, "second queued", Vec::new(), false, None) .unwrap(); tokio::time::sleep(Duration::from_millis(150)).await; assert_eq!(queue_texts(&core), vec!["first queued", "second queued"]); @@ -1147,7 +1148,7 @@ async fn explicit_queue_delivery_is_rejected_off_host_without_taking_the_row() { .expect("move chat host"); let id = core .doc_host - .queue_message_with_behavior(CHAT, "remote-only", Vec::new(), true) + .queue_message_with_behavior(CHAT, "remote-only", Vec::new(), true, None) .expect("queue held row"); let send_error = core diff --git a/crates/engine/tests/pi_forks.rs b/crates/engine/tests/pi_forks.rs new file mode 100644 index 0000000000..4de585c148 --- /dev/null +++ b/crates/engine/tests/pi_forks.rs @@ -0,0 +1,348 @@ +//! Pi native forks through the REAL engine surfaces (t3-orchestration-v2 +//! ticket 19): forking a Pi chat continues the source's real session. The +//! source turn's driver stamps a provider turn ref on the assistant entry +//! (the session file plus the turn's user entry id, `get_entries` after the +//! settle), the ForkSideChat RPC records the native-fork intent from it, +//! and the fork's first run performs the `pi --fork` launch plus the +//! exclusive fork RPC through the fixture's fork process — nothing is +//! pasted. A boundary the session tree no longer holds falls back to the +//! budgeted paste. Prior art: `pi_resume.rs` / `pi_steering.rs` drive the +//! real `PiHarness` against the fixture RPC script; `side_chats.rs` covers +//! the harness-agnostic fork RPC plumbing with the mock harness. +#![cfg(unix)] + +use std::{path::PathBuf, sync::Arc, time::Duration}; + +use roboco_doc::{MessagePart, MessageRole, SessionMessageEntry}; +use roboco_engine::{EngineCore, HarnessRegistry}; +use roboco_harness::PiHarness; +use roboco_proto::{HarnessId, SessionStatus}; + +const SRC: &str = "pi-fork-src"; +const SIDE: &str = "pi-fork-side"; + +fn fixture_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../harness/tests/fixtures/fake-pi-rpc.sh") +} + +/// Assemble a full engine around the REAL Pi harness pointed at the fixture +/// RPC process (short interrupt graces so a stuck scenario fails fast +/// instead of hanging the suite). +fn assemble(dir: &std::path::Path) -> EngineCore { + let registry = HarnessRegistry::new(); + registry.register(Arc::new( + PiHarness::new() + .with_executable(fixture_path()) + .with_graces(Duration::from_millis(150), Duration::from_millis(300)), + )); + EngineCore::assemble(dir, Arc::new(registry), HarnessId::Pi).expect("engine core assembles") +} + +/// A pi chat in a real temp cwd (the fixture writes its session file and +/// runs there), pre-titled so the auto-titler never spawns a fixture run of +/// its own. +async fn create_chat(core: &EngineCore, chat: &str, cwd: &str) { + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": chat, + "deviceId": core.device_id, + "cwd": cwd, + "config": { + "harness": "pi", + "model": null, + "reasoning": null, + "sandbox": "workspace-write", + }, + }), + ) + .await + .expect("createChat"); + core.workspace + .rename_chat(chat, "Pre-titled") + .expect("rename chat"); +} + +fn texts_of(core: &EngineCore, chat: &str, role: MessageRole) -> Vec { + core.doc_host + .open(chat) + .expect("chat doc") + .doc() + .read_entries() + .unwrap_or_default() + .into_iter() + .filter(|entry| entry.role == role) + .map(|entry| { + entry + .parts + .iter() + .filter_map(|part| match part { + MessagePart::Text { text, .. } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("") + }) + .collect() +} + +fn error_texts_of(core: &EngineCore, chat: &str) -> Vec { + core.doc_host + .open(chat) + .expect("chat doc") + .doc() + .read_entries() + .unwrap_or_default() + .into_iter() + .flat_map(|entry| entry.parts) + .filter_map(|part| match part { + MessagePart::Error { message, .. } => Some(message), + _ => None, + }) + .collect() +} + +async fn wait_for(mut predicate: F, what: &str) +where + F: FnMut() -> bool, +{ + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while !predicate() { + assert!( + tokio::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +async fn wait_idle(core: &EngineCore, chat: &str) { + wait_for( + || { + core.sessions + .session_status(chat) + .map(|s| s.status) + .is_some_and(|status| status == SessionStatus::Idle) + }, + "the turn to settle", + ) + .await; +} + +async fn fork_side_chat(core: &EngineCore, side: &str, source: &str) { + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + roboco_rpc::methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": side, "sourceChatId": source }), + ) + .await + .expect("the fork RPC succeeded"); +} + +/// A hand-built message entry (the fallback test writes a source doc the +/// driver never ran). +fn message(id: &str, role: MessageRole, text: &str) -> SessionMessageEntry { + SessionMessageEntry { + origin_chat_id: None, + duration_ms: None, + provider_turn: None, + id: id.into(), + role, + parts: vec![MessagePart::Text { + id: format!("{id}-text"), + text: text.into(), + }], + created_at: 1, + device_id: "device".into(), + status: Some(roboco_doc::MessageStatus::Complete), + continuation_of: None, + } +} + +/// Forking a Pi chat continues the source's REAL session: the source +/// turn's ref drives the fork, the fork process launches with `--fork` and +/// sends the exclusive fork RPC at the turn after the boundary (the +/// fixture asserts both through markers, plus that the main run resumes +/// the FORKED session file with a BARE prompt — a pasted history would +/// fail its scenario), and the forked chat's own turns then run normally +/// on the forked session. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_pi_fork_continues_the_source_session_and_pastes_nothing() { + let dir = tempfile::tempdir().unwrap(); + let cwd = dir.path().join("chat"); + std::fs::create_dir_all(&cwd).unwrap(); + let core = assemble(&dir.path().join("data")); + create_chat(&core, SRC, &cwd.display().to_string()).await; + + // The source turn: the driver stamps the boundary ref on the reply. + core.doc_host + .queue_message(SRC, "hello", Vec::new()) + .expect("queue the source turn"); + wait_for( + || { + texts_of(&core, SRC, MessageRole::Assistant) + .iter() + .any(|t| t.contains("reply:hello")) + }, + "the source reply to stream", + ) + .await; + wait_idle(&core, SRC).await; + let reference = core + .doc_host + .open(SRC) + .expect("chat doc") + .doc() + .read_entries() + .expect("entries") + .into_iter() + .find(|entry| entry.role == MessageRole::Assistant) + .and_then(|entry| entry.provider_turn) + .expect("the driver stamped a turn ref on the reply"); + assert_eq!(reference.harness, HarnessId::Pi); + assert!(std::path::Path::new(&reference.session_id).is_file(), "{reference:?}"); + assert_eq!(reference.turn_id, "u1"); + + // Fork it, then run the fork's first turn. + fork_side_chat(&core, SIDE, SRC).await; + core.doc_host + .queue_message(SIDE, "fork-turn", Vec::new()) + .expect("queue the fork turn"); + wait_for( + || { + texts_of(&core, SIDE, MessageRole::Assistant) + .iter() + .any(|t| t.contains("reply:fork-turn")) + }, + "the forked reply to stream", + ) + .await; + wait_idle(&core, SIDE).await; + + // The fork process ran with --fork and sent the RPC fork at the turn + // AFTER the boundary (the source's first turn): the markers prove both. + let forked_path = cwd.join(".pi-fixture-forked.jsonl"); + let forked = std::fs::read_to_string(&forked_path).expect("the forked session file"); + assert!(forked.contains("fork-launch:"), "{forked}"); + assert!(forked.contains("fork-rpc:u2"), "{forked}"); + + // The copied boundary keeps its ref, and the fork's own reply carries a + // ref naming the FORKED session (its own process mints fresh entry ids). + let side_entries = core + .doc_host + .open(SIDE) + .expect("chat doc") + .doc() + .read_entries() + .expect("entries"); + assert!( + side_entries + .iter() + .any(|entry| entry.provider_turn.as_ref() == Some(&reference)), + "the copied boundary keeps its ref: {side_entries:?}" + ); + let own = side_entries + .into_iter() + .find(|entry| { + entry.role == MessageRole::Assistant + && entry + .provider_turn + .as_ref() + .is_some_and(|r| r.session_id.ends_with(".pi-fixture-forked.jsonl")) + }) + .expect("the forked turn stamped its own ref") + .provider_turn + .expect("ref"); + assert_eq!(own.turn_id, "u1"); + + // The intent is spent: the next turn is a normal turn on the forked + // session — no second fork launch (the marker count proves it). + core.doc_host + .queue_message(SIDE, "fork-turn", Vec::new()) + .expect("queue the second fork turn"); + wait_for( + || { + texts_of(&core, SIDE, MessageRole::Assistant) + .iter() + .filter(|t| t.as_str() == "reply:fork-turn") + .count() + == 2 + }, + "the second forked turn to stream", + ) + .await; + wait_idle(&core, SIDE).await; + let forked = std::fs::read_to_string(&forked_path).expect("the forked session file"); + assert_eq!(forked.matches("fork-launch:").count(), 1, "{forked}"); + assert_eq!(forked.matches("fork-rpc:").count(), 1, "{forked}"); + core.shutdown().await; +} + +/// A fork whose boundary ref names no entry the source session holds (a +/// stale boundary, an old chat or a re-rooted branch) never re-roots: the +/// fork falls back to a fresh session with the budgeted transcript in +/// front of the prompt (the fixture asserts the pasted history and refuses +/// a resume), the refusal surfaces as an error chip, and the turn still +/// completes. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_pi_fork_without_a_usable_boundary_falls_back_to_the_budgeted_paste() { + let dir = tempfile::tempdir().unwrap(); + let cwd = dir.path().join("chat"); + std::fs::create_dir_all(&cwd).unwrap(); + let core = assemble(&dir.path().join("data")); + create_chat(&core, SRC, &cwd.display().to_string()).await; + + // A source doc the driver never ran: the boundary entry carries a ref + // whose turn id the fixture session tree does not hold. + let stale = cwd.join("stale-session.jsonl"); + std::fs::write(&stale, b"fixture session\n").unwrap(); + let source = core.doc_host.open(SRC).expect("chat doc"); + source + .doc() + .push_message(&message("u1", MessageRole::User, "Remember PINEAPPLE")) + .expect("user entry"); + let mut boundary = message("a1", MessageRole::Assistant, "I remember PINEAPPLE"); + boundary.provider_turn = Some(roboco_proto::ProviderTurnRef { + harness: HarnessId::Pi, + session_id: stale.display().to_string(), + turn_id: "u-gone".into(), + }); + source.doc().push_message(&boundary).expect("boundary"); + + fork_side_chat(&core, SIDE, SRC).await; + core.doc_host + .queue_message(SIDE, "fork-fallback", Vec::new()) + .expect("queue the fork turn"); + wait_for( + || { + texts_of(&core, SIDE, MessageRole::Assistant) + .iter() + .any(|t| t.contains("reply:fork-fallback")) + }, + "the fallback reply to stream", + ) + .await; + wait_idle(&core, SIDE).await; + + // The fork process DID run (the launch marker) but never re-rooted (no + // fork RPC marker): the stale boundary refused the fork. + let forked_path = cwd.join(".pi-fixture-forked.jsonl"); + let forked = std::fs::read_to_string(&forked_path).expect("the forked session file"); + assert!(forked.contains("fork-launch:"), "{forked}"); + assert!(!forked.contains("fork-rpc:"), "{forked}"); + + // The refusal surfaced as an error chip on the forked chat. + assert!( + error_texts_of(&core, SIDE) + .iter() + .any(|message| message.contains("pi could not fork")), + "the fork refusal surfaced: {:?}", + error_texts_of(&core, SIDE) + ); + core.shutdown().await; +} diff --git a/crates/engine/tests/pi_resume.rs b/crates/engine/tests/pi_resume.rs index 3772d43bc3..3d49568a18 100644 --- a/crates/engine/tests/pi_resume.rs +++ b/crates/engine/tests/pi_resume.rs @@ -36,9 +36,11 @@ async fn pi_idle_crash_next_dispatch_resumes_the_stored_session() { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }; core.sessions - .dispatch(chat, HarnessId::Pi, req, None) + .dispatch(chat, HarnessId::Pi, req, None, None, None) .await .unwrap(); tokio::time::timeout(Duration::from_secs(10), async { diff --git a/crates/engine/tests/pi_steering.rs b/crates/engine/tests/pi_steering.rs new file mode 100644 index 0000000000..2b34d79cb2 --- /dev/null +++ b/crates/engine/tests/pi_steering.rs @@ -0,0 +1,377 @@ +//! Pi steers mid-turn through the REAL engine surfaces (orchestration v2, +//! ticket 03): a message sent to a working Pi chat reaches the fake pi at +//! its next step boundary — steered into the live run, never held to turn +//! end — and a Stop with steers pi never consumed cancels them exactly like +//! any other held send. Prior art: `pi_resume.rs` drives the real +//! `PiHarness` against the fixture RPC script through engine dispatch; +//! `message_queue.rs` drives the queue + steer surfaces. The fixture's +//! `steer-flow` scenario is the ordering oracle: it cannot settle before +//! the steer arrives, so a message held to turn end would hang the run. +#![cfg(unix)] + +use std::{path::PathBuf, sync::Arc, time::Duration}; + +use roboco_doc::{MessagePart, MessageRole, SessionCommandPayload, SessionMessageEntry}; +use roboco_engine::{EngineCore, HarnessRegistry}; +use roboco_harness::PiHarness; +use roboco_proto::{HarnessId, SessionStatus}; + +const CHAT: &str = "pi-steer"; + +fn fixture_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../harness/tests/fixtures/fake-pi-rpc.sh") +} + +/// Assemble a full engine around the REAL Pi harness pointed at the fixture +/// RPC process (short interrupt graces so a stuck scenario fails fast +/// instead of hanging the suite). +fn assemble(dir: &std::path::Path, agent_dir: &std::path::Path) -> EngineCore { + let registry = HarnessRegistry::new(); + registry.register(Arc::new( + PiHarness::new() + .with_executable(fixture_path()) + .with_agent_dir(agent_dir) + .with_graces(Duration::from_millis(150), Duration::from_millis(300)), + )); + EngineCore::assemble(dir, Arc::new(registry), HarnessId::Pi).expect("engine core assembles") +} + +/// A pi chat in a real temp cwd (the fixture writes its session file and +/// runs there), pre-titled so the auto-titler never spawns a fixture run of +/// its own. +async fn create_pi_chat(core: &EngineCore, cwd: &str) { + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + roboco_rpc::methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": CHAT, + "deviceId": core.device_id, + "cwd": cwd, + "config": { + "harness": "pi", + "model": null, + "reasoning": null, + "sandbox": "workspace-write", + }, + }), + ) + .await + .expect("createChat"); + core.workspace.rename_chat(CHAT, "Pre-titled").expect("rename chat"); +} + +fn entries(core: &EngineCore) -> Vec { + core.doc_host + .open(CHAT) + .expect("chat doc") + .doc() + .read_entries() + .unwrap_or_default() +} + +fn texts_of(core: &EngineCore, role: MessageRole) -> Vec { + entries(core) + .iter() + .filter(|e| e.role == role) + .map(|e| { + e.parts + .iter() + .filter_map(|p| match p { + MessagePart::Text { text, .. } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("") + }) + .collect() +} + +fn queue_texts(core: &EngineCore) -> Vec { + core.doc_host + .open(CHAT) + .expect("chat doc") + .doc() + .read_queue() + .unwrap_or_default() + .into_iter() + .map(|row| row.text) + .collect() +} + +async fn wait_for(mut predicate: F, what: &str) +where + F: FnMut() -> bool, +{ + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while !predicate() { + assert!( + tokio::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +/// A message sent to a working Pi chat from the composer's queue row (the +/// "Steer" action) reaches pi at its next step boundary: the user message +/// lands in the transcript and the row leaves the queue WHILE the turn is +/// still in flight, and the message folds into that turn — it never waits +/// for the run to end and never starts a follow-up turn of its own. The +/// fixture's `steer-flow` script structurally guarantees the wire order +/// (it reads the steer before it settles); a held message would deadlock +/// the scenario and this test would time out instead. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_message_sent_mid_run_is_steered_not_held_to_turn_end() { + let dir = tempfile::tempdir().unwrap(); + let agent_dir = tempfile::tempdir().unwrap(); + let cwd = dir.path().join("chat"); + std::fs::create_dir_all(&cwd).unwrap(); + let core = assemble(&dir.path().join("data"), agent_dir.path()); + create_pi_chat(&core, &cwd.display().to_string()).await; + + // The opening turn: the fixture streams its first segment and then + // waits for the steer — the window where the message must go IN. + core.doc_host + .queue_message(CHAT, "steer-flow", Vec::new()) + .expect("queue opening"); + wait_for( + || { + texts_of(&core, MessageRole::Assistant) + .iter() + .any(|t| t.contains("first part ")) + }, + "the opening turn to stream", + ) + .await; + assert!(core.sessions.turn_in_flight(CHAT)); + + // The composer path while busy: the row lands in the queue, then its + // primary action — Steer, because pi is a step-boundary harness — + // routes it into the live run. + let row = core + .doc_host + .queue_message(CHAT, "steered text", Vec::new()) + .expect("queue the mid-run message"); + assert_eq!(queue_texts(&core), vec!["steered text"]); + let sent = core.doc_host.steer_queued_now(CHAT, &row).await.expect("steer now"); + assert!(sent); + + // The message reached the transcript, the row left the queue, and the + // turn is STILL in flight (the fixture pauses after the queued ack, so + // this window is wide): steered, not held to turn end. + wait_for( + || { + texts_of(&core, MessageRole::User).iter().any(|m| m == "steered text") + && queue_texts(&core).is_empty() + }, + "the steered message to land in the transcript", + ) + .await; + assert!( + core.sessions.turn_in_flight(CHAT), + "the steer must land while the turn is still working" + ); + assert!( + !texts_of(&core, MessageRole::Assistant) + .iter() + .any(|t| t.contains("reply-to-steer")), + "the run cannot have answered the steer before accepting it" + ); + + // The whole exchange stays one turn: the steer folds into the live run + // and pi answers it there (never a fresh follow-up turn, which the + // fixture would answer as its own reply). + wait_for( + || { + texts_of(&core, MessageRole::Assistant) + .iter() + .any(|t| t.contains("reply-to-steer")) + }, + "the steered reply to stream", + ) + .await; + wait_for( + || !core.sessions.turn_in_flight(CHAT), + "the turn to settle", + ) + .await; + assert_eq!( + texts_of(&core, MessageRole::User), + vec!["steer-flow", "steered text"] + ); + assert!( + !texts_of(&core, MessageRole::Assistant) + .iter() + .any(|t| t.contains("reply:steered text")), + "the steered message must not re-run as its own turn" + ); + assert_eq!(queue_texts(&core), Vec::::new()); + assert_eq!( + core.sessions.session_status(CHAT).map(|s| s.status), + Some(SessionStatus::Idle) + ); + core.shutdown().await; +} + +/// The MCP `send_message` auto-mode route: a `Steer` command queued while +/// pi works goes through the command log and `deliver_prompt`, which — for +/// a step-boundary harness — routes it into the live run instead of holding +/// it to turn end (the pre-flip behavior this ticket replaces). Same +/// fixture ordering oracle: the scenario settles only after the steer. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_mcp_steer_command_reaches_the_live_run() { + let dir = tempfile::tempdir().unwrap(); + let agent_dir = tempfile::tempdir().unwrap(); + let cwd = dir.path().join("chat"); + std::fs::create_dir_all(&cwd).unwrap(); + let core = assemble(&dir.path().join("data"), agent_dir.path()); + create_pi_chat(&core, &cwd.display().to_string()).await; + + core.doc_host + .queue_message(CHAT, "steer-flow", Vec::new()) + .expect("queue opening"); + wait_for( + || { + texts_of(&core, MessageRole::Assistant) + .iter() + .any(|t| t.contains("first part ")) + }, + "the opening turn to stream", + ) + .await; + + // The exact durable command the MCP server queues for a working chat + // (send_message auto mode resolves to a Steer command). + core.doc_host + .queue_command( + CHAT, + SessionCommandPayload::Steer { + prompt: "steered text".into(), + message_id: Some("mcp-steer-1".into()), + }, + ) + .expect("queue the steer command"); + + // Steered into the live run: the message lands in the transcript while + // the turn is still working, and never sits in the queue as a held row. + wait_for( + || { + texts_of(&core, MessageRole::User).iter().any(|m| m == "steered text") + && queue_texts(&core).is_empty() + }, + "the steered message to land in the transcript", + ) + .await; + assert!(core.sessions.turn_in_flight(CHAT)); + + wait_for( + || { + texts_of(&core, MessageRole::Assistant) + .iter() + .any(|t| t.contains("reply-to-steer")) + }, + "the steered reply to stream", + ) + .await; + wait_for( + || !core.sessions.turn_in_flight(CHAT), + "the turn to settle", + ) + .await; + assert_eq!( + texts_of(&core, MessageRole::User), + vec!["steer-flow", "steered text"] + ); + assert!(!texts_of(&core, MessageRole::Assistant) + .iter() + .any(|t| t.contains("reply:steered text"))); + core.shutdown().await; +} + +/// Stop with steers pi never consumed: the driver clears pi queue before +/// the abort (the fixture asserts the wire order and the stranded text), +/// the turn ends interrupted, and the message is handled exactly like a +/// held send cancelled by Stop — it stays in the transcript, unrun, and no +/// fresh runtime picks it back up. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stop_cancels_unconsumed_steers_like_any_held_send() { + let dir = tempfile::tempdir().unwrap(); + let agent_dir = tempfile::tempdir().unwrap(); + let cwd = dir.path().join("chat"); + std::fs::create_dir_all(&cwd).unwrap(); + let core = assemble(&dir.path().join("data"), agent_dir.path()); + create_pi_chat(&core, &cwd.display().to_string()).await; + + core.doc_host + .queue_message(CHAT, "stop-clears-queue", Vec::new()) + .expect("queue opening"); + wait_for( + || { + texts_of(&core, MessageRole::Assistant) + .iter() + .any(|t| t.contains("working")) + }, + "the opening turn to stream", + ) + .await; + + // The steer is accepted into pi queue but never consumed (the fixture + // streams its marker and waits for the Stop). + let row = core + .doc_host + .queue_message(CHAT, "stranded steer", Vec::new()) + .expect("queue the steer"); + let sent = core.doc_host.steer_queued_now(CHAT, &row).await.expect("steer now"); + assert!(sent); + wait_for( + || { + texts_of(&core, MessageRole::Assistant) + .iter() + .any(|t| t.contains("steer queued")) + }, + "pi to accept the steer into its queue", + ) + .await; + + core.doc_host + .queue_command(CHAT, SessionCommandPayload::Interrupt {}) + .expect("queue interrupt"); + wait_for( + || !core.sessions.turn_in_flight(CHAT), + "the interrupted turn to settle", + ) + .await; + + // The message was sent — it is in the transcript, like every held send + // Stop cancels — and it never runs: no reply, no re-dispatch, no new + // turn. The fixture asserts clear_queue carried the stranded text away + // BEFORE the abort. + wait_for( + || texts_of(&core, MessageRole::User).iter().any(|m| m == "stranded steer"), + "the cancelled message to stay in the transcript", + ) + .await; + assert!( + !texts_of(&core, MessageRole::Assistant) + .iter() + .any(|t| t.contains("reply:")), + "the cancelled steer must never run" + ); + tokio::time::sleep(Duration::from_millis(400)).await; + assert!( + !texts_of(&core, MessageRole::Assistant) + .iter() + .any(|t| t.contains("reply:")), + "no fresh runtime may pick the cancelled steer back up" + ); + assert!(!core.sessions.turn_in_flight(CHAT)); + assert_eq!( + core.sessions.session_status(CHAT).map(|s| s.status), + Some(SessionStatus::Idle) + ); + assert_eq!(queue_texts(&core), Vec::::new()); + core.shutdown().await; +} diff --git a/crates/engine/tests/projectless.rs b/crates/engine/tests/projectless.rs index 5cd755cf16..1cf4a547e9 100644 --- a/crates/engine/tests/projectless.rs +++ b/crates/engine/tests/projectless.rs @@ -293,6 +293,8 @@ async fn exercise_projectless(command_first: bool) { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }, message_id: message_id.into(), }, diff --git a/crates/engine/tests/queued_attachments.rs b/crates/engine/tests/queued_attachments.rs index cec1131141..f36a6774af 100644 --- a/crates/engine/tests/queued_attachments.rs +++ b/crates/engine/tests/queued_attachments.rs @@ -132,6 +132,8 @@ fn run_payload(message_id: &str, pending_ref: &str) -> SessionCommandPayload { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }, message_id: message_id.into(), } @@ -274,7 +276,7 @@ async fn queued_row_waits_for_attachment_bytes_then_sends_resolved_paths() { // have not landed yet: the idle chat must NOT send it. let pending_ref = "pending://att-q1/queued shot.png"; core.doc_host - .queue_message_with_behavior(CHAT, "what is this", vec![pending_ref.into()], false) + .queue_message_with_behavior(CHAT, "what is this", vec![pending_ref.into()], false, None) .expect("queue row"); tokio::time::sleep(Duration::from_millis(400)).await; assert_eq!( diff --git a/crates/engine/tests/restart_resume.rs b/crates/engine/tests/restart_resume.rs index 76114d2c4c..d97d9e62fc 100644 --- a/crates/engine/tests/restart_resume.rs +++ b/crates/engine/tests/restart_resume.rs @@ -11,7 +11,11 @@ //! - a startup crash retries once with the resume kept, and a helper that is //! down hard never tombstones the stored session id; //! - a steer with no live run after a restart dispatches as a new turn that -//! still resumes the prior conversation. +//! still resumes the prior conversation; +//! - a chat auto-resumed at boot re-delivers only input its harness had not +//! yet consumed — a consumed steer never replays (it revives the turn with +//! the continue prompt instead), while an accepted-but-undelivered steer +//! is still delivered, exactly once. use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -49,6 +53,8 @@ fn run_request(prompt: &str, cwd: &str) -> RunRequest { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } @@ -334,6 +340,7 @@ async fn kill_crash_recovers_resume_from_journal_and_stamps_aborted() { let store = DocsStore::open(dir.join("orgs/dev-org/dev-user")).unwrap(); let doc = SessionDoc::init(CHAT).unwrap(); doc.push_message(&SessionMessageEntry { + origin_chat_id: None, id: "msg-user-1".into(), role: MessageRole::User, parts: vec![MessagePart::Text { @@ -344,10 +351,12 @@ async fn kill_crash_recovers_resume_from_journal_and_stamps_aborted() { device_id: "dev-crash".into(), status: Some(MessageStatus::Complete), continuation_of: None, + provider_turn: None, duration_ms: None, }) .unwrap(); doc.push_message(&SessionMessageEntry { + origin_chat_id: None, id: "msg-assistant-1".into(), role: MessageRole::Assistant, parts: vec![MessagePart::Text { @@ -358,6 +367,7 @@ async fn kill_crash_recovers_resume_from_journal_and_stamps_aborted() { device_id: "dev-crash".into(), status: Some(MessageStatus::Streaming), continuation_of: None, + provider_turn: None, duration_ms: None, }) .unwrap(); @@ -582,6 +592,7 @@ async fn fresh_crash_auto_resumes_and_notes_the_interruption() { let store = DocsStore::open(dir.join("orgs/dev-org/dev-user")).unwrap(); let doc = SessionDoc::init(CHAT).unwrap(); doc.push_message(&SessionMessageEntry { + origin_chat_id: None, id: "msg-user-1".into(), role: MessageRole::User, parts: vec![MessagePart::Text { @@ -592,10 +603,12 @@ async fn fresh_crash_auto_resumes_and_notes_the_interruption() { device_id: "dev-crash".into(), status: Some(MessageStatus::Complete), continuation_of: None, + provider_turn: None, duration_ms: None, }) .unwrap(); doc.push_message(&SessionMessageEntry { + origin_chat_id: None, id: "msg-assistant-1".into(), role: MessageRole::Assistant, parts: vec![MessagePart::Text { @@ -606,6 +619,7 @@ async fn fresh_crash_auto_resumes_and_notes_the_interruption() { device_id: "dev-crash".into(), status: Some(MessageStatus::Streaming), continuation_of: None, + provider_turn: None, duration_ms: None, }) .unwrap(); @@ -856,6 +870,8 @@ async fn real_claude_remembers_codeword_across_engine_restart() { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }; let assemble_real = || { EngineCore::assemble( @@ -998,6 +1014,7 @@ async fn a_crash_before_the_reply_reached_disk_still_auto_resumes() { let store = DocsStore::open(dir.join("orgs/dev-org/dev-user")).unwrap(); let doc = SessionDoc::init(CHAT).unwrap(); doc.push_message(&SessionMessageEntry { + origin_chat_id: None, id: "msg-user-1".into(), role: MessageRole::User, parts: vec![MessagePart::Text { @@ -1008,6 +1025,7 @@ async fn a_crash_before_the_reply_reached_disk_still_auto_resumes() { device_id: "dev-crash".into(), status: Some(MessageStatus::Complete), continuation_of: None, + provider_turn: None, duration_ms: None, }) .unwrap(); @@ -1098,6 +1116,7 @@ async fn a_crash_recovers_the_conversation_through_a_symlinked_folder() { let store = DocsStore::open(dir.join("orgs/dev-org/dev-user")).unwrap(); let doc = SessionDoc::init(CHAT).unwrap(); doc.push_message(&SessionMessageEntry { + origin_chat_id: None, id: "msg-user-1".into(), role: MessageRole::User, parts: vec![MessagePart::Text { @@ -1108,6 +1127,7 @@ async fn a_crash_recovers_the_conversation_through_a_symlinked_folder() { device_id: "dev-crash".into(), status: Some(MessageStatus::Complete), continuation_of: None, + provider_turn: None, duration_ms: None, }) .unwrap(); @@ -1164,3 +1184,428 @@ async fn a_crash_recovers_the_conversation_through_a_symlinked_folder() { ); core.shutdown().await; } + +// ── Restart correctness: no steer replay after a restart (ticket 04) ──────── +// +// The user's incident (orchestrator chat ff441279, 2026-10-15): a parent +// steered the working orchestrator mid-run, the harness consumed the steer, +// the engine restarted, and boot auto-resume re-sent the steer as the +// revival prompt — the model saw the same instruction twice and answered it +// twice ("Acknowledged again (my session restarted)"). A chat auto-resumed +// at boot must re-deliver only input its harness had not yet consumed. + +/// The steer the orchestrator's parent sent mid-run. +const STEER_TEXT: &str = "actually, use glm-5.3 everywhere"; + +/// Manufacture the on-disk state a kill -9 leaves behind when a run was +/// steered mid-turn: the starting message and its partial reply, the steer +/// (its user entry was written at acceptance), and — when `consumed` — the +/// reply that streamed after the harness confirmed consuming the steer. +/// +/// `consumed = true` also journals the `Steered` boundary and records the +/// steer's consumption durably, exactly the way the live engine does when +/// the harness confirms delivery: that record is what lets boot recovery +/// tell a consumed steer from one still sitting in a dead mailbox. +fn manufacture_crashed_steer_turn(dir: &std::path::Path, consumed: bool) { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + fn entry( + id: &str, + role: MessageRole, + text: &str, + created_at: i64, + status: Option, + ) -> SessionMessageEntry { + SessionMessageEntry { + origin_chat_id: None, + id: id.into(), + role, + parts: vec![MessagePart::Text { + id: "t0".into(), + text: text.into(), + }], + created_at, + device_id: "dev-crash".into(), + status, + continuation_of: None, + provider_turn: None, + duration_ms: None, + } + }; + { + let store = DocsStore::open(dir.join("orgs/dev-org/dev-user")).unwrap(); + let doc = SessionDoc::init(CHAT).unwrap(); + doc.push_message(&entry( + "msg-user-1", + MessageRole::User, + "long task", + now - 60_000, + Some(MessageStatus::Complete), + )) + .unwrap(); + doc.push_message(&entry( + "msg-assistant-1", + MessageRole::Assistant, + "partial…", + now - 30_000, + Some(MessageStatus::Streaming), + )) + .unwrap(); + doc.push_message(&entry( + "msg-user-2", + MessageRole::User, + STEER_TEXT, + now - 20_000, + Some(MessageStatus::Complete), + )) + .unwrap(); + if consumed { + // The harness consumed the steer and the model kept working — + // the reply after the steer is what was streaming at the kill. + doc.push_message(&entry( + "msg-assistant-2", + MessageRole::Assistant, + "after the steer…", + now - 10_000, + Some(MessageStatus::Streaming), + )) + .unwrap(); + } + store + .save_snapshot(CHAT, &doc.export_snapshot().unwrap()) + .unwrap(); + + let journal = RunJournal::open(dir.join("orgs/dev-org/dev-user/journals")).unwrap(); + journal + .append( + CHAT, + &AgentEvent::SessionStarted { + harness: HarnessId::Mock, + model: "mock-1".into(), + tools: vec![], + cwd: "/tmp".into(), + session_id: "hs-crash".into(), + assistant_message_id: "msg-assistant-1".into(), + }, + ) + .unwrap(); + journal + .append( + CHAT, + &AgentEvent::TextDelta { + text: "partial…".into(), + }, + ) + .unwrap(); + if consumed { + journal + .append( + CHAT, + &AgentEvent::Steered { + assistant_message_id: Some("msg-assistant-1".into()), + next_assistant_message_id: Some("msg-assistant-2".into()), + }, + ) + .unwrap(); + journal + .append( + CHAT, + &AgentEvent::TextDelta { + text: "after the steer…".into(), + }, + ) + .unwrap(); + // The durable consumed-steer record the boundary writes — one + // message id per line, written directly (see + // `consumed_steers_path`) so this state compiles on the pre-fix + // engine too. + std::fs::write(consumed_steers_path(dir), "msg-user-2\n").unwrap(); + } + } +} + +/// The durable consumed-steer record's file, as the engine lays it out +/// beside the chat's journal. The tests write and read it directly (not +/// through `RunJournal`) so the reproduction below also compiles — and goes +/// red — against the pre-fix engine, which has no record at all and re-sends +/// the consumed steer as its revival prompt. +fn consumed_steers_path(dir: &std::path::Path) -> std::path::PathBuf { + dir.join("orgs/dev-org/dev-user/journals") + .join(format!("{CHAT}.steers")) +} + +fn user_texts(core: &EngineCore) -> Vec { + entries_now(core) + .iter() + .filter(|e| e.role == MessageRole::User) + .filter_map(|e| { + e.parts.iter().find_map(|p| match p { + MessagePart::Text { text, .. } => Some(text.clone()), + _ => None, + }) + }) + .collect() +} + +/// The replay, reproduced: the steer was consumed before the kill, so the +/// boot auto-resume must NOT re-send it — the revival continues the crashed +/// turn with the engine's continue prompt instead, on the same harness +/// conversation, with every other revival behaviour intact. +#[tokio::test] +async fn an_auto_resumed_chat_never_replays_a_consumed_steer() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("data"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("device-id"), "dev-crash").unwrap(); + manufacture_crashed_steer_turn(&dir, true); + + let requests: RequestLog = Arc::new(Mutex::new(Vec::new())); + let core = assemble( + &dir, + RecordingHarness { + requests: requests.clone(), + session_id: "hs-after-crash".into(), + fail_starts: Default::default(), + }, + ); + pre_title(&core); + + wait_for( + || complete_assistant_count(&core) == 1, + "auto-resumed turn to complete", + ) + .await; + + // THE fix: the consumed steer never reaches the harness again. + let recorded = requests.lock().unwrap().clone(); + assert!( + !recorded.iter().any(|r| r.prompt == STEER_TEXT), + "a steer the harness already consumed must not be re-delivered: {recorded:#?}" + ); + // The revival continues the crashed turn with the engine's continue + // prompt, on the crashed run's own harness conversation. + let revived = recorded + .iter() + .find(|r| r.prompt == roboco_proto::RESUME_CONTINUE_PROMPT) + .expect("the revival continues the turn with the continue prompt"); + assert_eq!(revived.resume.as_deref(), Some("hs-crash")); + + // The transcript: the steer appears exactly once, and the engine's + // continue prompt is its own entry — never a duplicate of the steer. + assert_eq!( + user_texts(&core), + vec![ + "long task".to_string(), + STEER_TEXT.to_string(), + roboco_proto::RESUME_CONTINUE_PROMPT.to_string(), + ] + ); + // The crash note still says the run is being picked back up. + let entries = entries_now(&core); + let aborted = entries + .iter() + .find(|e| e.status == Some(MessageStatus::Aborted)) + .expect("crashed entries stay, stamped aborted"); + assert!(aborted.parts.iter().any(|p| matches!( + p, + MessagePart::Error { message, .. } + if message.contains("engine restart") && message.contains("resuming") + ))); + core.shutdown().await; +} + +/// A steer that was accepted but NOT yet consumed when the engine died is +/// still delivered after the resume — exactly once, under its own message +/// id, so the transcript never shows a duplicate. +#[tokio::test] +async fn a_steer_accepted_but_not_consumed_is_delivered_once_after_resume() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("data"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("device-id"), "dev-crash").unwrap(); + // No `Steered` boundary, no consumption record: the steer sat accepted + // in the dead run's mailbox and the harness never saw it. + manufacture_crashed_steer_turn(&dir, false); + + let requests: RequestLog = Arc::new(Mutex::new(Vec::new())); + let core = assemble( + &dir, + RecordingHarness { + requests: requests.clone(), + session_id: "hs-after-crash".into(), + fail_starts: Default::default(), + }, + ); + pre_title(&core); + + wait_for( + || complete_assistant_count(&core) == 1, + "the steer's turn to be delivered and complete", + ) + .await; + + // The steer IS the revival prompt — delivered exactly once, on the + // crashed run's harness conversation. + let recorded = requests.lock().unwrap().clone(); + let deliveries = recorded.iter().filter(|r| r.prompt == STEER_TEXT).count(); + assert_eq!( + deliveries, 1, + "an accepted-but-unconsumed steer is delivered exactly once: {recorded:#?}" + ); + let revived = recorded + .iter() + .find(|r| r.prompt == STEER_TEXT) + .expect("the steer's delivery"); + assert_eq!(revived.resume.as_deref(), Some("hs-crash")); + // No duplicate user entry, and no engine continue prompt this time. + assert_eq!( + user_texts(&core), + vec!["long task".to_string(), STEER_TEXT.to_string()] + ); + core.shutdown().await; +} + +/// A steerable harness that keeps turns in flight: the first prompt parks +/// without a Done, and a steer into it confirms consumption (its `Steered` +/// boundary) and keeps working — the exact shape a crash interrupts. A +/// steer containing `release` finishes the turn instead. +struct HoldingHarness; + +#[async_trait] +impl Harness for HoldingHarness { + fn id(&self) -> HarnessId { + HarnessId::Mock + } + fn display_name(&self) -> &str { + "Holding" + } + fn supports_steering(&self) -> bool { + true + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::StepBoundary + } + fn deterministic_turn_end(&self) -> bool { + true + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[ReasoningLevel::Medium] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + mut controls: RunControls, + ) -> Result>, HarnessError> { + let (tx, rx) = tokio::sync::mpsc::channel::>(32); + let mut steering = controls.steering; + tokio::spawn(async move { + let send = |event: AgentEvent| { + let tx = tx.clone(); + async move { tx.send(Ok(event)).await.is_ok() } + }; + if !send(AgentEvent::SessionStarted { + harness: HarnessId::Mock, + model: "mock-1".into(), + tools: vec![], + cwd: "/tmp".into(), + session_id: "hs-holding".into(), + assistant_message_id: "a-1".into(), + }) + .await + { + return; + } + if !send(AgentEvent::TextDelta { + text: format!("on it: {}", request.prompt), + }) + .await + { + return; + } + // Holding: no Done — the turn stays in flight (the crash shape). + while let Some(steer) = steering.recv().await { + let boundary = AgentEvent::Steered { + assistant_message_id: None, + next_assistant_message_id: steer.message_id.map(|id| format!("a-{id}")), + }; + if !send(boundary).await + || !send(AgentEvent::TextDelta { text: format!("on it: {}", steer.prompt) }).await + { + return; + } + if steer.prompt.contains("release") + && !send(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some("hs-holding".into()), + }) + .await + { + return; + } + } + }); + Ok(futures::stream::unfold( + rx, + |mut rx| async move { rx.recv().await.map(|ev| (ev, rx)) }, + ) + .boxed()) + } +} + +/// The write side of the fix, live: a steer the harness confirms consuming +/// mid-turn is recorded durably right at the boundary — while the engine is +/// still running — so the boot path the tests above exercise can tell. This +/// is the record `an_auto_resumed_chat_never_replays_a_consumed_steer` +/// manufactures. +#[tokio::test] +async fn a_consumed_steer_is_recorded_durably_while_the_engine_runs() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("data"); + std::fs::create_dir_all(&dir).unwrap(); + + let registry = HarnessRegistry::new(); + registry.register(Arc::new(HoldingHarness)); + let core = EngineCore::assemble(&dir, Arc::new(registry), HarnessId::Mock) + .expect("engine core assembles"); + pre_title(&core); + + queue_run(&core, "hold the line", "/tmp", "msg-user-1"); + wait_for(|| entries_now(&core).len() == 2, "the holding turn to stream").await; + + // The steer: accepted into the live run's mailbox, then confirmed + // consumed by the harness's `Steered` boundary. + core.doc_host + .queue_command( + CHAT, + SessionCommandPayload::Steer { + prompt: "hold the detail too".into(), + message_id: Some("msg-user-2".into()), + }, + ) + .expect("queue steer command"); + + // Read the record the engine writes, in the engine's own layout. + let steers = consumed_steers_path(&dir); + wait_for( + || { + std::fs::read_to_string(&steers) + .is_ok_and(|record| record.lines().any(|line| line.trim() == "msg-user-2")) + }, + "the consumed steer to be recorded durably", + ) + .await; + + // The steer is on the transcript once, delivered into the live turn. + assert_eq!( + user_texts(&core), + vec!["hold the line".to_string(), "hold the detail too".to_string()] + ); + core.shutdown().await; +} diff --git a/crates/engine/tests/resume_at_reset.rs b/crates/engine/tests/resume_at_reset.rs new file mode 100644 index 0000000000..a9c66a6e2f --- /dev/null +++ b/crates/engine/tests/resume_at_reset.rs @@ -0,0 +1,987 @@ +//! Resume at reset (orchestration v2, ticket 07): a Limited chat's durable +//! continuation, owned by the engine. It fires the continue prompt at the +//! limit's reset time exactly like the manual Resume (ticket 06) — queued +//! messages behind it; it survives engine restarts and an overdue one fires +//! after boot; a new message, an archive or a delete cancels it; the +//! **Auto-resume limited chats** setting (off by default) arms it +//! automatically when a chat becomes Limited; and a continuation that hits +//! the limit again re-arms only via a NEW limit record whose reset advances +//! past the fired one — never in a loop. +//! +//! The harness seam is the same scripted stand-in as `limited_chats.rs` +//! (`ResetHarness`): each run's script says whether it ends on a classified +//! usage-limit error (with the reset time to report) or completes, every run +//! waits for a `finish` signal before its terminal event, and the script is +//! editable between runs so a continuation can hit the limit AGAIN. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use futures::StreamExt; +use futures::stream::BoxStream; + +use roboco_doc::{MessagePart, MessageRole, SessionCommandPayload}; +use roboco_engine::{AgentAccounts, AgentAccountsConfig, EngineCore, HarnessRegistry}; +use roboco_harness::{Harness, HarnessError, RunControls}; +use roboco_proto::{ + AgentErrorKind, AgentEvent, DoneStatus, HarnessId, Model, ReasoningLevel, RunRequest, + SandboxLevel, SessionStatus, SteeringMode, +}; +use roboco_rpc::methods; + +const CHAT: &str = "chat-resume-reset"; +const CONTINUE: &str = roboco_proto::RESUME_CONTINUE_PROMPT; + +/// The per-run script: `Some(resets_at)` ends the run on a classified +/// usage-limit error reporting that reset time; `None` completes normally. +type Script = Arc>>>; + +/// A scripted harness: run `n` ends on a usage-limit error with +/// `script[n]`'s reset time when `Some`, else completes. Every run holds +/// until the test releases it, so turn-end order is deterministic. +struct ResetHarness { + finish: tokio::sync::broadcast::Sender<()>, + prompts: Arc>>, + script: Script, + runs: std::sync::atomic::AtomicUsize, +} + +impl ResetHarness { + /// `script` is the initial per-run plan (extendable between runs with + /// [`Self::set_script`]). Returns the prompts log with the harness. + fn new(script: Vec>) -> (Arc, Arc>>) { + let (finish, _) = tokio::sync::broadcast::channel(16); + let prompts = Arc::new(Mutex::new(Vec::new())); + ( + Arc::new(Self { + finish, + prompts: prompts.clone(), + script: Arc::new(Mutex::new(script)), + runs: std::sync::atomic::AtomicUsize::new(0), + }), + prompts, + ) + } + + /// Rewrite one run's outcome — the "hits the limit again" scripts. + fn set_script(&self, run: usize, resets_at: Option) { + let mut script = self.script.lock().unwrap(); + while script.len() <= run { + script.push(None); + } + script[run] = resets_at; + } +} + +#[async_trait] +impl Harness for ResetHarness { + fn id(&self) -> HarnessId { + HarnessId::ClaudeCode + } + fn display_name(&self) -> &str { + "Reset" + } + fn supports_steering(&self) -> bool { + false + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::TurnBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[ReasoningLevel::Medium] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + _controls: RunControls, + ) -> Result>, HarnessError> { + // Subscribe before recording the prompt: a test that sees the prompt + // may send `finish` at once. + let mut finish = self.finish.subscribe(); + self.prompts.lock().unwrap().push(request.prompt.clone()); + let run = self.runs.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let resets_at = self.script.lock().unwrap().get(run).copied().flatten(); + let session = format!("sess-{run}"); + let started = futures::stream::iter(vec![Ok(AgentEvent::SessionStarted { + harness: HarnessId::ClaudeCode, + model: "reset-1".into(), + tools: vec![], + cwd: request.cwd.clone(), + session_id: session.clone(), + assistant_message_id: format!("a-{run}"), + })]); + let tail = futures::stream::once(async move { + // The turn runs until the test says it ends. + let _ = finish.recv().await; + match resets_at { + Some(resets_at) => vec![ + Ok(AgentEvent::Error { + message: "Claude usage limit reached — try again after the limit resets." + .into(), + kind: Some(AgentErrorKind::UsageLimit { + window: "5-hour".into(), + resets_at: Some(resets_at), + }), + }), + Ok(AgentEvent::Done { + status: DoneStatus::Errored, + result: None, + error: Some("the turn failed".into()), + session_id: Some(session), + }), + ], + None => vec![ + Ok(AgentEvent::TextDelta { + text: "Picked the work back up.".into(), + }), + Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some(session), + }), + ], + } + }) + .flat_map(futures::stream::iter); + Ok(started.chain(tail).boxed()) + } +} + +async fn wait_for(mut predicate: F, what: &str) +where + F: FnMut() -> bool, +{ + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while !predicate() { + assert!( + tokio::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(Duration::from_millis(15)).await; + } +} + +/// Epoch ms `ms` from now — the tests' short reset times. +fn in_ms(ms: i64) -> i64 { + chrono::Utc::now().timestamp_millis() + ms +} + +fn run_request(prompt: &str) -> RunRequest { + RunRequest { + prompt: prompt.into(), + harness: Some(HarnessId::ClaudeCode), + model: None, + reasoning: None, + model_options: Default::default(), + cwd: ".".into(), + sandbox: SandboxLevel::WorkspaceWrite, + auto_approve: true, + attachments: Vec::new(), + worktree: None, + mcp: None, + resume: None, + provider_fork: None, + resume_fallback_history: None, + } +} + +/// Queue a Run command the way the composer's send does. +fn queue_run(core: &EngineCore, prompt: &str, message_id: &str) { + core.doc_host + .queue_command( + CHAT, + SessionCommandPayload::Run { + request: run_request(prompt), + message_id: message_id.into(), + }, + ) + .expect("queue run command"); +} + +fn prompts_contain(prompts: &Arc>>, needle: &str) -> bool { + prompts.lock().unwrap().iter().any(|p| p == needle) +} + +fn sent_prompts(prompts: &Arc>>) -> Vec { + prompts.lock().unwrap().clone() +} + +fn user_texts(core: &EngineCore) -> Vec { + core.doc_host + .open(CHAT) + .ok() + .and_then(|h| h.doc().read_entries().ok()) + .unwrap_or_default() + .iter() + .filter(|e| e.role == MessageRole::User) + .map(|e| { + e.parts + .iter() + .filter_map(|p| match p { + MessagePart::Text { text, .. } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("") + }) + .collect() +} + +/// Assemble a core around the scripted harness, pre-titled, with an +/// ISOLATED accounts store, and a chat carrying a Claude config row — the +/// continuation's run configuration (the same one the completion-notice +/// turn uses: the chat row first, the last request as fallback). +async fn setup( + script: Vec>, +) -> ( + EngineCore, + Arc, + Arc>>, + roboco_rpc::RpcClient, +) { + let tmp = tempfile::tempdir().unwrap(); + // Leak the guard: the engine outlives this helper; uniqueness is all the + // test needs. + let path = tmp.keep(); + setup_at(&path.join("data"), script).await +} + +/// [`setup`] over an explicit data dir — the restart test reuses one dir for +/// both engines. +async fn setup_at( + data_dir: &std::path::Path, + script: Vec>, +) -> ( + EngineCore, + Arc, + Arc>>, + roboco_rpc::RpcClient, +) { + let (harness, prompts) = ResetHarness::new(script); + let registry = HarnessRegistry::new(); + registry.register(harness.clone()); + let core = EngineCore::assemble(data_dir, Arc::new(registry), HarnessId::Mock) + .expect("engine core assembles"); + core.sessions.set_agent_accounts(AgentAccounts::new( + AgentAccountsConfig::isolated(&data_dir.parent().unwrap().join("accounts")), + )); + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + methods::MUTATE, + serde_json::json!({ + "op": "createChat", + "chatId": CHAT, + "deviceId": core.device_id, + "config": { + "harness": "claude-code", + "sandbox": "workspace-write", + }, + }), + ) + .await + .expect("createChat"); + // Pre-title so the auto-titler never dispatches a harness request of its own. + core.workspace + .rename_chat(CHAT, "Pre-titled") + .expect("rename chat"); + (core, harness, prompts, client) +} + +/// Run the opening turn into the scripted limit and wait for the chat to +/// settle Limited. +async fn run_into_limit( + core: &EngineCore, + harness: &Arc, + prompts: &Arc>>, +) { + queue_run(core, "do the work", "m-open"); + wait_for( + || prompts_contain(prompts, "do the work"), + "the limited run to start", + ) + .await; + let _ = harness.finish.send(()); + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == SessionStatus::Errored && s.limit.is_some()) + }, + "the chat to settle Limited", + ) + .await; +} + +fn scheduled_fire_at(core: &EngineCore) -> Option { + core.sessions + .session_status(CHAT) + .and_then(|s| s.scheduled_resume.map(|r| r.fire_at)) +} + +/// Arm the continuation through the client the way the banner's action does. +async fn schedule_resume(client: &roboco_rpc::RpcClient) { + client + .call( + methods::MUTATE, + serde_json::json!({ "op": "scheduleResumeAtReset", "chatId": CHAT }), + ) + .await + .expect("scheduleResumeAtReset"); +} + +/// Flip the Auto-resume limited chats engine setting through the settings RPC. +async fn set_auto_resume(client: &roboco_rpc::RpcClient, on: bool) { + let saved = client + .call( + methods::SET_ENGINE_SETTINGS, + serde_json::json!({ "autoResumeLimited": on }), + ) + .await + .expect("SetEngineSettings"); + assert_eq!(saved["autoResumeLimited"], on, "the reply is the stored value"); +} + +/// **Resume at reset fires the continuation at the reset time** — the same +/// run the manual Resume is: the continue prompt as a normal run, the queued +/// messages behind it, the limit record cleared by the new turn, the armed +/// record cleared by the firing, and the row mirrored to the workspace doc +/// (remote sidebars' banners) both ways. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resume_at_reset_fires_the_continue_prompt_at_the_reset_time() { + let (core, harness, prompts, client) = setup(vec![Some(in_ms(2_500)), None]).await; + + run_into_limit(&core, &harness, &prompts).await; + // Messages the user sent while the doomed run was working — they wait + // behind the continuation, exactly like the manual Resume. + core.doc_host + .queue_message(CHAT, "queued one", Vec::new()) + .expect("queue message"); + + schedule_resume(&client).await; + // The engine arms at the limit record's own reset time — the exact time + // the harness's error reported, read back from the settled row. + let fire_at = core + .sessions + .session_status(CHAT) + .and_then(|s| s.limit.and_then(|limit| limit.resets_at)) + .expect("the limit record carries the reset time"); + wait_for( + || scheduled_fire_at(&core) == Some(fire_at), + "the armed continuation to reach the session row", + ) + .await; + // The row rides the workspace doc too — remote devices' banners. + wait_for( + || { + core.workspace + .read_sessions() + .map(|rows| { + rows.iter().any(|row| { + row.chat_id == CHAT + && row + .scheduled_resume + .is_some_and(|r| r.fire_at == fire_at) + }) + }) + .unwrap_or(false) + }, + "the armed continuation to mirror into the workspace doc", + ) + .await; + + // The reset time arrives; the engine fires the continuation. + wait_for( + || prompts_contain(&prompts, CONTINUE), + "the continuation to fire at the reset time", + ) + .await; + // The new turn owns the chat: the limit record and the armed record are + // both gone. + wait_for( + || { + core.sessions.session_status(CHAT).is_some_and(|s| { + s.status == SessionStatus::Working + && s.limit.is_none() + && s.scheduled_resume.is_none() + }) + }, + "the continuation's turn to clear the limit and armed records", + ) + .await; + // The queued rows still wait behind the continuation's turn. + assert!(!prompts_contain(&prompts, "queued one")); + + let _ = harness.finish.send(()); + wait_for( + || prompts_contain(&prompts, "queued one"), + "the queued message to follow the continuation", + ) + .await; + let _ = harness.finish.send(()); + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == SessionStatus::Idle) + }, + "the followed-up turn to settle", + ) + .await; + assert_eq!( + sent_prompts(&prompts), + vec!["do the work", CONTINUE, "queued one"], + "the continuation fired the continue prompt, the queue followed" + ); + // The transcript carries the continuation as its user entry. + assert!(user_texts(&core).contains(&CONTINUE.to_string())); + core.shutdown().await; +} + +/// **Restarts:** a pending continuation survives an engine restart, and one +/// whose time passed while the engine was down fires after boot. The +/// transcript entry is idempotent by its deterministic message id. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_pending_continuation_survives_a_restart_and_an_overdue_one_fires_after_boot() { + let tmp = tempfile::tempdir().unwrap(); + let data_dir = tmp.path().join("data"); + let (core, harness, prompts, client) = setup_at(&data_dir, vec![Some(in_ms(2_000))]).await; + + run_into_limit(&core, &harness, &prompts).await; + schedule_resume(&client).await; + wait_for( + || scheduled_fire_at(&core).is_some(), + "the continuation to arm", + ) + .await; + + // A graceful shutdown while the continuation is still pending: it must + // NOT be consumed (shutting down never fires one) — the record survives. + core.shutdown().await; + drop(core); + // The reset time passes while no engine is running. + tokio::time::sleep(Duration::from_millis(2_200)).await; + // The first engine's harness never saw a continuation. + assert!(!prompts_contain(&prompts, CONTINUE)); + + // Boot a fresh engine over the same data dir: the overdue continuation + // fires after the engine is up. + let (core, harness, prompts, _client) = setup_at(&data_dir, vec![None]).await; + wait_for( + || prompts_contain(&prompts, CONTINUE), + "the overdue continuation to fire after boot", + ) + .await; + let _ = harness.finish.send(()); + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == SessionStatus::Idle) + }, + "the continuation's turn to settle", + ) + .await; + assert_eq!(sent_prompts(&prompts), vec![CONTINUE]); + // Exactly one continuation entry — the deterministic message id keeps a + // crash-raced re-fire from duplicating the transcript row. + assert_eq!( + user_texts(&core) + .iter() + .filter(|text| text.as_str() == CONTINUE) + .count(), + 1 + ); + core.shutdown().await; +} + +/// **Restart keeps the client-visible rows:** the in-memory status map is +/// the merged WatchSessions source for this device's chats, and it starts +/// empty at boot — without the republish, a still-Limited chat and its +/// armed continuation vanish from the banner (no "resumes in…", no +/// Cancel) and `ScheduleResumeAtReset` refuses ("not Limited"). The boot +/// re-creates both rows from the durable workspace row. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_restart_republishes_the_limit_and_scheduled_resume_rows() { + let tmp = tempfile::tempdir().unwrap(); + let data_dir = tmp.path().join("data"); + // A reset far enough out that nothing fires while we look. + let (core, harness, prompts, client) = setup_at(&data_dir, vec![Some(in_ms(60_000))]).await; + + run_into_limit(&core, &harness, &prompts).await; + schedule_resume(&client).await; + let fire_at = scheduled_fire_at(&core).expect("the continuation to arm"); + + core.shutdown().await; + drop(core); + tokio::time::sleep(Duration::from_millis(200)).await; + + // Boot a fresh engine over the same data dir: the client-visible rows + // are republished at boot, before anything fires. + let (core, _harness, prompts, _client) = setup_at(&data_dir, vec![None]).await; + let row = core.sessions.session_status(CHAT).expect("the republished row"); + assert_eq!(row.status, SessionStatus::Errored, "still Limited"); + assert!(row.limit.is_some(), "the limit record survived the restart"); + assert_eq!( + row.scheduled_resume.map(|resume| resume.fire_at), + Some(fire_at), + "the armed continuation's row survived the restart" + ); + // The workspace mirror carries it too (remote devices' banners). + assert!(core + .workspace + .read_sessions() + .map(|rows| rows.iter().any(|row| { + row.chat_id == CHAT + && row.limit.is_some() + && row.scheduled_resume.is_some_and(|r| r.fire_at == fire_at) + })) + .unwrap_or(false)); + // Nothing fired while we looked. + assert!(!prompts_contain(&prompts, CONTINUE)); + core.shutdown().await; +} + +/// **Cancellation by a new message:** a normal send (the composer's Run +/// command) cancels the pending continuation — the user's newer intent wins. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_new_message_cancels_the_pending_continuation() { + let (core, harness, prompts, client) = setup(vec![Some(in_ms(2_500)), None]).await; + + run_into_limit(&core, &harness, &prompts).await; + schedule_resume(&client).await; + wait_for( + || scheduled_fire_at(&core).is_some(), + "the continuation to arm", + ) + .await; + + // The user's own next message (a normal send, not the resume prompt). + queue_run(&core, "try a different angle", "m-next"); + wait_for( + || scheduled_fire_at(&core).is_none(), + "the new message to cancel the continuation", + ) + .await; + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == SessionStatus::Working) + }, + "the new turn to start", + ) + .await; + let _ = harness.finish.send(()); + + // The reset time comes and goes: no continuation ever fires. + tokio::time::sleep(Duration::from_millis(3_000)).await; + assert!(!prompts_contain(&prompts, CONTINUE)); + assert_eq!( + sent_prompts(&prompts), + vec!["do the work", "try a different angle"] + ); + core.shutdown().await; +} + +/// **Queueing is not sending (story 41):** a message the user queued on the +/// Limited chat does NOT cancel the armed continuation — queued messages +/// wait BEHIND it, order preserved. The continuation fires at the reset +/// time and the queued row follows it out (a queued row is not the newer +/// intent a direct send is; the user can still cancel with a message or an +/// archive). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_queued_message_waits_behind_the_pending_continuation() { + // Three runs: the doomed turn, the continuation, the queued row's own. + let (core, harness, prompts, client) = setup(vec![Some(in_ms(2_500)), None, None]).await; + + run_into_limit(&core, &harness, &prompts).await; + schedule_resume(&client).await; + wait_for( + || scheduled_fire_at(&core).is_some(), + "the continuation to arm", + ) + .await; + + core.doc_host + .queue_message(CHAT, "queued while limited", Vec::new()) + .expect("queue message"); + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + scheduled_fire_at(&core).is_some(), + "queueing a row does not cancel the armed continuation" + ); + + // The reset time arrives: the continuation fires, and the queued row + // follows it — the queue thaws with the continuation's dispatch and + // drains when its turn ends. + wait_for( + || prompts_contain(&prompts, CONTINUE), + "the continuation to fire at the reset time", + ) + .await; + // The continuation owns the turn: the queued row waits behind it. + assert!(!prompts_contain(&prompts, "queued while limited")); + let _ = harness.finish.send(()); + wait_for( + || prompts_contain(&prompts, "queued while limited"), + "the queued row to follow the continuation", + ) + .await; + assert_eq!( + sent_prompts(&prompts), + vec!["do the work", CONTINUE, "queued while limited"], + "order preserved: the continuation first, the queued row behind it" + ); + core.shutdown().await; +} + +/// **Cancellation by archiving:** putting the Limited chat away cancels its +/// pending continuation — archived work must not resume on its own +/// overnight. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn archiving_the_chat_cancels_the_pending_continuation() { + let (core, harness, prompts, client) = setup(vec![Some(in_ms(2_500))]).await; + + run_into_limit(&core, &harness, &prompts).await; + schedule_resume(&client).await; + wait_for( + || scheduled_fire_at(&core).is_some(), + "the continuation to arm", + ) + .await; + + client + .call( + methods::MUTATE, + serde_json::json!({ "op": "setChatArchived", "chatId": CHAT, "archived": true }), + ) + .await + .expect("setChatArchived"); + wait_for( + || scheduled_fire_at(&core).is_none(), + "archiving to cancel the continuation", + ) + .await; + + tokio::time::sleep(Duration::from_millis(3_000)).await; + assert!(!prompts_contain(&prompts, CONTINUE)); + assert_eq!(sent_prompts(&prompts), vec!["do the work"]); + core.shutdown().await; +} + +/// **Archive that landed from another device** — through the synced +/// workspace row, not this engine's RPC: the arm survives (nothing told +/// this engine), and the fire path's own guard must drop it at the reset +/// time instead of resuming archived work on its own. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_remote_archive_drops_the_continuation_at_fire_time() { + let (core, harness, prompts, client) = setup(vec![Some(in_ms(2_500))]).await; + + run_into_limit(&core, &harness, &prompts).await; + schedule_resume(&client).await; + wait_for( + || scheduled_fire_at(&core).is_some(), + "the continuation to arm", + ) + .await; + // The other device's archive reaches this engine only through the + // workspace doc — the mutate RPC never ran here. + core.workspace + .set_chat_archived(CHAT, true) + .expect("remote setChatArchived"); + assert!( + scheduled_fire_at(&core).is_some(), + "nothing cancelled the arm on this engine" + ); + + // The reset time comes; the firing consumes the continuation and drops + // it at the archived-chat guard instead of dispatching. + wait_for( + || scheduled_fire_at(&core).is_none(), + "the fired continuation to consume the arm", + ) + .await; + tokio::time::sleep(Duration::from_millis(3_000)).await; + assert!(!prompts_contain(&prompts, CONTINUE), "archived work never resumes"); + assert_eq!(sent_prompts(&prompts), vec!["do the work"]); + core.shutdown().await; +} + +/// **Cancellation by deleting:** the chat is gone; its continuation is too. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn deleting_the_chat_cancels_the_pending_continuation() { + let (core, harness, prompts, client) = setup(vec![Some(in_ms(2_500))]).await; + + run_into_limit(&core, &harness, &prompts).await; + schedule_resume(&client).await; + wait_for( + || scheduled_fire_at(&core).is_some(), + "the continuation to arm", + ) + .await; + + client + .call( + methods::MUTATE, + serde_json::json!({ "op": "deleteChat", "chatId": CHAT }), + ) + .await + .expect("deleteChat"); + wait_for( + || scheduled_fire_at(&core).is_none(), + "deleting to cancel the continuation", + ) + .await; + + tokio::time::sleep(Duration::from_millis(3_000)).await; + assert!(!prompts_contain(&prompts, CONTINUE)); + core.shutdown().await; +} + +/// **The setting, off (the default):** a chat becoming Limited arms nothing — +/// the engine settings read back off, no continuation rides the row, and +/// nothing fires when the reset time passes. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_setting_defaults_to_off_and_arms_nothing() { + let (core, harness, prompts, client) = setup(vec![Some(in_ms(500))]).await; + + let settings = client + .call(methods::GET_ENGINE_SETTINGS, serde_json::json!({})) + .await + .expect("GetEngineSettings"); + assert_eq!(settings["autoResumeLimited"], false, "off by default"); + + run_into_limit(&core, &harness, &prompts).await; + assert!(scheduled_fire_at(&core).is_none(), "nothing auto-armed"); + tokio::time::sleep(Duration::from_millis(3_000)).await; + assert!(!prompts_contain(&prompts, CONTINUE)); + core.shutdown().await; +} + +/// **The setting, on:** the engine arms the continuation automatically when +/// the chat becomes Limited — no click needed — and it fires at the reset +/// time exactly like the manual arm. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_setting_on_arms_the_continuation_automatically() { + let (core, harness, prompts, client) = setup(vec![Some(in_ms(2_500)), None]).await; + set_auto_resume(&client, true).await; + + run_into_limit(&core, &harness, &prompts).await; + // The auto-arm targets the record's reset time — still ahead. + let fire_at = scheduled_fire_at(&core).expect("auto-armed when the chat went Limited"); + let reported = core + .sessions + .session_status(CHAT) + .and_then(|s| s.limit.and_then(|limit| limit.resets_at)) + .expect("the limit record"); + assert_eq!(fire_at, reported, "the auto-arm fires at the reset time"); + + wait_for( + || prompts_contain(&prompts, CONTINUE), + "the auto-armed continuation to fire at the reset time", + ) + .await; + let _ = harness.finish.send(()); + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == SessionStatus::Idle) + }, + "the continuation's turn to settle", + ) + .await; + assert_eq!(sent_prompts(&prompts), vec!["do the work", CONTINUE]); + core.shutdown().await; +} + +/// **Overlap, re-armed via a new limit record:** with the setting on, a +/// continuation that fires into a STILL-limited chat becomes Limited again +/// with the new reset time — and the fresh record re-arms it (the new reset +/// genuinely advances past the fired one). The work continues across two +/// limit windows without a click. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_continuation_that_hits_the_limit_again_rearms_via_the_new_record() { + let (core, harness, prompts, client) = setup(vec![Some(in_ms(2_500)), Some(in_ms(6_000)), None]).await; + set_auto_resume(&client, true).await; + + // Turn one ends limited at +400ms → auto-armed. + run_into_limit(&core, &harness, &prompts).await; + wait_for( + || scheduled_fire_at(&core).is_some(), + "the first limit to auto-arm", + ) + .await; + + // The continuation fires into the same wall — still limited — with a NEW + // (later) reset time. + wait_for( + || prompts_contain(&prompts, CONTINUE), + "the first continuation to fire", + ) + .await; + let _ = harness.finish.send(()); + wait_for( + || { + core.sessions.session_status(CHAT).is_some_and(|s| { + s.status == SessionStatus::Errored + && s.limit.is_some_and(|l| l.resets_at.is_some()) + }) + }, + "the continuation to hit the limit again", + ) + .await; + // The fresh record re-armed the continuation at the NEW reset time. + wait_for( + || { + scheduled_fire_at(&core) + .is_some_and(|at| at > chrono::Utc::now().timestamp_millis()) + }, + "the new limit record to re-arm the continuation", + ) + .await; + + // The second continuation fires at the new reset time and completes. + wait_for( + || { + prompts + .lock() + .unwrap() + .iter() + .filter(|p| p.as_str() == CONTINUE) + .count() + == 2 + }, + "the re-armed continuation to fire", + ) + .await; + let _ = harness.finish.send(()); + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == SessionStatus::Idle) + }, + "the second continuation's turn to settle", + ) + .await; + assert_eq!(sent_prompts(&prompts), vec!["do the work", CONTINUE, CONTINUE]); + core.shutdown().await; +} + +/// **Overlap, never in a loop:** the fired continuation is consumed — a +/// second limit record whose reset does NOT advance past the fired one (the +/// same wall, already past) re-arms nothing. One continuation, then the +/// chat sits Limited waiting for the user, not cycling on its own. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_stale_reset_time_never_rearms_into_a_loop() { + let (core, harness, prompts, client) = setup(vec![Some(in_ms(2_500)), Some(in_ms(2_500))]).await; + set_auto_resume(&client, true).await; + + run_into_limit(&core, &harness, &prompts).await; + wait_for( + || scheduled_fire_at(&core).is_some(), + "the first limit to auto-arm", + ) + .await; + + // The continuation fires; the run hits the limit again reporting the + // SAME reset time — already in the past by now. + wait_for( + || prompts_contain(&prompts, CONTINUE), + "the continuation to fire", + ) + .await; + let _ = harness.finish.send(()); + wait_for( + || { + core.sessions + .session_status(CHAT) + .is_some_and(|s| s.status == SessionStatus::Errored) + }, + "the continuation to hit the limit again", + ) + .await; + // No re-arm: the record's reset does not advance past the fired one. + assert!( + scheduled_fire_at(&core).is_none(), + "a stale reset time never re-arms the continuation" + ); + + // And it stays that way — no second continuation, ever. + tokio::time::sleep(Duration::from_millis(3_000)).await; + assert_eq!(sent_prompts(&prompts), vec!["do the work", CONTINUE]); + // The chat is Limited again with the new record — waiting for the user. + assert!(core + .sessions + .session_status(CHAT) + .is_some_and(|s| s.limit.is_some())); + core.shutdown().await; +} + +/// Scheduling requires a Limited chat with a reset time still ahead: a +/// not-limited chat refuses with an actionable error (the honest offer is +/// the manual Resume), and a working arm cancels cleanly and idempotently. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn scheduling_refuses_chats_without_an_upcoming_reset_time() { + let (core, harness, prompts, client) = setup(vec![Some(in_ms(60_000))]).await; + + // Not Limited yet. + let err = client + .call( + methods::MUTATE, + serde_json::json!({ "op": "scheduleResumeAtReset", "chatId": CHAT }), + ) + .await + .expect_err("not Limited"); + assert!(err.to_string().contains("not Limited"), "{err}"); + + run_into_limit(&core, &harness, &prompts).await; + // A reset far in the future arms fine — then cancels cleanly. + schedule_resume(&client).await; + wait_for( + || scheduled_fire_at(&core).is_some(), + "the continuation to arm", + ) + .await; + client + .call( + methods::MUTATE, + serde_json::json!({ "op": "cancelScheduledResume", "chatId": CHAT }), + ) + .await + .expect("cancelScheduledResume"); + assert!(scheduled_fire_at(&core).is_none()); + // Cancelling again is a no-op, not an error. + client + .call( + methods::MUTATE, + serde_json::json!({ "op": "cancelScheduledResume", "chatId": CHAT }), + ) + .await + .expect("cancelScheduledResume is idempotent"); + core.shutdown().await; +} + +/// An already-due reset refuses: there is nothing left to wait for, and the +/// caller is pointed back at the manual Resume. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn scheduling_refuses_an_already_due_reset() { + let (core, harness, prompts, client) = setup(vec![Some(in_ms(-1))]).await; + run_into_limit(&core, &harness, &prompts).await; + let err = client + .call( + methods::MUTATE, + serde_json::json!({ "op": "scheduleResumeAtReset", "chatId": CHAT }), + ) + .await + .expect_err("the reset already passed"); + assert!(err.to_string().contains("already reset"), "{err}"); + core.shutdown().await; +} diff --git a/crates/engine/tests/rich_composer_delivery.rs b/crates/engine/tests/rich_composer_delivery.rs index a7fbb0851c..736c83cf2a 100644 --- a/crates/engine/tests/rich_composer_delivery.rs +++ b/crates/engine/tests/rich_composer_delivery.rs @@ -159,6 +159,8 @@ fn request(prompt: &str) -> RunRequest { sandbox: SandboxLevel::WorkspaceWrite, auto_approve: true, resume: None, + provider_fork: None, + resume_fallback_history: None, attachments: vec![], worktree: None, mcp: None, @@ -278,7 +280,7 @@ async fn rich_selections_survive_fresh_warm_steer_and_attachment_delivery_for_ev let (_tmp, core, _harness, mut rx) = setup(id).await; let (raw, readable) = rich_text("first"); core.sessions - .dispatch(CHAT, id, request(&raw), None) + .dispatch(CHAT, id, request(&raw), None, None, None) .await .unwrap(); let Delivery::Run(run) = receive(&mut rx).await else { @@ -290,7 +292,7 @@ async fn rich_selections_survive_fresh_warm_steer_and_attachment_delivery_for_ev let (warm, readable) = rich_text("warm"); core.sessions - .dispatch(CHAT, id, request(&warm), None) + .dispatch(CHAT, id, request(&warm), None, None, None) .await .unwrap(); let Delivery::Steer(text, _) = receive(&mut rx).await else { @@ -301,7 +303,7 @@ async fn rich_selections_survive_fresh_warm_steer_and_attachment_delivery_for_ev let (steer, readable) = rich_text("steer"); assert!(matches!( - core.sessions.steer(CHAT, &steer, None).await.unwrap(), + core.sessions.steer(CHAT, &steer, None, None).await.unwrap(), SteerOutcome::Accepted )); let Delivery::Steer(text, _) = receive(&mut rx).await else { @@ -316,7 +318,7 @@ async fn rich_selections_survive_fresh_warm_steer_and_attachment_delivery_for_ev let mut req = request(&attached); req.attachments.push(path.to_string_lossy().into_owned()); core.sessions - .dispatch(CHAT, id, req.clone(), None) + .dispatch(CHAT, id, req.clone(), None, None, None) .await .unwrap(); let Delivery::Steer(text, attachments) = receive(&mut rx).await else { @@ -338,7 +340,7 @@ async fn rich_selections_are_converted_once_on_startup_retry_for_every_harness() // Startup retries are intentionally limited to an engine-injected // resume: establish a real prior session before simulating its crash. core.sessions - .dispatch(CHAT, id, request("seed session"), None) + .dispatch(CHAT, id, request("seed session"), None, None, None) .await .unwrap(); let Delivery::Run(_) = receive(&mut rx).await else { @@ -362,7 +364,7 @@ async fn rich_selections_are_converted_once_on_startup_retry_for_every_harness() harness.fail_start.store(true, Ordering::SeqCst); let (raw, readable) = rich_text("retry"); core.sessions - .dispatch(CHAT, id, request(&raw), Some("same-user".into())) + .dispatch(CHAT, id, request(&raw), Some("same-user".into()), None, None) .await .unwrap(); for _ in 0..2 { @@ -382,7 +384,7 @@ async fn queue_edits_preserve_reselected_skills_until_delivery_for_every_harness for id in HARNESSES { let (_tmp, core, _harness, mut rx) = setup(id).await; core.sessions - .dispatch(CHAT, id, request("opening"), None) + .dispatch(CHAT, id, request("opening"), None, None, None) .await .unwrap(); let Delivery::Run(_) = receive(&mut rx).await else { diff --git a/crates/engine/tests/runtime_survival.rs b/crates/engine/tests/runtime_survival.rs index 30993e12c3..a05287fe90 100644 --- a/crates/engine/tests/runtime_survival.rs +++ b/crates/engine/tests/runtime_survival.rs @@ -244,6 +244,8 @@ fn request(prompt: &str) -> RunRequest { attachments: Vec::new(), worktree: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } @@ -278,7 +280,7 @@ async fn sends_with_images_or_respelled_options_join_the_live_runtime() { let rig = rig(); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, request("launch the scouts"), None) + .dispatch(CHAT, HarnessId::Mock, request("launch the scouts"), None, None, None) .await .unwrap(); settle(&rig, 1).await; @@ -292,7 +294,7 @@ async fn sends_with_images_or_respelled_options_join_the_live_runtime() { follow_up.attachments = vec!["/tmp/shot.png".into()]; rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, follow_up, None) + .dispatch(CHAT, HarnessId::Mock, follow_up, None, None, None) .await .unwrap(); settle(&rig, 2).await; @@ -323,7 +325,7 @@ async fn sends_with_images_or_respelled_options_join_the_live_runtime() { other_model.model = Some("process-2".into()); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, other_model, None) + .dispatch(CHAT, HarnessId::Mock, other_model, None, None, None) .await .unwrap(); settle(&rig, 4).await; @@ -339,7 +341,7 @@ async fn a_model_switch_is_adopted_by_the_live_runtime() { let rig = rig_with(true); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, request("launch the scouts"), None) + .dispatch(CHAT, HarnessId::Mock, request("launch the scouts"), None, None, None) .await .unwrap(); settle(&rig, 1).await; @@ -349,7 +351,7 @@ async fn a_model_switch_is_adopted_by_the_live_runtime() { other_model.reasoning = Some(ReasoningLevel::Medium); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, other_model, None) + .dispatch(CHAT, HarnessId::Mock, other_model, None, None, None) .await .unwrap(); settle(&rig, 2).await; @@ -361,7 +363,7 @@ async fn a_model_switch_is_adopted_by_the_live_runtime() { again.reasoning = Some(ReasoningLevel::Medium); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, again, None) + .dispatch(CHAT, HarnessId::Mock, again, None, None, None) .await .unwrap(); settle(&rig, 3).await; @@ -378,7 +380,7 @@ async fn a_model_switch_is_adopted_by_the_live_runtime() { elsewhere.cwd = "/var".into(); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, elsewhere, None) + .dispatch(CHAT, HarnessId::Mock, elsewhere, None, None, None) .await .unwrap(); settle(&rig, 4).await; @@ -394,7 +396,7 @@ async fn a_message_orphaned_by_a_dying_runtime_replays_as_sent() { let rig = rig_with(true); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, request("hold: working"), None) + .dispatch(CHAT, HarnessId::Mock, request("hold: working"), None, None, None) .await .unwrap(); wait_for( @@ -408,7 +410,7 @@ async fn a_message_orphaned_by_a_dying_runtime_replays_as_sent() { sent.attachments = vec!["/tmp/shot.png".into()]; rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, sent, Some("m-orphan".into())) + .dispatch(CHAT, HarnessId::Mock, sent, Some("m-orphan".into()), None, None) .await .unwrap(); settle(&rig, 2).await; @@ -489,7 +491,7 @@ async fn a_send_held_for_the_turn_end_runs_as_sent_without_a_ping() { let (pings, recorder) = record_pings(&rig); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, request("wait: first"), None) + .dispatch(CHAT, HarnessId::Mock, request("wait: first"), None, None, None) .await .unwrap(); wait_for( @@ -565,7 +567,7 @@ async fn stop_cancels_sends_held_for_the_stopped_turn() { let rig = rig_full(false, true); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, request("wait: first"), None) + .dispatch(CHAT, HarnessId::Mock, request("wait: first"), None, None, None) .await .unwrap(); wait_for( @@ -645,7 +647,7 @@ async fn a_message_the_stop_cancelled_never_reaches_a_fresh_runtime() { let rig = rig(); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, request("hold: working"), None) + .dispatch(CHAT, HarnessId::Mock, request("hold: working"), None, None, None) .await .unwrap(); wait_for( @@ -666,7 +668,8 @@ async fn a_message_the_stop_cancelled_never_reaches_a_fresh_runtime() { HarnessId::Mock, request("hold: sent before the stop"), Some("m-late".into()), - ) + None, + None) .await .unwrap(); tokio::time::sleep(Duration::from_millis(300)).await; @@ -684,7 +687,8 @@ async fn a_message_the_stop_cancelled_never_reaches_a_fresh_runtime() { HarnessId::Mock, request("after the stop"), Some("m-next".into()), - ) + None, + None) .await .unwrap(); settle(&rig, 2).await; @@ -700,7 +704,7 @@ async fn send_now_keeps_the_runtime_and_stop_tears_it_down() { let rig = rig(); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, request("hold: build it"), None) + .dispatch(CHAT, HarnessId::Mock, request("hold: build it"), None, None, None) .await .unwrap(); wait_for( @@ -735,7 +739,7 @@ async fn send_now_keeps_the_runtime_and_stop_tears_it_down() { ); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, request("hold: again"), None) + .dispatch(CHAT, HarnessId::Mock, request("hold: again"), None, None, None) .await .unwrap(); wait_for( @@ -769,7 +773,7 @@ async fn send_now_keeps_the_runtime_and_stop_tears_it_down() { // The next message starts a fresh process. rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, request("status?"), None) + .dispatch(CHAT, HarnessId::Mock, request("status?"), None, None, None) .await .unwrap(); settle(&rig, 3).await; @@ -819,7 +823,7 @@ async fn a_send_right_after_stop_runs_in_a_fresh_runtime() { let rig = rig(); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, request("hold: working"), None) + .dispatch(CHAT, HarnessId::Mock, request("hold: working"), None, None, None) .await .unwrap(); wait_for( @@ -870,7 +874,7 @@ async fn a_send_accepted_by_a_stopping_runtime_is_redispatched() { let rig = rig(); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, request("hold: working"), None) + .dispatch(CHAT, HarnessId::Mock, request("hold: working"), None, None, None) .await .unwrap(); wait_for( @@ -882,7 +886,7 @@ async fn a_send_accepted_by_a_stopping_runtime_is_redispatched() { rig.core.sessions.interrupt(CHAT), rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, request("racing the stop"), None) + .dispatch(CHAT, HarnessId::Mock, request("racing the stop"), None, None, None) ); stop.unwrap(); send.unwrap(); @@ -916,7 +920,7 @@ async fn a_steer_sent_before_stop_is_cancelled_with_it() { let rig = rig(); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, request("hold: working"), None) + .dispatch(CHAT, HarnessId::Mock, request("hold: working"), None, None, None) .await .unwrap(); wait_for( @@ -968,7 +972,8 @@ async fn stop_ends_a_parked_runtime_holding_background_work() { HarnessId::Mock, request("launch a background scout"), None, - ) + None, + None) .await .unwrap(); settle(&rig, 1).await; @@ -1005,7 +1010,8 @@ async fn the_idle_reaper_spares_a_runtime_holding_background_work() { HarnessId::Mock, request("launch a background scout"), None, - ) + None, + None) .await .unwrap(); settle(&rig, 1).await; diff --git a/crates/engine/tests/runtime_survival_live.rs b/crates/engine/tests/runtime_survival_live.rs index 667b1f93e7..c0e3fdfe05 100644 --- a/crates/engine/tests/runtime_survival_live.rs +++ b/crates/engine/tests/runtime_survival_live.rs @@ -111,6 +111,8 @@ fn run( attachments: vec![], worktree: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } @@ -311,7 +313,8 @@ async fn agent_process_survives_sends_and_cancel() { &cwd, ), Some("m-2b-steer".into()), - ) + None, + None) .await .unwrap(); let steered_at = std::time::Instant::now(); diff --git a/crates/engine/tests/self_continued_quiesce.rs b/crates/engine/tests/self_continued_quiesce.rs index 707d38ee12..19e0e30651 100644 --- a/crates/engine/tests/self_continued_quiesce.rs +++ b/crates/engine/tests/self_continued_quiesce.rs @@ -60,6 +60,8 @@ fn run_request(prompt: &str) -> RunRequest { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } @@ -212,7 +214,7 @@ async fn self_continued_turn_parks_on_the_short_window() { let rig = assemble("watch the build"); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, run_request("watch the build"), None) + .dispatch(CHAT, HarnessId::Mock, run_request("watch the build"), None, None, None) .await .expect("dispatch"); @@ -258,7 +260,8 @@ async fn steered_turn_keeps_the_normal_window() { HarnessId::Mock, run_request("watch the build again"), None, - ) + None, + None) .await .expect("dispatch"); @@ -286,7 +289,7 @@ async fn steered_turn_keeps_the_normal_window() { // the short window (its Done is genuinely coming). rig.core .sessions - .steer(CHAT, "and then?", None) + .steer(CHAT, "and then?", None, None) .await .expect("steer accepted"); wait_for( diff --git a/crates/engine/tests/side_chats.rs b/crates/engine/tests/side_chats.rs index 9f68aa5320..221cb3ab8f 100644 --- a/crates/engine/tests/side_chats.rs +++ b/crates/engine/tests/side_chats.rs @@ -9,18 +9,39 @@ //! with main chats, and resets provider-session continuity so the copy //! never resumes the parent's conversation; //! - a side chat's first send bootstraps the provider session with the -//! conversation it lacks (the copied history), once per provider session, -//! never in front of a native command; +//! transcript it lacks — budgeted (t3-orchestration-v2 ticket 01): a +//! 16k-byte selection of the copied transcript with a coverage header +//! naming the omitted messages and the `read_chat` source chat — once per +//! provider session, never in front of a native command; //! - a fork whose first turn was a native command owes its copied history to //! the first ordinary send — warm dispatch, steer, or a cold resume — -//! exactly once, recorded durably per provider session. +//! exactly once, recorded durably per provider session; +//! - the fork of a chat whose boundary entry carries a provider turn ref +//! continues the SOURCE's provider session natively instead +//! (t3-orchestration-v2 ticket 17): the RPC records the fork source and +//! boundary, the first run's request carries them (Codex `thread/fork +//! {threadId, lastTurnId}`), nothing is pasted, and the budgeted +//! transcript rides the request as the refused-fork fallback. A missing +//! ref or a changed harness keeps the ticket-01 budgeted paste. The turn +//! refs themselves ride assistant entries via a harness event the engine +//! stamps at segment finish. Claude Code (ticket 18) speaks the same +//! contract with its own ref shape — the provider session id plus the +//! uuid of the turn's last top-level message frame — and its stand-in +//! tests pin that shape and the consumed-once intent on a ClaudeCode +//! harness. +//! - "Fork from here" (t3-orchestration-v2 ticket 21): the RPC's optional +//! `boundary` picks the reply to fork at — any settled assistant entry, +//! an earlier, errored or interrupted one — and the copy (and the budgeted +//! fallback) stop there, inclusive, while the native fork boundary becomes +//! exactly that reply's turn ref. Without it, today's behaviour stays: the +//! latest complete reply. use async_trait::async_trait; use futures::StreamExt; use futures::stream::BoxStream; use std::sync::{Arc, Mutex}; use roboco_doc::{MessagePart, MessageRole, MessageStatus, SessionMessageEntry}; -use roboco_engine::{EngineCore, HarnessRegistry}; +use roboco_engine::{transcript_budget::TRANSCRIPT_BUDGET_BYTES, EngineCore, HarnessRegistry}; use roboco_harness::{Harness, HarnessError, RunControls}; use roboco_proto::{ AgentEvent, DoneStatus, HarnessId, Model, ReasoningLevel, RunRequest, SandboxLevel, @@ -71,6 +92,7 @@ impl Harness for Capture { } fn message(id: &str, role: MessageRole, text: &str, status: MessageStatus) -> SessionMessageEntry { SessionMessageEntry { + origin_chat_id: None, duration_ms: None, id: id.into(), role, @@ -81,10 +103,24 @@ fn message(id: &str, role: MessageRole, text: &str, status: MessageStatus) -> Se created_at: 1, device_id: "device".into(), status: Some(status), + provider_turn: None, continuation_of: None, } } +/// An assistant entry carrying a provider turn ref, as a run that reports +/// one stamps it (ticket 17): the harness is the chat's own so the dispatch +/// below takes the native-fork path. +fn turn_entry(id: &str, text: &str, status: MessageStatus, turn: &str) -> SessionMessageEntry { + let mut entry = message(id, MessageRole::Assistant, text, status); + entry.provider_turn = Some(roboco_proto::ProviderTurnRef { + harness: HarnessId::Mock, + session_id: "parent-provider-session".into(), + turn_id: turn.into(), + }); + entry +} + #[tokio::test] async fn fork_is_frozen_durable_idempotent_and_has_an_independent_provider_session() { let dir = tempfile::tempdir().unwrap(); @@ -186,11 +222,14 @@ async fn fork_is_frozen_durable_idempotent_and_has_an_independent_provider_sessi sandbox: SandboxLevel::WorkspaceWrite, auto_approve: true, resume: None, + provider_fork: None, + resume_fallback_history: None, attachments: vec![], worktree: None, }, Some("side-user".into()), - ) + None, + None) .await .unwrap(); tokio::time::timeout(std::time::Duration::from_secs(5), async { @@ -261,169 +300,91 @@ async fn fork_is_frozen_durable_idempotent_and_has_an_independent_provider_sessi restarted.shutdown().await; } -#[tokio::test] -async fn cannot_fork_an_empty_chat_or_overwrite_a_main_chat() { - let dir = tempfile::tempdir().unwrap(); - let core = EngineCore::assemble( - dir.path(), - Arc::new(HarnessRegistry::new()), - HarnessId::Mock, - ) - .unwrap(); - core.workspace - .create_chat("main", None, Some(&core.device_id), None, None) - .unwrap(); - let client = roboco_rpc::memory_client(core.rpc_service()); - assert!( - client - .call( - methods::FORK_SIDE_CHAT, - serde_json::json!({ "chatId": "side", "sourceChatId": "main" }) - ) - .await - .is_err() - ); - assert!(core.workspace.chat("side").unwrap().is_none()); - assert!( - client - .call( - methods::FORK_SIDE_CHAT, - serde_json::json!({ "chatId": "main", "sourceChatId": "main" }) - ) - .await - .is_err() - ); +/// A harness that reports the provider turn ref the Codex driver emits +/// (ticket 17): one turn whose events name the provider session and turn. +struct RefCapture(Arc>>); +#[async_trait] +impl Harness for RefCapture { + fn id(&self) -> HarnessId { + HarnessId::Mock + } + fn display_name(&self) -> &str { + "RefCapture" + } + fn supports_steering(&self) -> bool { + false + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::TurnBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + _: RunControls, + ) -> Result>, HarnessError> { + self.0.lock().unwrap().push(request); + Ok(futures::stream::iter(vec![ + Ok(AgentEvent::SessionStarted { + harness: HarnessId::Mock, + model: "mock-1".into(), + tools: vec![], + cwd: "/tmp".into(), + session_id: "ref-provider-session".into(), + assistant_message_id: uuid::Uuid::new_v4().to_string(), + }), + Ok(AgentEvent::TurnRef { + reference: roboco_proto::ProviderTurnRef { + harness: HarnessId::Mock, + session_id: "ref-provider-session".into(), + turn_id: "turn-42".into(), + }, + }), + Ok(AgentEvent::TextDelta { + text: "Ref answer".into(), + }), + Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some("ref-provider-session".into()), + }), + ]) + .boxed()) + } } +/// Codex assistant entries carry a provider turn reference (ticket 17): the +/// harness event rides the stream and the engine stamps it on the entry the +/// turn streamed into, so a later fork finds its boundary there. #[tokio::test] -async fn forking_a_side_chat_can_land_as_a_sibling_under_the_main_chat() { +async fn assistant_entries_record_the_provider_turn_ref() { let dir = tempfile::tempdir().unwrap(); - let core = EngineCore::assemble( - dir.path(), - Arc::new(HarnessRegistry::new()), - HarnessId::Mock, - ) - .unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(RefCapture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); core.workspace - .create_chat("main", None, Some(&core.device_id), None, None) - .unwrap(); - let main = core.doc_host.open("main").unwrap(); - main.doc() - .push_message(&message( - "u1", - MessageRole::User, - "hi", - MessageStatus::Complete, - )) - .unwrap(); - main.doc() - .push_message(&message( - "a1", - MessageRole::Assistant, - "hello", - MessageStatus::Complete, - )) - .unwrap(); - let client = roboco_rpc::memory_client(core.rpc_service()); - // A first-level side chat, then its own fork button: the copy hangs - // under MAIN (the side chat's parent), not under the side chat. - let side = client - .call_as::( - methods::FORK_SIDE_CHAT, - serde_json::json!({ "chatId": "side", "sourceChatId": "main" }), - ) - .await - .unwrap(); - core.workspace.rename_chat("side", "Side quest").unwrap(); - assert_eq!(side.parent_chat_id.as_deref(), Some("main")); - // A turn of its own after the seam, so the seam sits inside the frozen - // prefix the next fork copies. - let side_doc = core.doc_host.open("side").unwrap(); - side_doc - .doc() - .push_message(&message( - "u2", - MessageRole::User, - "more", - MessageStatus::Complete, - )) - .unwrap(); - side_doc - .doc() - .push_message(&message( - "a2", - MessageRole::Assistant, - "sure", - MessageStatus::Complete, - )) - .unwrap(); - let sibling = client - .call_as::( - methods::FORK_SIDE_CHAT, - serde_json::json!({ - "chatId": "side-2", - "sourceChatId": "side", - "parentChatId": "main", - }), - ) - .await - .unwrap(); - assert_eq!(sibling.parent_chat_id.as_deref(), Some("main")); - let entries = core - .doc_host - .open("side-2") - .unwrap() - .doc() - .read_entries() - .unwrap(); - // Lineage survives: the side chat's own seam is copied, then a new one - // names the side chat (by its title at fork time) as this copy's source. - let seams: Vec<_> = entries - .iter() - .filter_map(|e| match e.parts.first() { - Some(roboco_doc::MessagePart::Fork { source_title, .. }) => Some(source_title.clone()), - _ => None, - }) - .collect(); - // The untitled main chat falls back to the placeholder title. - assert_eq!(seams, ["New session", "Side quest"]); - // An empty parent falls back to the source, like an omitted one. - let nested = client - .call_as::( - methods::FORK_SIDE_CHAT, - serde_json::json!({ - "chatId": "side-3", - "sourceChatId": "side", - "parentChatId": "", - }), + .create_chat( + "chat", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), ) - .await .unwrap(); - assert_eq!(nested.parent_chat_id.as_deref(), Some("side")); -} - -/// One turn through the engine; returns the request the provider received. -async fn side_turn( - core: &EngineCore, - requests: &Arc>>, - chat: &str, - prompt: &str, - message_id: &str, -) -> RunRequest { - let before = requests.lock().unwrap().len(); - let resume = core - .workspace - .chat(chat) - .unwrap() - .unwrap() - .harness_session_id; core.sessions .dispatch( - chat, + "chat", HarnessId::Mock, RunRequest { mcp: None, - prompt: prompt.into(), + prompt: "turn one".into(), harness: Some(HarnessId::Mock), model: None, reasoning: None, @@ -431,177 +392,71 @@ async fn side_turn( cwd: "/tmp".into(), sandbox: SandboxLevel::WorkspaceWrite, auto_approve: true, - resume, + resume: None, + provider_fork: None, + resume_fallback_history: None, attachments: vec![], worktree: None, }, - Some(message_id.into()), - ) + Some("u1".into()), + None, + None) .await .unwrap(); tokio::time::timeout(std::time::Duration::from_secs(5), async { - while requests.lock().unwrap().len() == before - || core - .sessions - .session_status(chat) - .is_some_and(|s| s.status != roboco_proto::SessionStatus::Idle) + while requests.lock().unwrap().is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while core + .sessions + .session_status("chat") + .is_some_and(|s| s.status != roboco_proto::SessionStatus::Idle) { tokio::time::sleep(std::time::Duration::from_millis(5)).await; } }) .await .unwrap(); - requests.lock().unwrap()[before].clone() + let entries = core + .doc_host + .open("chat") + .unwrap() + .doc() + .read_entries() + .unwrap(); + let assistant: Vec<_> = entries + .iter() + .filter(|e| e.role == MessageRole::Assistant) + .collect(); + assert_eq!(assistant.len(), 1, "{entries:?}"); + assert_eq!( + assistant[0].provider_turn.as_ref(), + Some(&roboco_proto::ProviderTurnRef { + harness: HarnessId::Mock, + session_id: "ref-provider-session".into(), + turn_id: "turn-42".into(), + }), + "the turn ref rides the entry it streamed into: {entries:?}" + ); + core.shutdown().await; } -/// A native command must lead the delivered prompt so the provider routes it -/// (Codex `command_request`, OpenCode commands): no bootstrap wrapper in front -/// of it, and none at all when there is no prior conversation. +/// The boundary entry's ref drives the fork: the RPC records the source +/// provider session and boundary, the first dispatch carries them to the +/// harness (Codex's `thread/fork {threadId, lastTurnId}`), nothing is +/// pasted, and the budgeted transcript rides the request as the refused-fork +/// fallback. #[tokio::test] -async fn native_commands_and_empty_side_chats_skip_the_history_wrapper() { +async fn a_fork_with_a_provider_turn_ref_forks_the_source_session() { let dir = tempfile::tempdir().unwrap(); let requests = Arc::new(Mutex::new(Vec::new())); let registry = HarnessRegistry::new(); registry.register(Arc::new(Capture(requests.clone()))); let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); - for (id, parent) in [ - ("main", None), - ("plus-review", Some("main")), - ("plus", Some("main")), - ] { - core.workspace - .create_chat_with_parent( - id, - None, - Some(&core.device_id), - None, - Some("/tmp".into()), - parent.map(str::to_owned), - ) - .unwrap(); - } - // A fresh `+` side chat: nothing to bootstrap. - let request = side_turn(&core, &requests, "plus-review", "/review", "u-review").await; - assert_eq!(request.prompt, "/review"); - let request = side_turn(&core, &requests, "plus", "hello there", "u-hello").await; - assert_eq!(request.prompt, "hello there"); - - // A fork whose first turn is a command: the command goes out bare, and - // the copied history rides the first ordinary turn, once. - let source = core.doc_host.open("main").unwrap(); - for (id, role, text) in [ - ("u1", MessageRole::User, "Remember PINEAPPLE"), - ("a1", MessageRole::Assistant, "I remember PINEAPPLE"), - ] { - source - .doc() - .push_message(&message(id, role, text, MessageStatus::Complete)) - .unwrap(); - } - let client = roboco_rpc::memory_client(core.rpc_service()); - client - .call( - methods::FORK_SIDE_CHAT, - serde_json::json!({ "chatId": "fork", "sourceChatId": "main" }), - ) - .await - .unwrap(); - let request = side_turn(&core, &requests, "fork", " /review tests", "f1").await; - assert_eq!(request.resume, None); - assert_eq!(request.prompt, " /review tests"); - let request = side_turn(&core, &requests, "fork", "What should I remember?", "f2").await; - assert_eq!(request.resume.as_deref(), Some("side-provider-session")); - assert!(request.prompt.contains("PINEAPPLE"), "{}", request.prompt); - assert!(request.prompt.ends_with("What should I remember?")); - assert!(!request.prompt.contains("/review"), "{}", request.prompt); - let request = side_turn(&core, &requests, "fork", "And now?", "f3").await; - assert_eq!(request.prompt, "And now?"); - core.shutdown().await; -} - -/// A provider runtime that stays alive between turns: later sends arrive -/// through the steering mailbox (the engine's warm dispatch), not new runs. -/// Each record: (prompt, resumed session, arrived through the mailbox). -type WarmLog = Arc, bool)>>>; -struct Warm(WarmLog); -#[async_trait] -impl Harness for Warm { - fn id(&self) -> HarnessId { - HarnessId::Mock - } - fn display_name(&self) -> &str { - "Warm" - } - fn supports_steering(&self) -> bool { - true - } - fn steering_mode(&self) -> SteeringMode { - SteeringMode::TurnBoundary - } - fn reasoning_levels(&self) -> &[ReasoningLevel] { - &[] - } - async fn models(&self) -> Result, HarnessError> { - Ok(vec![]) - } - async fn run( - &self, - request: RunRequest, - controls: RunControls, - ) -> Result>, HarnessError> { - self.0 - .lock() - .unwrap() - .push((request.prompt.clone(), request.resume.clone(), false)); - let turn = |text: &str| { - vec![ - Ok(AgentEvent::TextDelta { text: text.into() }), - Ok(AgentEvent::Done { - status: DoneStatus::Completed, - result: None, - error: None, - session_id: Some("warm-session".into()), - }), - ] - }; - let mut first = vec![Ok(AgentEvent::SessionStarted { - harness: HarnessId::Mock, - model: "mock-1".into(), - tools: vec![], - cwd: request.cwd.clone(), - session_id: "warm-session".into(), - assistant_message_id: uuid::Uuid::new_v4().to_string(), - })]; - first.extend(turn("first answer")); - let seen = self.0.clone(); - let later = futures::stream::unfold(controls.steering, move |mut steering| { - let seen = seen.clone(); - async move { - let message = steering.recv().await?; - seen.lock().unwrap().push((message.prompt, None, true)); - let mut events = vec![Ok(AgentEvent::Steered { - assistant_message_id: None, - next_assistant_message_id: Some(uuid::Uuid::new_v4().to_string()), - })]; - events.extend(turn("later answer")); - Some((futures::stream::iter(events), steering)) - } - }) - .flatten(); - Ok(futures::stream::iter(first).chain(later).boxed()) - } -} - -/// A fork whose first turn is a native command, continued in the SAME live -/// provider runtime: the copied history must ride the next ordinary send -/// (warm dispatch or steer), exactly once, and survive a cold resume. -#[tokio::test] -async fn warm_side_chat_sends_owed_fork_history_once() { - let dir = tempfile::tempdir().unwrap(); - let seen = Arc::new(Mutex::new(Vec::new())); - let registry = HarnessRegistry::new(); - registry.register(Arc::new(Warm(seen.clone()))); - let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); core.workspace .create_chat( "main", @@ -611,118 +466,103 @@ async fn warm_side_chat_sends_owed_fork_history_once() { Some("/tmp".into()), ) .unwrap(); + core.workspace + .set_chat_harness_session("main", "parent-provider-session", "/tmp"); let source = core.doc_host.open("main").unwrap(); - for (id, role, text) in [ - ("u1", MessageRole::User, "Remember PINEAPPLE"), - ("a1", MessageRole::Assistant, "I remember PINEAPPLE"), - ] { - source - .doc() - .push_message(&message(id, role, text, MessageStatus::Complete)) - .unwrap(); - } + source + .doc() + .push_message(&message( + "u1", + MessageRole::User, + "Remember PINEAPPLE", + MessageStatus::Complete, + )) + .unwrap(); + // The boundary entry: complete, and carrying the provider turn ref a + // Codex run would have stamped (harness = the chat's own harness here, + // so the dispatch below takes the native path). + let mut boundary = message( + "a1", + MessageRole::Assistant, + "I remember PINEAPPLE", + MessageStatus::Complete, + ); + boundary.provider_turn = Some(roboco_proto::ProviderTurnRef { + harness: HarnessId::Mock, + session_id: "parent-provider-session".into(), + turn_id: "turn-1".into(), + }); + source.doc().push_message(&boundary).unwrap(); let client = roboco_rpc::memory_client(core.rpc_service()); client .call( methods::FORK_SIDE_CHAT, - serde_json::json!({ "chatId": "fork", "sourceChatId": "main" }), + serde_json::json!({ "chatId": "side", "sourceChatId": "main" }), ) .await .unwrap(); - let request = |prompt: &str| RunRequest { - mcp: None, - prompt: prompt.into(), - harness: Some(HarnessId::Mock), - model: None, - reasoning: None, - model_options: Default::default(), - cwd: "/tmp".into(), - sandbox: SandboxLevel::WorkspaceWrite, - auto_approve: true, - resume: None, - attachments: vec![], - worktree: None, - }; - let settle = |count: usize| { - let seen = seen.clone(); - let core = &core; - async move { - tokio::time::timeout(std::time::Duration::from_secs(5), async { - while seen.lock().unwrap().len() < count - || core - .sessions - .session_status("fork") - .is_some_and(|s| s.status != roboco_proto::SessionStatus::Idle) - { - tokio::time::sleep(std::time::Duration::from_millis(5)).await; - } - }) - .await - .unwrap(); - seen.lock().unwrap()[count - 1].0.clone() - } - }; - let dispatch = |prompt: &'static str, id: &'static str| { - core.sessions - .dispatch("fork", HarnessId::Mock, request(prompt), Some(id.into())) - }; - dispatch("/review", "f1").await.unwrap(); - assert_eq!(settle(1).await, "/review"); - let doc = core.doc_host.open("fork").unwrap(); - assert_eq!( - doc.doc().fork_history_session(), - None, - "nothing delivered yet" - ); - // Warm dispatch into the live runtime: the owed history goes out. - dispatch("What should you remember?", "f2").await.unwrap(); - let warm = settle(2).await; - assert!(warm.contains("PINEAPPLE"), "{warm}"); - assert!(warm.ends_with("What should you remember?"), "{warm}"); - assert!(seen.lock().unwrap()[1].2, "delivered to the live runtime"); + + // The fork's first run: the native-fork directive rides the request and + // the delivered prompt carries no transcript. + let request = side_turn(&core, &requests, "side", "What did I ask?", "s1").await; assert_eq!( - doc.doc().fork_history_session().as_deref(), - Some("warm-session") + request.resume, None, + "the fork never resumes the source thread" ); - // Delivered: an explicit steer and later sends go out bare. - core.sessions - .steer("fork", "And now?", Some("f3".into())) - .await + let fork = request + .provider_fork + .expect("native fork directive rides the run"); + assert_eq!(fork.session_id, "parent-provider-session"); + assert_eq!(fork.last_turn, "turn-1"); + // Nothing pasted: the forked session carries the history natively. + assert_eq!(request.prompt, "What did I ask?", "{}", request.prompt); + // The budgeted transcript still rides the request as the refused-fork + // fallback (what the driver pastes onto a fresh session). + let fallback = fork.fallback_history.expect("fallback history attached"); + assert!(fallback.contains("PINEAPPLE"), "{fallback}"); + assert!(fallback.ends_with("\n\n"), "{fallback}"); + // The copied boundary keeps its ref, so a later fork of the fork still + // finds a boundary. + let copied = core + .doc_host + .open("side") + .unwrap() + .doc() + .read_entries() .unwrap(); - assert_eq!(settle(3).await, "And now?"); - assert!(seen.lock().unwrap()[2].2); - // A cold resume of the same provider session owes nothing either. - core.sessions.interrupt("fork").await.unwrap(); - dispatch("Again", "f4").await.unwrap(); - assert_eq!(settle(4).await, "Again"); - assert!(!seen.lock().unwrap()[3].2, "a new runtime"); assert_eq!( - seen.lock().unwrap()[3].1.as_deref(), - Some("warm-session"), - "resumed the provider session" + copied.iter().find(|e| e.id == "a1").unwrap().provider_turn, + boundary.provider_turn ); + + // The second run: the intent is spent, the fork's own session resumes, + // and nothing is pasted (the session already holds the history). + let request = side_turn(&core, &requests, "side", "And now?", "s2").await; + assert_eq!(request.provider_fork, None); + assert_eq!(request.resume.as_deref(), Some("side-provider-session")); + assert_eq!(request.prompt, "And now?", "{}", request.prompt); core.shutdown().await; } -/// A runtime that answers its first turn, then exits once released without -/// reading its mailbox: sends routed into it are orphaned and re-dispatched. -struct Dropping { - runs: Arc)>>>, - release: Arc, -} +/// A Claude Code stand-in (t3-orchestration-v2 ticket 18): one run that +/// reports the fork's NEW session id (what the CLI's init frame carries) +/// and a provider turn ref — the uuid of the last top-level message frame +/// of the turn, exactly the event the Claude driver emits — capturing the +/// request it ran on. +struct ClaudeCapture(Arc>>); #[async_trait] -impl Harness for Dropping { +impl Harness for ClaudeCapture { fn id(&self) -> HarnessId { - HarnessId::Mock + HarnessId::ClaudeCode } fn display_name(&self) -> &str { - "Dropping" + "ClaudeCapture" } fn supports_steering(&self) -> bool { true } fn steering_mode(&self) -> SteeringMode { - SteeringMode::TurnBoundary + SteeringMode::StepBoundary } fn reasoning_levels(&self) -> &[ReasoningLevel] { &[] @@ -733,58 +573,2112 @@ impl Harness for Dropping { async fn run( &self, request: RunRequest, - controls: RunControls, + _: RunControls, ) -> Result>, HarnessError> { - let first = { - let mut runs = self.runs.lock().unwrap(); - runs.push((request.prompt.clone(), request.resume.clone())); - runs.len() == 1 - }; - let events = vec![ + self.0.lock().unwrap().push(request); + Ok(futures::stream::iter(vec![ Ok(AgentEvent::SessionStarted { - harness: HarnessId::Mock, - model: "mock-1".into(), + harness: HarnessId::ClaudeCode, + model: "claude-fable-5".into(), tools: vec![], - cwd: request.cwd.clone(), - session_id: "drop-session".into(), + cwd: "/tmp".into(), + session_id: "claude-fork-new-session".into(), assistant_message_id: uuid::Uuid::new_v4().to_string(), }), + Ok(AgentEvent::TurnRef { + reference: roboco_proto::ProviderTurnRef { + harness: HarnessId::ClaudeCode, + session_id: "claude-fork-new-session".into(), + turn_id: "msg-uuid-9".into(), + }, + }), Ok(AgentEvent::TextDelta { - text: "answer".into(), + text: "Claude fork answer".into(), }), Ok(AgentEvent::Done { status: DoneStatus::Completed, result: None, error: None, - session_id: Some("drop-session".into()), + session_id: Some("claude-fork-new-session".into()), }), - ]; - let release = self.release.clone(); - // The first runtime lingers (mailbox unread) until released, then - // ends; later ones end after their turn. - let tail = futures::stream::once(async move { - let _mailbox = controls.steering; - if first { - release.notified().await; - } - }) + ]) + .boxed()) + } +} + +/// Claude Code assistant entries carry the provider turn reference +/// (ticket 18): the harness event rides the stream and the engine stamps +/// it on the entry the turn streamed into — the session id plus the uuid +/// of the turn's last top-level message frame — so a later fork finds its +/// boundary there. +#[tokio::test] +async fn claude_assistant_entries_record_the_provider_turn_ref() { + let dir = tempfile::tempdir().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(ClaudeCapture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::ClaudeCode).unwrap(); + core.workspace + .create_chat( + "chat", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + core.sessions + .dispatch( + "chat", + HarnessId::ClaudeCode, + RunRequest { + mcp: None, + prompt: "turn one".into(), + harness: Some(HarnessId::ClaudeCode), + model: None, + reasoning: None, + model_options: Default::default(), + cwd: "/tmp".into(), + sandbox: SandboxLevel::WorkspaceWrite, + auto_approve: true, + resume: None, + provider_fork: None, + resume_fallback_history: None, + attachments: vec![], + worktree: None, + }, + Some("u1".into()), + None, + None) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while requests.lock().unwrap().is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while core + .sessions + .session_status("chat") + .is_some_and(|s| s.status != roboco_proto::SessionStatus::Idle) + { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + let entries = core + .doc_host + .open("chat") + .unwrap() + .doc() + .read_entries() + .unwrap(); + let assistant: Vec<_> = entries + .iter() + .filter(|e| e.role == MessageRole::Assistant) + .collect(); + assert_eq!(assistant.len(), 1, "{entries:?}"); + assert_eq!( + assistant[0].provider_turn.as_ref(), + Some(&roboco_proto::ProviderTurnRef { + harness: HarnessId::ClaudeCode, + session_id: "claude-fork-new-session".into(), + turn_id: "msg-uuid-9".into(), + }), + "the turn ref rides the entry it streamed into: {entries:?}" + ); + core.shutdown().await; +} + +/// A Claude Code fork's first run carries the source session and the +/// boundary message uuid to the driver (ticket 18): `--resume +/// --fork-session` (plus `--resume-session-at ` at the driver when the +/// boundary is an earlier turn), nothing pasted, the budgeted transcript as +/// the refused-fork fallback — and the intent is consumed once, with the +/// fork's own NEW session resuming afterwards. +#[tokio::test] +async fn a_claude_fork_carries_the_source_session_and_boundary_uuid_to_its_first_run() { + let dir = tempfile::tempdir().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(ClaudeCapture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::ClaudeCode).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + core.workspace + .set_chat_harness_session("main", "claude-source-session", "/tmp"); + let source = core.doc_host.open("main").unwrap(); + source + .doc() + .push_message(&message( + "u1", + MessageRole::User, + "Remember PINEAPPLE", + MessageStatus::Complete, + )) + .unwrap(); + // The boundary reply, stamped the way a Claude run does: the provider + // session plus the uuid of the turn's last top-level message frame. + let mut boundary = message( + "a1", + MessageRole::Assistant, + "I remember PINEAPPLE", + MessageStatus::Complete, + ); + boundary.provider_turn = Some(roboco_proto::ProviderTurnRef { + harness: HarnessId::ClaudeCode, + session_id: "claude-source-session".into(), + turn_id: "msg-uuid-7".into(), + }); + source.doc().push_message(&boundary).unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "side", "sourceChatId": "main" }), + ) + .await + .unwrap(); + + // The fork's first run: the native-fork directive names the source + // session and the boundary message uuid, and the delivered prompt + // carries no transcript — the forked session holds the history. + let request = side_turn_on_harness( + &core, + &requests, + "side", + "What did I ask?", + "s1", + HarnessId::ClaudeCode, + ) + .await; + assert_eq!( + request.resume, None, + "the fork never resumes the source session as a plain resume" + ); + let fork = request + .provider_fork + .expect("native fork directive rides the run"); + assert_eq!(fork.session_id, "claude-source-session"); + assert_eq!(fork.last_turn, "msg-uuid-7"); + assert_eq!(request.prompt, "What did I ask?", "{}", request.prompt); + // The budgeted transcript rides the request as the refused-fork + // fallback (what the driver pastes onto the fresh session the CLI + // starts when it rejects the flags or the session is gone). + let fallback = fork.fallback_history.expect("fallback history attached"); + assert!(fallback.contains("PINEAPPLE"), "{fallback}"); + assert!(fallback.ends_with("\n\n"), "{fallback}"); + + // The second run: the intent is spent, and the fork's own NEW session + // (the init frame's id) resumes — nothing pasted. + let request = side_turn_on_harness( + &core, + &requests, + "side", + "And now?", + "s2", + HarnessId::ClaudeCode, + ) + .await; + assert_eq!(request.provider_fork, None); + assert_eq!( + request.resume.as_deref(), + Some("claude-fork-new-session"), + "the fork's own session resumes" + ); + assert_eq!(request.prompt, "And now?", "{}", request.prompt); + core.shutdown().await; +} + +/// Old chats (no refs) and a changed harness fall back to the budgeted paste +/// from ticket 01: no native-fork directive, history in front of the prompt. +#[tokio::test] +async fn forks_without_refs_or_with_a_changed_harness_paste_the_budget() { + let dir = tempfile::tempdir().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Capture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "plain", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + let source = core.doc_host.open("plain").unwrap(); + source + .doc() + .push_message(&message( + "u1", + MessageRole::User, + "Remember PINEAPPLE", + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&message( + "a1", + MessageRole::Assistant, + "I remember PINEAPPLE", + MessageStatus::Complete, + )) + .unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + // An old chat: no provider turn ref on the boundary entry. + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "old-fork", "sourceChatId": "plain" }), + ) + .await + .unwrap(); + let request = side_turn(&core, &requests, "old-fork", "What did I ask?", "o1").await; + assert_eq!(request.provider_fork, None, "no ref, no native fork"); + assert!(request.prompt.contains("PINEAPPLE"), "{}", request.prompt); + assert!(request.prompt.ends_with("What did I ask?")); + + // A ref recorded for a DIFFERENT harness: the run's harness spends the + // intent without ever taking the native path. + let mut boundary = message( + "a2", + MessageRole::Assistant, + "answer with a foreign ref", + MessageStatus::Complete, + ); + boundary.provider_turn = Some(roboco_proto::ProviderTurnRef { + harness: HarnessId::Codex, + session_id: "codex-thread".into(), + turn_id: "turn-9".into(), + }); + source.doc().push_message(&boundary).unwrap(); + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "switched-fork", "sourceChatId": "plain" }), + ) + .await + .unwrap(); + let request = side_turn(&core, &requests, "switched-fork", "What did I ask?", "c1").await; + assert_eq!(request.provider_fork, None, "a changed harness never forks"); + assert!(request.prompt.contains("PINEAPPLE"), "{}", request.prompt); + core.shutdown().await; +} + +/// "Fork from here" on an earlier reply (t3-orchestration-v2 ticket 21): +/// the copy stops at the chosen entry, inclusive, and the first run's +/// native-fork boundary is THAT entry's turn ref — not the latest one's. +/// The budgeted fallback history is cut at the boundary too, so the fork +/// never sees later messages even when the harness refuses the fork. +#[tokio::test] +async fn forking_from_an_earlier_reply_cuts_the_copy_and_forks_at_that_turn() { + let dir = tempfile::tempdir().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Capture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + core.workspace + .set_chat_harness_session("main", "parent-provider-session", "/tmp"); + let source = core.doc_host.open("main").unwrap(); + source + .doc() + .push_message(&message( + "u1", + MessageRole::User, + "Remember PINEAPPLE", + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&turn_entry( + "a1", + "I remember PINEAPPLE", + MessageStatus::Complete, + "turn-1", + )) + .unwrap(); + source + .doc() + .push_message(&message( + "u2", + MessageRole::User, + "And the code?", + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&turn_entry( + "a2", + "the code is fine", + MessageStatus::Complete, + "turn-2", + )) + .unwrap(); + source + .doc() + .push_message(&message( + "u3", + MessageRole::User, + "ship it", + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&turn_entry( + "a3", + "shipping", + MessageStatus::Complete, + "turn-3", + )) + .unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + + // Fork from the FIRST reply. + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "side-early", "sourceChatId": "main", "boundary": "a1" }), + ) + .await + .unwrap(); + // The transcript ends at that reply: only u1, a1, and the seam — the + // fork never sees u2, a2, u3, or a3. + let ids = |chat: &str| { + core.doc_host + .open(chat) + .unwrap() + .doc() + .read_entries() + .unwrap() + .into_iter() + .map(|entry| entry.id) + .collect::>() + }; + assert_eq!( + ids("side-early"), + vec!["u1".to_string(), "a1".to_string(), "fork:side-early".to_string()], + "the copy stops at the boundary, inclusive" + ); + // The copied boundary keeps its ref, so a later fork of the fork still + // finds a boundary. + let copied = core + .doc_host + .open("side-early") + .unwrap() + .doc() + .read_entries() + .unwrap(); + assert_eq!( + copied.iter().find(|e| e.id == "a1").unwrap().provider_turn, + Some(roboco_proto::ProviderTurnRef { + harness: HarnessId::Mock, + session_id: "parent-provider-session".into(), + turn_id: "turn-1".into(), + }) + ); + + // The first run forks natively at the CHOSEN boundary… + let request = side_turn(&core, &requests, "side-early", "What did I ask?", "s1").await; + let fork = request + .provider_fork + .expect("native fork at the chosen boundary"); + assert_eq!(fork.session_id, "parent-provider-session"); + assert_eq!(fork.last_turn, "turn-1"); + // …nothing is pasted onto the prompt… + assert_eq!(request.prompt, "What did I ask?", "{}", request.prompt); + // …and the refused-fork fallback is cut at the boundary: it holds the + // copied prefix and never a later message. + let fallback = fork.fallback_history.expect("fallback history attached"); + assert!(fallback.contains("PINEAPPLE"), "{fallback}"); + assert!(!fallback.contains("the code is fine"), "{fallback}"); + assert!(!fallback.contains("shipping"), "{fallback}"); + + // Without a boundary, today's behaviour stays: the fork continues from + // the LATEST reply. + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "side-latest", "sourceChatId": "main" }), + ) + .await + .unwrap(); + assert_eq!( + ids("side-latest"), + vec![ + "u1".to_string(), + "a1".to_string(), + "u2".to_string(), + "a2".to_string(), + "u3".to_string(), + "a3".to_string(), + "fork:side-latest".to_string(), + ], + "no boundary copies through the latest reply" + ); + let request = side_turn(&core, &requests, "side-latest", "What next?", "s2").await; + let fork = request + .provider_fork + .expect("the latest reply's ref drives the default fork"); + assert_eq!(fork.last_turn, "turn-3"); + core.shutdown().await; +} + +/// Errored and interrupted replies are forkable (ticket 21): an errored +/// turn is a COMPLETE entry with an error part, an interrupted one an +/// ABORTED entry — both keep the turn ref the run stamped before it ended, +/// and a fork at either continues the source session natively at exactly +/// that turn. +#[tokio::test] +async fn errored_and_interrupted_replies_can_be_forked() { + let dir = tempfile::tempdir().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Capture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + core.workspace + .set_chat_harness_session("main", "parent-provider-session", "/tmp"); + let source = core.doc_host.open("main").unwrap(); + source + .doc() + .push_message(&turn_entry( + "a1", + "all good", + MessageStatus::Complete, + "turn-1", + )) + .unwrap(); + // An errored reply: complete status with an error part, ref stamped. + let mut errored = turn_entry("a2", "", MessageStatus::Complete, "turn-2"); + errored.parts = vec![ + MessagePart::Text { + id: "a2-text".into(), + text: "half an answer".into(), + }, + MessagePart::Error { + id: "a2-err".into(), + message: "the model blew up".into(), + }, + ]; + source.doc().push_message(&errored).unwrap(); + source + .doc() + .push_message(&message( + "u1", + MessageRole::User, + "retry?", + MessageStatus::Complete, + )) + .unwrap(); + // An interrupted reply: aborted status, ref stamped before the stop. + source + .doc() + .push_message(&turn_entry( + "a3", + "partial answer", + MessageStatus::Aborted, + "turn-3", + )) + .unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + + // Fork at the ERRORED reply. + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "side-err", "sourceChatId": "main", "boundary": "a2" }), + ) + .await + .unwrap(); + let ids = |chat: &str| { + core.doc_host + .open(chat) + .unwrap() + .doc() + .read_entries() + .unwrap() + .into_iter() + .map(|entry| entry.id) + .collect::>() + }; + assert_eq!( + ids("side-err"), + vec![ + "a1".to_string(), + "a2".to_string(), + "fork:side-err".to_string(), + ], + "the errored reply is the boundary, inclusive" + ); + let request = side_turn(&core, &requests, "side-err", "What went wrong?", "e1").await; + let fork = request + .provider_fork + .expect("the errored turn's ref drives the native fork"); + assert_eq!(fork.last_turn, "turn-2"); + + // Fork at the INTERRUPTED reply — the boundary a default fork would + // never pick (it is aborted, not complete). + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "side-int", "sourceChatId": "main", "boundary": "a3" }), + ) + .await + .unwrap(); + assert_eq!( + ids("side-int"), + vec![ + "a1".to_string(), + "a2".to_string(), + "u1".to_string(), + "a3".to_string(), + "fork:side-int".to_string(), + ], + "the interrupted reply is the boundary, inclusive" + ); + let request = side_turn(&core, &requests, "side-int", "Continue", "i1").await; + let fork = request + .provider_fork + .expect("the interrupted turn's ref drives the native fork"); + assert_eq!(fork.last_turn, "turn-3"); + core.shutdown().await; +} + +/// A boundary that is not a settled assistant reply is refused, and no +/// chat row appears: an unknown id, a user entry, and a reply still +/// streaming (its run is writing it, and its turn ref is not stamped yet). +#[tokio::test] +async fn an_invalid_boundary_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let core = EngineCore::assemble( + dir.path(), + Arc::new(HarnessRegistry::new()), + HarnessId::Mock, + ) + .unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + let source = core.doc_host.open("main").unwrap(); + source + .doc() + .push_message(&message( + "u1", + MessageRole::User, + "hi", + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&turn_entry( + "a1", + "hello", + MessageStatus::Complete, + "turn-1", + )) + .unwrap(); + source + .doc() + .push_message(&turn_entry( + "a2", + "still going…", + MessageStatus::Streaming, + "turn-2", + )) + .unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + for boundary in ["nope", "u1", "a2"] { + let error = client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": "side-bad", + "sourceChatId": "main", + "boundary": boundary, + }), + ) + .await + .unwrap_err(); + assert!( + error.to_string().contains("settled assistant reply"), + "boundary {boundary}: {error}" + ); + assert!(core.workspace.chat("side-bad").unwrap().is_none()); + } + core.shutdown().await; +} + +/// A boundary without a provider turn ref (an old chat, or a run that never +/// reported one) falls back to the budgeted paste — cut at the boundary, +/// so the pasted history never carries a later message (ticket 21's rule: +/// the fork never sees anything after the boundary, whichever path it +/// takes). +#[tokio::test] +async fn a_boundary_without_a_ref_pastes_the_budget_up_to_the_boundary() { + let dir = tempfile::tempdir().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Capture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + let source = core.doc_host.open("main").unwrap(); + source + .doc() + .push_message(&message( + "u1", + MessageRole::User, + "Remember PINEAPPLE", + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&message( + "a1", + MessageRole::Assistant, + "I remember PINEAPPLE", + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&message( + "u2", + MessageRole::User, + "And the code?", + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&message( + "a2", + MessageRole::Assistant, + "the code is fine", + MessageStatus::Complete, + )) + .unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "side-early", "sourceChatId": "main", "boundary": "a1" }), + ) + .await + .unwrap(); + let request = side_turn(&core, &requests, "side-early", "What did I ask?", "s1").await; + assert_eq!(request.provider_fork, None, "no ref, no native fork"); + assert!(request.prompt.contains("PINEAPPLE"), "{}", request.prompt); + assert!( + !request.prompt.contains("the code is fine"), + "the paste is cut at the boundary: {}", + request.prompt + ); + assert!(request.prompt.ends_with("What did I ask?")); + core.shutdown().await; +} + +/// A logical message can span continuation fragments after its root +/// (oversized relayed replies). A boundary on the root copies the WHOLE +/// message — never a truncated prefix — and the native boundary ref comes +/// from the root, where a run stamps it. +/// "Fork from here" on a segment that is NOT its turn's last (story 60: a +/// steer-split reply): the turn's later segments share the boundary's +/// provider turn ref, and a native fork cuts at the WHOLE turn — content +/// the boundary excluded would ride the copy through the provider. The +/// mid-turn boundary falls back to the budgeted paste; the turn's LAST +/// segment forks natively. +#[tokio::test] +async fn a_fork_from_a_mid_turn_segment_falls_back_to_the_budgeted_paste() { + let dir = tempfile::tempdir().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Capture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + core.workspace + .set_chat_harness_session("main", "parent-provider-session", "/tmp"); + let source = core.doc_host.open("main").unwrap(); + source + .doc() + .push_message(&message( + "u1", + MessageRole::User, + "Remember PINEAPPLE", + MessageStatus::Complete, + )) + .unwrap(); + // A steer-split reply: two entries, ONE provider turn (the same ref on + // both — what the engine stamps when a steer splits the reply). + source + .doc() + .push_message(&turn_entry( + "a1", + "first half of the reply", + MessageStatus::Complete, + "turn-1", + )) + .unwrap(); + source + .doc() + .push_message(&turn_entry( + "a2", + "second half after the steer", + MessageStatus::Complete, + "turn-1", + )) + .unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + // Fork from the EARLIER segment: the turn continues past it, so no + // native intent — the budgeted paste carries the prefix instead. + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": "side-mid", + "sourceChatId": "main", + "boundary": "a1", + }), + ) + .await + .unwrap(); + let request = side_turn(&core, &requests, "side-mid", "Continue", "s1").await; + assert_eq!( + request.provider_fork, None, + "a mid-turn boundary must not take the native fork: the later \n content would ride the provider session" + ); + assert!( + request.prompt.contains("PINEAPPLE"), + "the budgeted paste covers the copy: {}", + request.prompt + ); + assert!( + !request.prompt.contains("second half after the steer"), + "the copy stops at the boundary: {}", + request.prompt + ); + // Fork from the turn's LAST segment: native, at that turn. + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": "side-last", + "sourceChatId": "main", + "boundary": "a2", + }), + ) + .await + .unwrap(); + let request = side_turn(&core, &requests, "side-last", "Continue", "s2").await; + let fork = request.provider_fork.expect("the last segment forks natively"); + assert_eq!(fork.session_id, "parent-provider-session"); + assert_eq!(fork.last_turn, "turn-1"); + core.shutdown().await; +} + +/// A fork whose copy lands on the BOUNDARY's harness while the source's +/// CURRENT config runs another (a chat that switched since the boundary +/// turn): the config the copy inherits belongs to the current harness, so +/// its model/reasoning/options ids are the wrong vocabulary — they drop, +/// exactly like a cross-harness fork (the sandbox survives). +#[tokio::test] +async fn a_fork_onto_the_boundarys_harness_drops_the_current_configs_picks() { + let dir = tempfile::tempdir().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Capture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + // The source's CURRENT config: another harness, with a model only that + // harness knows. + core.workspace + .set_chat_config( + "main", + &roboco_proto::ChatConfig { + harness: HarnessId::ClaudeCode, + model: Some("claude-sonnet".into()), + reasoning: Some(roboco_proto::ReasoningLevel::High), + model_options: serde_json::Map::from_iter([( + "claudeOption".into(), + serde_json::json!("on"), + )]), + sandbox: roboco_proto::SandboxLevel::WorkspaceWrite, + }, + ) + .unwrap(); + let source = core.doc_host.open("main").unwrap(); + source + .doc() + .push_message(&message( + "u1", + MessageRole::User, + "Remember PINEAPPLE", + MessageStatus::Complete, + )) + .unwrap(); + // The boundary's ref names the chat's OWN (registered) harness — the + // override onto it is NOT a cross-harness fork by the ref's judgement, + // but the config it inherits is the claude one. + source + .doc() + .push_message(&turn_entry( + "a1", + "I remember PINEAPPLE", + MessageStatus::Complete, + "turn-1", + )) + .unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": "side", + "sourceChatId": "main", + "harness": "mock", + }), + ) + .await + .unwrap(); + // The copy's config: the boundary's harness, none of the current + // config's vocabulary, sandbox kept. + let row = core.workspace.chat("side").unwrap().unwrap(); + let config = row.config.expect("the copy carries a config"); + assert_eq!(config.harness, HarnessId::Mock); + assert_eq!( + config.model, None, + "the claude model must not ride the mock copy" + ); + assert_eq!(config.reasoning, None, "the claude reasoning drops with it"); + assert!( + config.model_options.is_empty(), + "the claude-only option ids drop with it" + ); + assert_eq!(config.sandbox, roboco_proto::SandboxLevel::WorkspaceWrite); + // The run asks for no model the mock harness does not know. + let request = side_turn(&core, &requests, "side", "Continue", "s1").await; + assert_eq!(request.model, None, "the run inherits the dropped pick"); + core.shutdown().await; +} + +#[tokio::test] +async fn a_split_boundary_entry_copies_whole() { + let dir = tempfile::tempdir().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Capture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + core.workspace + .set_chat_harness_session("main", "parent-provider-session", "/tmp"); + let source = core.doc_host.open("main").unwrap(); + let root = turn_entry("a1", "first half", MessageStatus::Complete, "turn-1"); + source.doc().push_message(&root).unwrap(); + let mut fragment = message( + "a1#c1", + MessageRole::Assistant, + "second half", + MessageStatus::Complete, + ); + fragment.continuation_of = Some("a1".into()); + source.doc().push_message(&fragment).unwrap(); + // A second fragment — fragments name the ROOT, not the fragment before + // them, so a walk that steps back one entry at a time stops at the + // first fragment and loses the root (and its turn ref). Three entries + // make the chain long enough to tell. + let mut second_fragment = message( + "a1#c2", + MessageRole::Assistant, + "third half", + MessageStatus::Complete, + ); + second_fragment.continuation_of = Some("a1".into()); + source.doc().push_message(&second_fragment).unwrap(); + source + .doc() + .push_message(&turn_entry( + "a2", + "later reply", + MessageStatus::Complete, + "turn-2", + )) + .unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + // The root id and EITHER fragment id resolve to the whole message. + for (chat, boundary) in [ + ("side-root", "a1"), + ("side-fragment", "a1#c1"), + ("side-second-fragment", "a1#c2"), + ] { + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": chat, + "sourceChatId": "main", + "boundary": boundary, + }), + ) + .await + .unwrap(); + let ids: Vec = core + .doc_host + .open(chat) + .unwrap() + .doc() + .read_entries() + .unwrap() + .into_iter() + .map(|entry| entry.id) + .collect(); + assert_eq!( + ids, + vec![ + "a1".to_string(), + "a1#c1".to_string(), + "a1#c2".to_string(), + format!("fork:{chat}"), + ], + "boundary {boundary} copies the whole logical message" + ); + let request = side_turn(&core, &requests, chat, "Continue", "s1").await; + let fork = request.provider_fork.expect("the root's ref drives the fork"); + assert_eq!(fork.last_turn, "turn-1", "boundary {boundary}"); + } + core.shutdown().await; +} + +#[tokio::test] +async fn cannot_fork_an_empty_chat_or_overwrite_a_main_chat() { + let dir = tempfile::tempdir().unwrap(); + let core = EngineCore::assemble( + dir.path(), + Arc::new(HarnessRegistry::new()), + HarnessId::Mock, + ) + .unwrap(); + core.workspace + .create_chat("main", None, Some(&core.device_id), None, None) + .unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + assert!( + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "side", "sourceChatId": "main" }) + ) + .await + .is_err() + ); + assert!(core.workspace.chat("side").unwrap().is_none()); + assert!( + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "main", "sourceChatId": "main" }) + ) + .await + .is_err() + ); +} + +#[tokio::test] +async fn forking_a_side_chat_can_land_as_a_sibling_under_the_main_chat() { + let dir = tempfile::tempdir().unwrap(); + let core = EngineCore::assemble( + dir.path(), + Arc::new(HarnessRegistry::new()), + HarnessId::Mock, + ) + .unwrap(); + core.workspace + .create_chat("main", None, Some(&core.device_id), None, None) + .unwrap(); + let main = core.doc_host.open("main").unwrap(); + main.doc() + .push_message(&message( + "u1", + MessageRole::User, + "hi", + MessageStatus::Complete, + )) + .unwrap(); + main.doc() + .push_message(&message( + "a1", + MessageRole::Assistant, + "hello", + MessageStatus::Complete, + )) + .unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + // A first-level side chat, then its own fork button: the copy hangs + // under MAIN (the side chat's parent), not under the side chat. + let side = client + .call_as::( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "side", "sourceChatId": "main" }), + ) + .await + .unwrap(); + core.workspace.rename_chat("side", "Side quest").unwrap(); + assert_eq!(side.parent_chat_id.as_deref(), Some("main")); + // A turn of its own after the seam, so the seam sits inside the frozen + // prefix the next fork copies. + let side_doc = core.doc_host.open("side").unwrap(); + side_doc + .doc() + .push_message(&message( + "u2", + MessageRole::User, + "more", + MessageStatus::Complete, + )) + .unwrap(); + side_doc + .doc() + .push_message(&message( + "a2", + MessageRole::Assistant, + "sure", + MessageStatus::Complete, + )) + .unwrap(); + let sibling = client + .call_as::( + methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": "side-2", + "sourceChatId": "side", + "parentChatId": "main", + }), + ) + .await + .unwrap(); + assert_eq!(sibling.parent_chat_id.as_deref(), Some("main")); + let entries = core + .doc_host + .open("side-2") + .unwrap() + .doc() + .read_entries() + .unwrap(); + // Lineage survives: the side chat's own seam is copied, then a new one + // names the side chat (by its title at fork time) as this copy's source. + let seams: Vec<_> = entries + .iter() + .filter_map(|e| match e.parts.first() { + Some(roboco_doc::MessagePart::Fork { source_title, .. }) => Some(source_title.clone()), + _ => None, + }) + .collect(); + // The untitled main chat falls back to the placeholder title. + assert_eq!(seams, ["New session", "Side quest"]); + // An empty parent falls back to the source, like an omitted one. + let nested = client + .call_as::( + methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": "side-3", + "sourceChatId": "side", + "parentChatId": "", + }), + ) + .await + .unwrap(); + assert_eq!(nested.parent_chat_id.as_deref(), Some("side")); +} + +/// One turn through the engine; returns the request the provider received. +async fn side_turn( + core: &EngineCore, + requests: &Arc>>, + chat: &str, + prompt: &str, + message_id: &str, +) -> RunRequest { + side_turn_on_harness(core, requests, chat, prompt, message_id, HarnessId::Mock).await +} + +/// [`side_turn`] on an explicit harness — the Claude Code fork tests +/// dispatch on the stand-in registered under that id. +async fn side_turn_on_harness( + core: &EngineCore, + requests: &Arc>>, + chat: &str, + prompt: &str, + message_id: &str, + harness: HarnessId, +) -> RunRequest { + let before = requests.lock().unwrap().len(); + let resume = core + .workspace + .chat(chat) + .unwrap() + .unwrap() + .harness_session_id; + core.sessions + .dispatch( + chat, + harness, + RunRequest { + mcp: None, + prompt: prompt.into(), + harness: Some(harness), + model: None, + reasoning: None, + model_options: Default::default(), + cwd: "/tmp".into(), + sandbox: SandboxLevel::WorkspaceWrite, + auto_approve: true, + resume, + provider_fork: None, + resume_fallback_history: None, + attachments: vec![], + worktree: None, + }, + Some(message_id.into()), + None, + None) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while requests.lock().unwrap().len() == before + || core + .sessions + .session_status(chat) + .is_some_and(|s| s.status != roboco_proto::SessionStatus::Idle) + { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + requests.lock().unwrap()[before].clone() +} + +/// A native command must lead the delivered prompt so the provider routes it +/// (Codex `command_request`, OpenCode commands): no bootstrap wrapper in front +/// of it, and none at all when there is no prior conversation. +#[tokio::test] +async fn native_commands_and_empty_side_chats_skip_the_history_wrapper() { + let dir = tempfile::tempdir().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Capture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + for (id, parent) in [ + ("main", None), + ("plus-review", Some("main")), + ("plus", Some("main")), + ] { + core.workspace + .create_chat_with_parent( + id, + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + parent.map(str::to_owned), + None, + ) + .unwrap(); + } + // A fresh `+` side chat: nothing to bootstrap. + let request = side_turn(&core, &requests, "plus-review", "/review", "u-review").await; + assert_eq!(request.prompt, "/review"); + let request = side_turn(&core, &requests, "plus", "hello there", "u-hello").await; + assert_eq!(request.prompt, "hello there"); + + // A fork whose first turn is a command: the command goes out bare, and + // the copied history rides the first ordinary turn, once. + let source = core.doc_host.open("main").unwrap(); + for (id, role, text) in [ + ("u1", MessageRole::User, "Remember PINEAPPLE"), + ("a1", MessageRole::Assistant, "I remember PINEAPPLE"), + ] { + source + .doc() + .push_message(&message(id, role, text, MessageStatus::Complete)) + .unwrap(); + } + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "fork", "sourceChatId": "main" }), + ) + .await + .unwrap(); + let request = side_turn(&core, &requests, "fork", " /review tests", "f1").await; + assert_eq!(request.resume, None); + assert_eq!(request.prompt, " /review tests"); + let request = side_turn(&core, &requests, "fork", "What should I remember?", "f2").await; + assert_eq!(request.resume.as_deref(), Some("side-provider-session")); + assert!(request.prompt.contains("PINEAPPLE"), "{}", request.prompt); + assert!(request.prompt.ends_with("What should I remember?")); + assert!(!request.prompt.contains("/review"), "{}", request.prompt); + let request = side_turn(&core, &requests, "fork", "And now?", "f3").await; + assert_eq!(request.prompt, "And now?"); + core.shutdown().await; +} + +/// A provider runtime that stays alive between turns: later sends arrive +/// through the steering mailbox (the engine's warm dispatch), not new runs. +/// Each record: (prompt, resumed session, arrived through the mailbox). +type WarmLog = Arc, bool)>>>; +struct Warm(WarmLog); +#[async_trait] +impl Harness for Warm { + fn id(&self) -> HarnessId { + HarnessId::Mock + } + fn display_name(&self) -> &str { + "Warm" + } + fn supports_steering(&self) -> bool { + true + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::TurnBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + controls: RunControls, + ) -> Result>, HarnessError> { + self.0 + .lock() + .unwrap() + .push((request.prompt.clone(), request.resume.clone(), false)); + let turn = |text: &str| { + vec![ + Ok(AgentEvent::TextDelta { text: text.into() }), + Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some("warm-session".into()), + }), + ] + }; + let mut first = vec![Ok(AgentEvent::SessionStarted { + harness: HarnessId::Mock, + model: "mock-1".into(), + tools: vec![], + cwd: request.cwd.clone(), + session_id: "warm-session".into(), + assistant_message_id: uuid::Uuid::new_v4().to_string(), + })]; + first.extend(turn("first answer")); + let seen = self.0.clone(); + let later = futures::stream::unfold(controls.steering, move |mut steering| { + let seen = seen.clone(); + async move { + let message = steering.recv().await?; + seen.lock().unwrap().push((message.prompt, None, true)); + let mut events = vec![Ok(AgentEvent::Steered { + assistant_message_id: None, + next_assistant_message_id: Some(uuid::Uuid::new_v4().to_string()), + })]; + events.extend(turn("later answer")); + Some((futures::stream::iter(events), steering)) + } + }) + .flatten(); + Ok(futures::stream::iter(first).chain(later).boxed()) + } +} + +/// A fork whose first turn is a native command, continued in the SAME live +/// provider runtime: the copied history must ride the next ordinary send +/// (warm dispatch or steer), exactly once, and survive a cold resume. +#[tokio::test] +async fn warm_side_chat_sends_owed_fork_history_once() { + let dir = tempfile::tempdir().unwrap(); + let seen = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Warm(seen.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + let source = core.doc_host.open("main").unwrap(); + for (id, role, text) in [ + ("u1", MessageRole::User, "Remember PINEAPPLE"), + ("a1", MessageRole::Assistant, "I remember PINEAPPLE"), + ] { + source + .doc() + .push_message(&message(id, role, text, MessageStatus::Complete)) + .unwrap(); + } + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "fork", "sourceChatId": "main" }), + ) + .await + .unwrap(); + let request = |prompt: &str| RunRequest { + mcp: None, + prompt: prompt.into(), + harness: Some(HarnessId::Mock), + model: None, + reasoning: None, + model_options: Default::default(), + cwd: "/tmp".into(), + sandbox: SandboxLevel::WorkspaceWrite, + auto_approve: true, + resume: None, + provider_fork: None, + resume_fallback_history: None, + attachments: vec![], + worktree: None, + }; + let settle = |count: usize| { + let seen = seen.clone(); + let core = &core; + async move { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while seen.lock().unwrap().len() < count + || core + .sessions + .session_status("fork") + .is_some_and(|s| s.status != roboco_proto::SessionStatus::Idle) + { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + seen.lock().unwrap()[count - 1].0.clone() + } + }; + let dispatch = |prompt: &'static str, id: &'static str| { + core.sessions + .dispatch("fork", HarnessId::Mock, request(prompt), Some(id.into()), None, None) + }; + dispatch("/review", "f1").await.unwrap(); + assert_eq!(settle(1).await, "/review"); + let doc = core.doc_host.open("fork").unwrap(); + assert_eq!( + doc.doc().fork_history_session(), + None, + "nothing delivered yet" + ); + // Warm dispatch into the live runtime: the owed history goes out. + dispatch("What should you remember?", "f2").await.unwrap(); + let warm = settle(2).await; + assert!(warm.contains("PINEAPPLE"), "{warm}"); + assert!(warm.ends_with("What should you remember?"), "{warm}"); + assert!(seen.lock().unwrap()[1].2, "delivered to the live runtime"); + assert_eq!( + doc.doc().fork_history_session().as_deref(), + Some("warm-session") + ); + // Delivered: an explicit steer and later sends go out bare. + core.sessions + .steer("fork", "And now?", Some("f3".into()), None) + .await + .unwrap(); + assert_eq!(settle(3).await, "And now?"); + assert!(seen.lock().unwrap()[2].2); + // A cold resume of the same provider session owes nothing either. + core.sessions.interrupt("fork").await.unwrap(); + dispatch("Again", "f4").await.unwrap(); + assert_eq!(settle(4).await, "Again"); + assert!(!seen.lock().unwrap()[3].2, "a new runtime"); + assert_eq!( + seen.lock().unwrap()[3].1.as_deref(), + Some("warm-session"), + "resumed the provider session" + ); + core.shutdown().await; +} + +/// A runtime that answers its first turn, then exits once released without +/// reading its mailbox: sends routed into it are orphaned and re-dispatched. +struct Dropping { + runs: Arc)>>>, + release: Arc, +} +#[async_trait] +impl Harness for Dropping { + fn id(&self) -> HarnessId { + HarnessId::Mock + } + fn display_name(&self) -> &str { + "Dropping" + } + fn supports_steering(&self) -> bool { + true + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::TurnBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + controls: RunControls, + ) -> Result>, HarnessError> { + let first = { + let mut runs = self.runs.lock().unwrap(); + runs.push((request.prompt.clone(), request.resume.clone())); + runs.len() == 1 + }; + let events = vec![ + Ok(AgentEvent::SessionStarted { + harness: HarnessId::Mock, + model: "mock-1".into(), + tools: vec![], + cwd: request.cwd.clone(), + session_id: "drop-session".into(), + assistant_message_id: uuid::Uuid::new_v4().to_string(), + }), + Ok(AgentEvent::TextDelta { + text: "answer".into(), + }), + Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some("drop-session".into()), + }), + ]; + let release = self.release.clone(); + // The first runtime lingers (mailbox unread) until released, then + // ends; later ones end after their turn. + let tail = futures::stream::once(async move { + let _mailbox = controls.steering; + if first { + release.notified().await; + } + }) .filter_map(|_| async { None }); Ok(futures::stream::iter(events).chain(tail).boxed()) } } -/// A steer carrying the owed fork history counts as delivered only once the -/// runtime consumes it: orphaned in a dying runtime, its re-dispatch into -/// the same provider session still carries the history. +/// A steer carrying the owed fork history counts as delivered only once the +/// runtime consumes it: orphaned in a dying runtime, its re-dispatch into +/// the same provider session still carries the history. +#[tokio::test] +async fn orphaned_history_steer_still_owes_the_history() { + let dir = tempfile::tempdir().unwrap(); + let runs = Arc::new(Mutex::new(Vec::new())); + let release = Arc::new(tokio::sync::Notify::new()); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Dropping { + runs: runs.clone(), + release: release.clone(), + })); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + let source = core.doc_host.open("main").unwrap(); + for (id, role, text) in [ + ("u1", MessageRole::User, "Remember PINEAPPLE"), + ("a1", MessageRole::Assistant, "I remember PINEAPPLE"), + ] { + source + .doc() + .push_message(&message(id, role, text, MessageStatus::Complete)) + .unwrap(); + } + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "fork", "sourceChatId": "main" }), + ) + .await + .unwrap(); + let request = |prompt: &str| RunRequest { + mcp: None, + prompt: prompt.into(), + harness: Some(HarnessId::Mock), + model: None, + reasoning: None, + model_options: Default::default(), + cwd: "/tmp".into(), + sandbox: SandboxLevel::WorkspaceWrite, + auto_approve: true, + resume: None, + provider_fork: None, + resume_fallback_history: None, + attachments: vec![], + worktree: None, + }; + core.sessions + .dispatch( + "fork", + HarnessId::Mock, + request("/review"), + Some("f1".into()), + None, + None) + .await + .unwrap(); + let idle = || { + core.sessions + .session_status("fork") + .is_some_and(|s| s.status == roboco_proto::SessionStatus::Idle) + }; + for _ in 0..500 { + if idle() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + assert!(idle(), "the /review turn ends; its runtime lingers"); + // Routed into the live runtime's mailbox, never read. + core.sessions + .dispatch( + "fork", + HarnessId::Mock, + request("What should you remember?"), + Some("f2".into()), + None, + None) + .await + .unwrap(); + let doc = core.doc_host.open("fork").unwrap(); + assert_eq!(doc.doc().fork_history_session(), None, "not consumed yet"); + assert_eq!(runs.lock().unwrap().len(), 1); + release.notify_one(); + for _ in 0..500 { + if runs.lock().unwrap().len() == 2 && idle() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + let (prompt, resume) = runs.lock().unwrap()[1].clone(); + assert_eq!( + resume.as_deref(), + Some("drop-session"), + "same provider session" + ); + assert!(prompt.contains("PINEAPPLE"), "{prompt}"); + assert!(prompt.ends_with("What should you remember?"), "{prompt}"); + assert_eq!( + doc.doc().fork_history_session().as_deref(), + Some("drop-session") + ); + core.shutdown().await; +} + +// ── budgeted fork transcript (t3-orchestration-v2 ticket 01) ───────────────── + +/// The count inside the coverage header's "N messages were omitted". +fn omitted_count(prompt: &str) -> usize { + let at = prompt + .find(" messages were omitted") + .expect("coverage header"); + let start = prompt[..at] + .char_indices() + .rev() + .find(|(_, c)| !c.is_ascii_digit()) + .map(|(i, _)| i + 1) + .unwrap_or(0); + prompt[start..at].parse().expect("omitted count") +} + +/// A fork of a long chat no longer pastes the whole transcript: what goes +/// in front of the new message is a 16k-byte selection that keeps the first +/// and latest user messages and the latest assistant message, fills with +/// the newest remainder, and carries a coverage header with the omitted +/// count and the `read_chat` pointer to the source chat. The new user +/// message rides whole at the end. +#[tokio::test] +async fn a_long_fork_sends_budgeted_transcript_with_a_coverage_header() { + let dir = tempfile::tempdir().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Capture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + let source = core.doc_host.open("main").unwrap(); + for (id, role, text) in [ + ( + "u0", + MessageRole::User, + "FIRSTASK: build the budgeted fork transcript", + ), + ("a0", MessageRole::Assistant, "FIRSTANS: on it"), + ] { + source + .doc() + .push_message(&message(id, role, text, MessageStatus::Complete)) + .unwrap(); + } + for i in 1..=30 { + source + .doc() + .push_message(&message( + &format!("u{i}"), + MessageRole::User, + &format!("FILLER-{i}: {}", "x".repeat(900)), + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&message( + &format!("a{i}"), + MessageRole::Assistant, + &format!("FILLERA-{i}: {}", "y".repeat(500)), + MessageStatus::Complete, + )) + .unwrap(); + } + for (id, role, text) in [ + ("u99", MessageRole::User, "LASTASK: ship it"), + ("a99", MessageRole::Assistant, "LASTANS: done"), + ] { + source + .doc() + .push_message(&message(id, role, text, MessageStatus::Complete)) + .unwrap(); + } + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "fork", "sourceChatId": "main" }), + ) + .await + .unwrap(); + let new_message = "What did I ask you to do first?"; + let request = side_turn(&core, &requests, "fork", new_message, "f1").await; + let prompt = request.prompt; + // The new user message is never shortened and always last. + assert!(prompt.ends_with(new_message), "{prompt}"); + // The priorities survive every budget. + for marker in ["FIRSTASK", "LASTASK", "LASTANS"] { + assert!(prompt.contains(marker), "missing {marker}: {prompt}"); + } + // Recent context is preferred over the middle of the transcript. + assert!(prompt.contains("FILLER-30:"), "{prompt}"); + assert!(!prompt.contains("FILLER-2:"), "{prompt}"); + // The coverage header: omitted count, read_chat, the source chat id. + assert!(omitted_count(&prompt) >= 1); + assert!(prompt.contains("read_chat"), "{prompt}"); + assert!(prompt.contains("chat id \"main\""), "{prompt}"); + // Everything in front of the new message fits the budget. + let delivered = &prompt[..prompt.len() - new_message.len()]; + assert!( + delivered.len() <= TRANSCRIPT_BUDGET_BYTES, + "delivered {} bytes", + delivered.len() + ); + core.shutdown().await; +} + +/// A fork of a chat whose whole transcript fits the budget sends it complete +/// — every message verbatim, no omission note, no truncation. +#[tokio::test] +async fn a_short_fork_sends_its_whole_transcript_without_an_omission_note() { + let dir = tempfile::tempdir().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Capture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + let source = core.doc_host.open("main").unwrap(); + for (id, role, text) in [ + ("u1", MessageRole::User, "Remember PINEAPPLE"), + ("a1", MessageRole::Assistant, "I remember PINEAPPLE"), + ("u2", MessageRole::User, "And the code?"), + ("a2", MessageRole::Assistant, "Checked in"), + ] { + source + .doc() + .push_message(&message(id, role, text, MessageStatus::Complete)) + .unwrap(); + } + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "fork", "sourceChatId": "main" }), + ) + .await + .unwrap(); + let new_message = "What did I ask you to remember?"; + let request = side_turn(&core, &requests, "fork", new_message, "f1").await; + let prompt = request.prompt; + assert!(prompt.ends_with(new_message), "{prompt}"); + assert!(prompt.contains(""), "{prompt}"); + // Complete means complete. + for text in [ + "Remember PINEAPPLE", + "I remember PINEAPPLE", + "And the code?", + "Checked in", + ] { + assert!(prompt.contains(text), "missing {text}: {prompt}"); + } + assert!(!prompt.contains("omitted"), "{prompt}"); + assert!(!prompt.contains("read_chat"), "{prompt}"); + assert!(!prompt.contains("…[truncated]"), "{prompt}"); + core.shutdown().await; +} + +/// A single message larger than the whole budget is truncated — present, +/// marked, bounded — while the rest of the transcript and the new user +/// message ride along whole. #[tokio::test] -async fn orphaned_history_steer_still_owes_the_history() { +async fn an_oversized_fork_message_is_truncated_not_dropped() { let dir = tempfile::tempdir().unwrap(); - let runs = Arc::new(Mutex::new(Vec::new())); - let release = Arc::new(tokio::sync::Notify::new()); + let requests = Arc::new(Mutex::new(Vec::new())); let registry = HarnessRegistry::new(); - registry.register(Arc::new(Dropping { - runs: runs.clone(), - release: release.clone(), + registry.register(Arc::new(Capture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + let source = core.doc_host.open("main").unwrap(); + let big = format!("BIGMARK {}", "z".repeat(30_000)); + for (id, role, text) in [ + ("u0", MessageRole::User, "FIRST: the original request"), + ("a0", MessageRole::Assistant, "acknowledged"), + ("u1", MessageRole::User, big.as_str()), + ("a1", MessageRole::Assistant, "done"), + ] { + source + .doc() + .push_message(&message(id, role, text, MessageStatus::Complete)) + .unwrap(); + } + let client = roboco_rpc::memory_client(core.rpc_service()); + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ "chatId": "fork", "sourceChatId": "main" }), + ) + .await + .unwrap(); + let new_message = "Summarize what I asked for."; + let request = side_turn(&core, &requests, "fork", new_message, "f1").await; + let prompt = request.prompt; + // The oversized message is present but cut, visibly. + assert!(prompt.contains("BIGMARK"), "{prompt}"); + assert!(prompt.contains("…[truncated]"), "{prompt}"); + assert!(prompt.contains(&"z".repeat(4000)), "{prompt}"); + assert!(!prompt.contains(&"z".repeat(5000)), "{prompt}"); + // The other messages still ride along. + assert!(prompt.contains("FIRST: the original request"), "{prompt}"); + assert!(prompt.contains("done"), "{prompt}"); + // Nothing was omitted, but the truncation still points at read_chat. + assert!(!prompt.contains("messages were omitted"), "{prompt}"); + assert!(prompt.contains("read_chat"), "{prompt}"); + assert!(prompt.contains("chat id \"main\""), "{prompt}"); + // Bounded, and the new message whole at the end. + let delivered = &prompt[..prompt.len() - new_message.len()]; + assert!( + delivered.len() <= TRANSCRIPT_BUDGET_BYTES, + "delivered {} bytes", + delivered.len() + ); + assert!(prompt.ends_with(new_message), "{prompt}"); + core.shutdown().await; +} + + +// ── Fork into another harness and model (t3-orchestration-v2 ticket 22) ── + +/// A capture harness pinned to a chosen harness id and provider session: +/// the fork's target harness in ticket 22's tests. Like [`Capture`] it never +/// supports title runs, so the auto-titler's throwaway attempts never touch +/// the recorded requests. +struct CaptureOn { + id: HarnessId, + session: &'static str, + requests: Arc>>, +} + +#[async_trait] +impl Harness for CaptureOn { + fn id(&self) -> HarnessId { + self.id + } + fn display_name(&self) -> &str { + "CaptureOn" + } + fn supports_steering(&self) -> bool { + false + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::TurnBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + _: RunControls, + ) -> Result>, HarnessError> { + self.requests.lock().unwrap().push(request); + let session = self.session; + Ok(futures::stream::iter(vec![ + Ok(AgentEvent::TextDelta { + text: "Side answer".into(), + }), + Ok(AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some(session.into()), + }), + ]) + .boxed()) + } +} + +/// A harness whose turn hits a plan usage limit (ticket 05's structured +/// error kind): it streams a little, reports the limit, and ends errored — +/// the state a Limited chat sits in until ticket 06 gives it its own +/// status. The boundary ref rides the entry like any turn. +struct LimitHarness { + requests: Arc>>, +} + +#[async_trait] +impl Harness for LimitHarness { + fn id(&self) -> HarnessId { + HarnessId::Mock + } + fn display_name(&self) -> &str { + "Limit" + } + fn supports_steering(&self) -> bool { + false + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::TurnBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + _: RunControls, + ) -> Result>, HarnessError> { + self.requests.lock().unwrap().push(request); + Ok(futures::stream::iter(vec![ + Ok(AgentEvent::SessionStarted { + harness: HarnessId::Mock, + model: "mock-1".into(), + tools: vec![], + cwd: "/tmp".into(), + session_id: "parent-provider-session".into(), + assistant_message_id: uuid::Uuid::new_v4().to_string(), + }), + Ok(AgentEvent::TurnRef { + reference: roboco_proto::ProviderTurnRef { + harness: HarnessId::Mock, + session_id: "parent-provider-session".into(), + turn_id: "turn-9".into(), + }, + }), + Ok(AgentEvent::TextDelta { + text: "half an answer".into(), + }), + Ok(AgentEvent::Error { + message: "Claude 5-hour limit reached — the turn was blocked.".into(), + kind: Some(roboco_proto::AgentErrorKind::UsageLimit { + window: "5-hour".into(), + resets_at: Some(4102444800000), + }), + }), + Ok(AgentEvent::Done { + status: DoneStatus::Errored, + result: None, + error: Some("Claude 5-hour limit reached — the turn was blocked.".into()), + session_id: Some("parent-provider-session".into()), + }), + ]) + .boxed()) + } +} + +/// Queue a message and let the doc host deliver it — the path a composer +/// send takes, so the run's harness, model, reasoning and options come from +/// the chat ROW (the fork's recorded config), not a hand-built request. +/// Waits for the turn to settle and returns the delivered request. +async fn queued_side_turn( + core: &EngineCore, + requests: &Arc>>, + chat: &str, + prompt: &str, +) -> RunRequest { + let before = requests.lock().unwrap().len(); + core.doc_host.queue_message(chat, prompt, Vec::new()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let delivered = requests.lock().unwrap().len() > before; + let settled = core + .sessions + .session_status(chat) + .is_none_or(|status| status.status != roboco_proto::SessionStatus::Working); + if delivered && settled { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + requests.lock().unwrap()[before].clone() +} + +/// A fork with a harness and model override (ticket 22, spec story 61) runs +/// on that harness and model: the copy's row carries them, and its first +/// send goes to the TARGET harness with the budgeted history in front — +/// never a native fork of the source's provider session, and never a resume +/// of it. The turn after that resumes the target harness's own session with +/// nothing pasted again. +#[tokio::test] +async fn a_fork_with_a_harness_and_model_override_runs_on_that_harness() { + let dir = tempfile::tempdir().unwrap(); + let mock_requests = Arc::new(Mutex::new(Vec::new())); + let codex_requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Capture(mock_requests.clone()))); + registry.register(Arc::new(CaptureOn { + id: HarnessId::Codex, + session: "codex-provider-session", + requests: codex_requests.clone(), })); let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); core.workspace @@ -796,90 +2690,482 @@ async fn orphaned_history_steer_still_owes_the_history() { Some("/tmp".into()), ) .unwrap(); + // The source runs on Mock with a pinned model, reasoning level and model + // option of its own — a cross-harness fork must inherit none of them + // (they are ids in the old harness's vocabulary), while the sandbox is + // harness-agnostic and rides along. + core.workspace + .set_chat_config( + "main", + &roboco_proto::ChatConfig { + harness: HarnessId::Mock, + model: Some("mock-9".into()), + reasoning: Some(ReasoningLevel::Max), + model_options: serde_json::Map::from_iter([( + "contextWindow".to_string(), + serde_json::json!("1m"), + )]), + sandbox: SandboxLevel::ReadOnly, + }, + ) + .unwrap(); + core.workspace + .set_chat_harness_session("main", "parent-provider-session", "/tmp"); + let source = core.doc_host.open("main").unwrap(); + source + .doc() + .push_message(&message( + "u1", + MessageRole::User, + "Remember PINEAPPLE", + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&turn_entry( + "a1", + "I remember PINEAPPLE", + MessageStatus::Complete, + "turn-1", + )) + .unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + let fork = client + .call_as::( + methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": "side", + "sourceChatId": "main", + "harness": "codex", + "model": "gpt-6-codex", + }), + ) + .await + .unwrap(); + let config = fork.config.expect("the override writes the run config"); + assert_eq!(config.harness, HarnessId::Codex); + assert_eq!(config.model.as_deref(), Some("gpt-6-codex")); + assert_eq!(config.reasoning, None, "the old reasoning never crosses"); + assert!( + config.model_options.is_empty(), + "the old model options never cross" + ); + assert_eq!(config.sandbox, SandboxLevel::ReadOnly); + assert_eq!( + core.workspace.chat("side").unwrap().unwrap().config, + Some(config), + "the imported row carries the override" + ); + + // The first send — through the queue, the way a composer send goes, so + // the row's config (not a hand-built request) routes the harness. + let request = queued_side_turn(&core, &codex_requests, "side", "What did I ask?").await; + assert_eq!( + request.provider_fork, None, + "a cross-harness fork never forks the source provider session" + ); + assert_eq!(request.resume, None, "the target harness starts fresh"); + assert_eq!(request.model.as_deref(), Some("gpt-6-codex")); + assert_eq!(request.reasoning, None); + assert!( + request.model_options.is_empty(), + "the old model options never ride the run" + ); + assert!(request.prompt.contains("PINEAPPLE"), "{}", request.prompt); + assert!( + request.prompt.ends_with("What did I ask?"), + "{}", + request.prompt + ); + assert!( + mock_requests.lock().unwrap().is_empty(), + "the fork's sends never touch the source's harness" + ); + + // The next turn resumes the TARGET harness's own session, and the + // budgeted history is never delivered twice. + let request = queued_side_turn(&core, &codex_requests, "side", "And now?").await; + assert_eq!(request.resume.as_deref(), Some("codex-provider-session")); + assert_eq!(request.prompt, "And now?", "{}", request.prompt); + core.shutdown().await; +} + +/// A fork whose override names the SAME harness — or overrides only the +/// model — keeps the native fork behaviour (ticket 22): the boundary +/// reply's provider turn ref drives the first run, nothing is pasted, and +/// the model override rides the row's config out with the run. +#[tokio::test] +async fn a_same_harness_fork_keeps_the_native_fork_behaviour() { + let dir = tempfile::tempdir().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Capture(requests.clone()))); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + core.workspace + .set_chat_harness_session("main", "parent-provider-session", "/tmp"); + core.workspace + .set_chat_config( + "main", + &roboco_proto::ChatConfig { + harness: HarnessId::Mock, + model: Some("mock-9".into()), + reasoning: Some(ReasoningLevel::Medium), + model_options: Default::default(), + sandbox: SandboxLevel::WorkspaceWrite, + }, + ) + .unwrap(); let source = core.doc_host.open("main").unwrap(); - for (id, role, text) in [ - ("u1", MessageRole::User, "Remember PINEAPPLE"), - ("a1", MessageRole::Assistant, "I remember PINEAPPLE"), - ] { - source - .doc() - .push_message(&message(id, role, text, MessageStatus::Complete)) - .unwrap(); - } + source + .doc() + .push_message(&message( + "u1", + MessageRole::User, + "Remember PINEAPPLE", + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&turn_entry( + "a1", + "I remember PINEAPPLE", + MessageStatus::Complete, + "turn-1", + )) + .unwrap(); let client = roboco_rpc::memory_client(core.rpc_service()); + + // An explicit override naming the source's own harness. client .call( methods::FORK_SIDE_CHAT, - serde_json::json!({ "chatId": "fork", "sourceChatId": "main" }), + serde_json::json!({ + "chatId": "side", + "sourceChatId": "main", + "harness": "mock", + "model": "mock-2", + }), ) .await .unwrap(); - let request = |prompt: &str| RunRequest { - mcp: None, - prompt: prompt.into(), - harness: Some(HarnessId::Mock), - model: None, - reasoning: None, - model_options: Default::default(), - cwd: "/tmp".into(), - sandbox: SandboxLevel::WorkspaceWrite, - auto_approve: true, - resume: None, - attachments: vec![], - worktree: None, - }; - core.sessions - .dispatch( - "fork", - HarnessId::Mock, - request("/review"), - Some("f1".into()), + let config = core.workspace.chat("side").unwrap().unwrap().config.unwrap(); + assert_eq!(config.harness, HarnessId::Mock); + assert_eq!(config.model.as_deref(), Some("mock-2")); + // A same-harness pick inherits the rest of the source's config: only + // the model was overridden. + assert_eq!(config.reasoning, Some(ReasoningLevel::Medium)); + let request = queued_side_turn(&core, &requests, "side", "What did I ask?").await; + let fork = request + .provider_fork + .expect("a same-harness fork keeps the native fork"); + assert_eq!(fork.session_id, "parent-provider-session"); + assert_eq!(fork.last_turn, "turn-1"); + assert_eq!(request.resume, None); + assert_eq!(request.prompt, "What did I ask?", "{}", request.prompt); + assert_eq!(request.model.as_deref(), Some("mock-2")); + + // A model-only override, no harness named: native fork again. + client + .call( + methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": "side-model-only", + "sourceChatId": "main", + "model": "mock-3", + }), ) .await .unwrap(); - let idle = || { - core.sessions - .session_status("fork") - .is_some_and(|s| s.status == roboco_proto::SessionStatus::Idle) - }; - for _ in 0..500 { - if idle() { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(5)).await; - } - assert!(idle(), "the /review turn ends; its runtime lingers"); - // Routed into the live runtime's mailbox, never read. + let config = core + .workspace + .chat("side-model-only") + .unwrap() + .unwrap() + .config + .unwrap(); + assert_eq!(config.harness, HarnessId::Mock); + assert_eq!(config.model.as_deref(), Some("mock-3")); + let request = queued_side_turn(&core, &requests, "side-model-only", "And now?").await; + let fork = request + .provider_fork + .expect("a model-only override never disables the native fork"); + assert_eq!(fork.last_turn, "turn-1"); + assert_eq!(request.prompt, "And now?", "{}", request.prompt); + core.shutdown().await; +} + +/// A source chat whose last turn hit a usage limit can still be forked into +/// another harness (spec story 64): the errored reply is the boundary, the +/// copy exists, and its first send carries the budgeted history on the new +/// harness — no native fork, even though the boundary reply carries the +/// provider turn ref a same-harness fork would have used. +#[tokio::test] +async fn a_limited_source_chat_can_be_forked_into_another_harness() { + let dir = tempfile::tempdir().unwrap(); + let limited_requests = Arc::new(Mutex::new(Vec::new())); + let codex_requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(LimitHarness { + requests: limited_requests.clone(), + })); + registry.register(Arc::new(CaptureOn { + id: HarnessId::Codex, + session: "codex-provider-session", + requests: codex_requests.clone(), + })); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + core.workspace + .set_chat_harness_session("main", "parent-provider-session", "/tmp"); + let source = core.doc_host.open("main").unwrap(); + source + .doc() + .push_message(&message( + "u1", + MessageRole::User, + "Remember PINEAPPLE", + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&turn_entry( + "a1", + "I remember PINEAPPLE", + MessageStatus::Complete, + "turn-1", + )) + .unwrap(); + // The next turn hits the plan limit for real: an Error event with the + // structured usage-limit kind, an errored Done. The chat sits errored — + // the state a Limited chat occupies before ticket 06's own status. core.sessions .dispatch( - "fork", + "main", HarnessId::Mock, - request("What should you remember?"), - Some("f2".into()), - ) + RunRequest { + mcp: None, + prompt: "keep going".into(), + harness: Some(HarnessId::Mock), + model: None, + reasoning: None, + model_options: Default::default(), + cwd: "/tmp".into(), + sandbox: SandboxLevel::WorkspaceWrite, + auto_approve: true, + resume: Some("parent-provider-session".into()), + provider_fork: None, + resume_fallback_history: None, + attachments: vec![], + worktree: None, + }, + Some("u2".into()), + None, + None) .await .unwrap(); - let doc = core.doc_host.open("fork").unwrap(); - assert_eq!(doc.doc().fork_history_session(), None, "not consumed yet"); - assert_eq!(runs.lock().unwrap().len(), 1); - release.notify_one(); - for _ in 0..500 { - if runs.lock().unwrap().len() == 2 && idle() { - break; + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while core + .sessions + .session_status("main") + .is_none_or(|status| status.status != roboco_proto::SessionStatus::Errored) + { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; } - tokio::time::sleep(std::time::Duration::from_millis(5)).await; - } - let (prompt, resume) = runs.lock().unwrap()[1].clone(); + }) + .await + .unwrap(); + let entries = source.doc().read_entries().unwrap(); + let errored = entries + .iter() + .find(|entry| entry.id != "a1" && entry.role == MessageRole::Assistant) + .expect("the limited turn left an errored reply"); + assert!( + errored + .parts + .iter() + .any(|part| matches!(part, MessagePart::Error { .. })), + "the reply carries the limit error: {:?}", + errored.parts + ); assert_eq!( - resume.as_deref(), - Some("drop-session"), - "same provider session" + errored.provider_turn.as_ref().map(|r| r.turn_id.as_str()), + Some("turn-9") ); - assert!(prompt.contains("PINEAPPLE"), "{prompt}"); - assert!(prompt.ends_with("What should you remember?"), "{prompt}"); + + // Fork the limited chat into another harness — the default boundary is + // the latest complete reply, which is the errored one. + let client = roboco_rpc::memory_client(core.rpc_service()); + let fork = client + .call_as::( + methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": "side", + "sourceChatId": "main", + "harness": "codex", + "model": "gpt-6-codex", + }), + ) + .await + .unwrap(); assert_eq!( - doc.doc().fork_history_session().as_deref(), - Some("drop-session") + fork.config.as_ref().map(|config| config.harness), + Some(HarnessId::Codex), + "the fork exists despite the source's errored state" + ); + let copied = core + .doc_host + .open("side") + .unwrap() + .doc() + .read_entries() + .unwrap(); + assert!(copied.iter().any(|entry| entry.id == errored.id)); + + // Its first send: budgeted history on the new harness, no native fork + // even though the boundary reply carries a ref, and no resume. + let request = queued_side_turn(&core, &codex_requests, "side", "What did I ask?").await; + assert_eq!(request.provider_fork, None); + assert_eq!(request.resume, None); + assert!(request.prompt.contains("PINEAPPLE"), "{}", request.prompt); + assert!( + request.prompt.ends_with("What did I ask?"), + "{}", + request.prompt + ); + core.shutdown().await; +} + +/// "Fork from here" INTO another harness (tickets 21+22 combined): the +/// boundary cuts the copy, and the cross-harness override rides on top — +/// no native fork even though the boundary reply carries a provider turn +/// ref, and the budgeted history the first send carries never mentions +/// anything after the boundary. +#[tokio::test] +async fn forking_from_an_earlier_reply_into_another_harness_cuts_and_pastes() { + let dir = tempfile::tempdir().unwrap(); + let mock_requests = Arc::new(Mutex::new(Vec::new())); + let codex_requests = Arc::new(Mutex::new(Vec::new())); + let registry = HarnessRegistry::new(); + registry.register(Arc::new(Capture(mock_requests.clone()))); + registry.register(Arc::new(CaptureOn { + id: HarnessId::Codex, + session: "codex-provider-session", + requests: codex_requests.clone(), + })); + let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock).unwrap(); + core.workspace + .create_chat( + "main", + None, + Some(&core.device_id), + None, + Some("/tmp".into()), + ) + .unwrap(); + core.workspace + .set_chat_harness_session("main", "parent-provider-session", "/tmp"); + let source = core.doc_host.open("main").unwrap(); + source + .doc() + .push_message(&message( + "u1", + MessageRole::User, + "Remember PINEAPPLE", + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&turn_entry( + "a1", + "I remember PINEAPPLE", + MessageStatus::Complete, + "turn-1", + )) + .unwrap(); + source + .doc() + .push_message(&message( + "u2", + MessageRole::User, + "And the code?", + MessageStatus::Complete, + )) + .unwrap(); + source + .doc() + .push_message(&turn_entry( + "a2", + "the code is fine", + MessageStatus::Complete, + "turn-2", + )) + .unwrap(); + let client = roboco_rpc::memory_client(core.rpc_service()); + let fork = client + .call_as::( + methods::FORK_SIDE_CHAT, + serde_json::json!({ + "chatId": "side", + "sourceChatId": "main", + "boundary": "a1", + "harness": "codex", + "model": "gpt-6-codex", + }), + ) + .await + .unwrap(); + assert_eq!( + fork.config.map(|config| (config.harness, config.model)), + Some((HarnessId::Codex, Some("gpt-6-codex".into()))) + ); + let ids = |chat: &str| { + core.doc_host + .open(chat) + .unwrap() + .doc() + .read_entries() + .unwrap() + .into_iter() + .map(|entry| entry.id) + .collect::>() + }; + assert_eq!( + ids("side"), + vec!["u1".to_string(), "a1".to_string(), "fork:side".to_string()], + "the copy stops at the boundary, inclusive" + ); + // The first send: budgeted history cut at the boundary, no native fork, + // no resume — and nothing from after the boundary rides along. + let request = queued_side_turn(&core, &codex_requests, "side", "What did I ask?").await; + assert_eq!(request.provider_fork, None); + assert_eq!(request.resume, None); + assert!(request.prompt.contains("PINEAPPLE"), "{}", request.prompt); + assert!( + !request.prompt.contains("the code is fine"), + "{}", + request.prompt ); + assert!(request.prompt.ends_with("What did I ask?"), "{}", request.prompt); core.shutdown().await; } diff --git a/crates/engine/tests/subagent_idle_reap.rs b/crates/engine/tests/subagent_idle_reap.rs index a08e3efed6..b82f7304f3 100644 --- a/crates/engine/tests/subagent_idle_reap.rs +++ b/crates/engine/tests/subagent_idle_reap.rs @@ -52,6 +52,8 @@ fn run_request(prompt: &str) -> RunRequest { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } @@ -195,7 +197,7 @@ async fn park_after_spawn() -> Parked { let core = EngineCore::assemble(dir.path(), Arc::new(registry), HarnessId::Mock) .expect("engine core assembles"); core.sessions - .dispatch(CHAT, HarnessId::Mock, run_request("fan out"), None) + .dispatch(CHAT, HarnessId::Mock, run_request("fan out"), None, None, None) .await .expect("dispatch"); diff --git a/crates/engine/tests/transcript_resume.rs b/crates/engine/tests/transcript_resume.rs index 51378ee602..8a573de507 100644 --- a/crates/engine/tests/transcript_resume.rs +++ b/crates/engine/tests/transcript_resume.rs @@ -25,6 +25,7 @@ const ENTRY_TEXT: usize = 300; fn assistant_entry(id: &str, text: &str) -> SessionMessageEntry { SessionMessageEntry { + origin_chat_id: None, id: id.into(), role: MessageRole::Assistant, parts: vec![MessagePart::Text { @@ -35,6 +36,7 @@ fn assistant_entry(id: &str, text: &str) -> SessionMessageEntry { device_id: "host".into(), status: None, continuation_of: None, + provider_turn: None, duration_ms: None, } } @@ -132,7 +134,7 @@ async fn transcript_resume_over_the_real_listener() { let handle = core.doc_host.open(CHAT).unwrap(); for i in 0..250 { handle - .write_user_message(&format!("u{i}"), &text(i), i as i64) + .write_user_message(&format!("u{i}"), &text(i), i as i64, None) .unwrap(); handle .doc() @@ -254,7 +256,7 @@ async fn transcript_resume_over_the_real_listener() { // 5. Live updates keep carrying digests, and the client map stays exact. handle - .write_user_message("u-after", &text(999), 2_000) + .write_user_message("u-after", &text(999), 2_000, None) .unwrap(); let live_value = next_value(&mut resumed, "live delta after a write").await; let live = decode(&live_value, "live delta"); @@ -371,7 +373,7 @@ async fn transcript_resume_over_the_real_listener() { let compat_reset_value = next_value(&mut compat_rx, "compat reset").await; assert!(compat_reset_value.get("digests").is_none()); let doc = core.doc_host.open(CHAT).unwrap(); - doc.write_user_message("u-compat", &text(998), 3_000).unwrap(); + doc.write_user_message("u-compat", &text(998), 3_000, None).unwrap(); let compat_delta_value = next_value(&mut compat_rx, "compat live delta").await; assert!( compat_delta_value.get("digests").is_none(), diff --git a/crates/engine/tests/transcript_salvage.rs b/crates/engine/tests/transcript_salvage.rs index a36cb9396f..cfea1d9ff3 100644 --- a/crates/engine/tests/transcript_salvage.rs +++ b/crates/engine/tests/transcript_salvage.rs @@ -25,6 +25,7 @@ fn entry( status: Option, ) -> SessionMessageEntry { SessionMessageEntry { + origin_chat_id: None, id: id.into(), role, parts: vec![MessagePart::Text { @@ -35,6 +36,7 @@ fn entry( device_id: "lost-device".into(), status, continuation_of: None, + provider_turn: None, duration_ms: None, } } diff --git a/crates/engine/tests/turn_quiesce.rs b/crates/engine/tests/turn_quiesce.rs index f523603920..fecc18a28d 100644 --- a/crates/engine/tests/turn_quiesce.rs +++ b/crates/engine/tests/turn_quiesce.rs @@ -61,6 +61,8 @@ fn run_request(prompt: &str) -> RunRequest { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } @@ -259,7 +261,8 @@ async fn parked_self_continuation_folds_and_requiesces() { HarnessId::Mock, run_request("pull the app and benchmark it"), None, - ) + None, + None) .await .expect("dispatch"); @@ -339,7 +342,8 @@ async fn missing_turn_end_settles_instead_of_working_forever() { HarnessId::Mock, run_request("pull the app and benchmark it"), None, - ) + None, + None) .await .expect("dispatch"); @@ -357,7 +361,7 @@ async fn missing_turn_end_settles_instead_of_working_forever() { let outcome = rig .core .sessions - .steer(CHAT, "what about now", None) + .steer(CHAT, "what about now", None, None) .await .expect("steer"); assert!(matches!(outcome, SteerOutcome::Accepted)); @@ -401,7 +405,8 @@ async fn open_tool_call_never_quiesces() { HarnessId::Mock, run_request("run the slow build"), None, - ) + None, + None) .await .expect("dispatch"); @@ -457,7 +462,7 @@ async fn stale_tool_echo_stays_parked() { let rig = assemble("do a thing"); rig.core .sessions - .dispatch(CHAT, HarnessId::Mock, run_request("do a thing"), None) + .dispatch(CHAT, HarnessId::Mock, run_request("do a thing"), None, None, None) .await .expect("dispatch"); diff --git a/crates/engine/tests/unarchive_on_send.rs b/crates/engine/tests/unarchive_on_send.rs index 8bd6b0e79e..007cfcb8e1 100644 --- a/crates/engine/tests/unarchive_on_send.rs +++ b/crates/engine/tests/unarchive_on_send.rs @@ -112,6 +112,8 @@ fn run_payload(message_id: &str) -> SessionCommandPayload { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }, message_id: message_id.into(), } diff --git a/crates/engine/tests/worktree_on_run.rs b/crates/engine/tests/worktree_on_run.rs index 700fbb1ee5..8ca1962f1b 100644 --- a/crates/engine/tests/worktree_on_run.rs +++ b/crates/engine/tests/worktree_on_run.rs @@ -121,6 +121,8 @@ fn run_payload(message_id: &str, repo_path: &str, space_id: Option<&str>) -> Ses auto_approve: true, attachments: Vec::new(), resume: None, + provider_fork: None, + resume_fallback_history: None, worktree: Some(WorktreeSpec { repo_path: repo_path.into(), base: "main".into(), diff --git a/crates/harness/examples/cursor_stability_probe.rs b/crates/harness/examples/cursor_stability_probe.rs index 2ee8a7c4bb..4b8a0dbfb3 100644 --- a/crates/harness/examples/cursor_stability_probe.rs +++ b/crates/harness/examples/cursor_stability_probe.rs @@ -45,6 +45,8 @@ async fn turn( worktree: None, mcp: None, resume: session.clone(), + provider_fork: None, + resume_fallback_history: None, }; let mut stream = harness .run(request, controls) @@ -166,6 +168,8 @@ async fn parked(harness: &CursorHarness, count: usize) { attachments: vec![], worktree: None, mcp: None, + provider_fork: None, + resume_fallback_history: None, resume: None, }; let mut stream = harness.run(request, controls).await.unwrap(); @@ -264,6 +268,8 @@ async fn history(harness: &CursorHarness, count: usize) { attachments: vec![], worktree: None, mcp: None, + provider_fork: None, + resume_fallback_history: None, resume: None, }; let mut stream = harness.run(request, controls).await.unwrap(); @@ -385,6 +391,8 @@ async fn burst(harness: &CursorHarness, count: usize, cancel: bool) { attachments: vec![], worktree: None, mcp: None, + provider_fork: None, + resume_fallback_history: None, resume: seed, }; let mut stream = harness.run(request, controls).await.unwrap(); diff --git a/crates/harness/examples/cursor_steering_probe.rs b/crates/harness/examples/cursor_steering_probe.rs index 12560cbf15..f4a1caa095 100644 --- a/crates/harness/examples/cursor_steering_probe.rs +++ b/crates/harness/examples/cursor_steering_probe.rs @@ -19,6 +19,8 @@ async fn main() -> anyhow::Result<()> { harness: None, model: Some(model), reasoning: None, model_options: Default::default(), cwd: workspace.path().to_string_lossy().into_owned(), sandbox: SandboxLevel::DangerFullAccess, auto_approve: true, + provider_fork: None, + resume_fallback_history: None, attachments: vec![], worktree: None, resume: None, mcp: None, }; let mut stream = CursorHarness::new() @@ -77,7 +79,7 @@ async fn main() -> anyhow::Result<()> { break; } } - AgentEvent::Error { message } => eprintln!("{message}"), + AgentEvent::Error { message, .. } => eprintln!("{message}"), _ => {} } } diff --git a/crates/harness/examples/devin_models_probe.rs b/crates/harness/examples/devin_models_probe.rs index 9fce6133fa..fc72b14697 100644 --- a/crates/harness/examples/devin_models_probe.rs +++ b/crates/harness/examples/devin_models_probe.rs @@ -50,6 +50,8 @@ async fn main() -> anyhow::Result<()> { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }; println!("Running real Devin ACP with {model}"); let mut stream = harness.run(request, controls).await?; diff --git a/crates/harness/examples/grok_subagent_probe.rs b/crates/harness/examples/grok_subagent_probe.rs index e6437be2ac..f36f74377f 100644 --- a/crates/harness/examples/grok_subagent_probe.rs +++ b/crates/harness/examples/grok_subagent_probe.rs @@ -51,6 +51,8 @@ async fn main() { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }; let mut stream = AcpHarness::grok() .run(request, controls) diff --git a/crates/harness/examples/opencode_subagent_probe.rs b/crates/harness/examples/opencode_subagent_probe.rs index 28355f0a7d..70d8c90252 100644 --- a/crates/harness/examples/opencode_subagent_probe.rs +++ b/crates/harness/examples/opencode_subagent_probe.rs @@ -61,6 +61,8 @@ async fn main() { auto_approve: true, attachments: Vec::new(), resume: None, + provider_fork: None, + resume_fallback_history: None, worktree: None, mcp: None, }; diff --git a/crates/harness/examples/opencode_turn_probe.rs b/crates/harness/examples/opencode_turn_probe.rs index 7c2f2fa7b9..93b3f699cc 100644 --- a/crates/harness/examples/opencode_turn_probe.rs +++ b/crates/harness/examples/opencode_turn_probe.rs @@ -33,6 +33,8 @@ async fn main() { auto_approve: true, attachments: Vec::new(), resume: None, + provider_fork: None, + resume_fallback_history: None, worktree: None, mcp: None, }; @@ -87,7 +89,7 @@ async fn main() { }) => { eprintln!("USAGE {input_tokens}/{output_tokens}"); } - Ok(AgentEvent::Error { message }) => eprintln!("CHIP {message}"), + Ok(AgentEvent::Error { message, .. }) => eprintln!("CHIP {message}"), Ok(other) => eprintln!("EV {other:?}"), Err(e) => eprintln!("ERR {e}"), } diff --git a/crates/harness/src/acp/mod.rs b/crates/harness/src/acp/mod.rs index b78263488a..79e20be495 100644 --- a/crates/harness/src/acp/mod.rs +++ b/crates/harness/src/acp/mod.rs @@ -3676,6 +3676,7 @@ async fn run_session(session: Session) { &event_tx, AgentEvent::Error { message: format!("{agent_name} could not restore session {resume}; starting a new session without the previous context: {e}"), + kind: None, }, ) .await; @@ -4639,7 +4640,7 @@ async fn run_session(session: Session) { assistant_message_id: Some(prev), next_assistant_message_id: Some(next), }).await - || !send(&event_tx, AgentEvent::Error { message: message.clone() }).await + || !send(&event_tx, AgentEvent::Error { message: message.clone(), kind: None }).await || !send(&event_tx, AgentEvent::Done { status: DoneStatus::Errored, result: None, @@ -4807,6 +4808,7 @@ async fn run_session(session: Session) { prompt_stall.map(|d| d.as_secs()).unwrap_or(0), stall_hint, ), + kind: None, }, ) .await; diff --git a/crates/harness/src/claude/mod.rs b/crates/harness/src/claude/mod.rs index f260f9d13f..c4e9a85a0e 100644 --- a/crates/harness/src/claude/mod.rs +++ b/crates/harness/src/claude/mod.rs @@ -42,6 +42,18 @@ //! either way. `background_tasks_changed` reports the live set to the host. //! - Interrupt: cancelling [`RunControls::interrupt`] tears the runtime down — //! the interrupt control request, then SIGTERM and SIGKILL. +//! - TURN REFS (t3-orchestration-v2 ticket 18): every `result` closes its +//! turn with a provider turn reference — the session id plus the uuid of +//! the last top-level (untagged) assistant/user frame before it — so a +//! fork can continue the provider session at exactly that turn. +//! - NATIVE FORK (same ticket): a fork's first run resumes the SOURCE +//! session with `--fork-session` (plus the hidden +//! `--resume-session-at ` when the boundary is an earlier turn, +//! inclusive), giving the fork a NEW session id that carries the copied +//! history — nothing is pasted. A CLI that rejects the flags (it exits +//! before ever speaking), or a source session the CLI's store no longer +//! has, falls back to a fresh session with the budgeted transcript: a +//! fork never fails the send. pub mod catalog; mod discovery; @@ -56,7 +68,7 @@ use async_trait::async_trait; use futures::StreamExt; use futures::stream::BoxStream; use serde_json::Value; -use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::io::{AsyncBufReadExt, AsyncReadExt as _, AsyncWriteExt, BufReader}; use tokio::sync::mpsc; use roboco_proto::{ @@ -103,6 +115,163 @@ fn mcp_config_arg(mcp: &roboco_proto::McpServer) -> String { .to_string() } +/// The launch probe's reading of the buffered stdout so far (B11/B12): an +/// errored first `result` BEFORE any assistant output refuses the launch +/// — the CLI accepted the flags but then failed the resumed session (a +/// `--resume-session-at` uuid validated late, or a session the account +/// can no longer read). Any assistant output accepts it (the launch is +/// serving the conversation). A clean `result` before any assistant +/// output also accepts: the launch itself is fine, whatever the empty +/// turn means. Only complete lines are judged — the tail of a partial +/// line rides to the session loop and is judged there. +enum LaunchReading { + Undecided, + Accepted, + Refused, +} + +fn launch_reading(head: &[u8]) -> LaunchReading { + for line in std::str::from_utf8(head) + .unwrap_or_default() + .lines() + .filter(|line| !line.trim().is_empty()) + { + let Ok(value) = serde_json::from_str::(line) else { + continue; + }; + match value.get("type").and_then(serde_json::Value::as_str) { + Some("assistant") => return LaunchReading::Accepted, + Some("result") => { + let errored = value + .get("is_error") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + || value.get("subtype").and_then(serde_json::Value::as_str) + == Some("error"); + return if errored { + LaunchReading::Refused + } else { + LaunchReading::Accepted + }; + } + _ => {} + } + } + LaunchReading::Undecided +} + +/// Bound on the launch probe (B11/B12): the probe buffers the stream until +/// the launch proves itself, so a wedged CLI must not hold it forever — +/// past the bound the buffered stream is handed over and the session loop +/// deals with whatever follows. Generous: a healthy launch speaks within +/// milliseconds; only Stop (the other escape) is faster. +const LAUNCH_PROBE_BOUND: Duration = Duration::from_secs(60); + +/// Whether the CLI's own session store knows `session_id` — one +/// `/projects/*/{session_id}.jsonl` file (the same walk +/// `Normalizer::for_resume` restores subagent chips from). A resume the +/// CLI would refuse is what triggers the account-failover fallback +/// (ticket 09): the driver starts a fresh session with the engine's +/// budgeted history instead of passing an id `--resume` will reject. +async fn session_known(config_root: &std::path::Path, session_id: &str) -> bool { + if session_id.is_empty() + || !session_id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-') + { + return false; + } + let config_root = config_root.to_owned(); + let session_id = session_id.to_owned(); + tokio::task::spawn_blocking(move || { + let Ok(projects) = std::fs::read_dir(config_root.join("projects")) else { + return false; + }; + projects + .flatten() + .any(|project| project.path().join(format!("{session_id}.jsonl")).exists()) + }) + .await + .unwrap_or(false) +} + +/// The uuid of the LAST top-level message record (`assistant` or `user`, +/// no parent tool-use id, not a sidechain) in the CLI's own session file +/// for `session_id` — how a fork launch tells the session's LATEST turn +/// (`--fork-session` alone continues the whole session) from a boundary at +/// an EARLIER turn (the hidden `--resume-session-at` cuts it there, +/// inclusive). `None` when the file is missing, unreadable, or holds no +/// message records: the boundary is then treated as an earlier turn — the +/// flag combination the CLI may refuse, which is exactly what the fork's +/// budgeted-paste fallback covers. +async fn session_last_message_uuid( + config_root: &std::path::Path, + session_id: &str, +) -> Option { + use std::io::BufRead as _; + if session_id.is_empty() + || !session_id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-') + { + return None; + } + let config_root = config_root.to_owned(); + let session_id = session_id.to_owned(); + tokio::task::spawn_blocking(move || { + let Ok(projects) = std::fs::read_dir(config_root.join("projects")) else { + return None; + }; + // Session ids are global; the project directory encoding does not + // matter here (the same walk `session_known` and + // `Normalizer::for_resume` use). + let file = projects.flatten().find_map(|project| { + std::fs::File::open(project.path().join(format!("{session_id}.jsonl"))).ok() + })?; + // Long sessions run to hundreds of MB: only lines that can carry a + // uuid are parsed, off the runtime, and only the LAST top-level + // message record is kept. + let mut reader = std::io::BufReader::with_capacity(1 << 16, file); + let mut line = Vec::new(); + let mut last = None; + loop { + line.clear(); + match reader.read_until(b'\n', &mut line) { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + if !line.windows(6).any(|window| window == b"\"uuid\"") { + continue; + } + let Ok(text) = std::str::from_utf8(&line) else { + continue; + }; + // Torn lines and older/newer unknown shapes are harmless. + let Ok(record) = serde_json::from_str::(text) else { + continue; + }; + if !matches!( + record.get("type").and_then(Value::as_str), + Some("assistant") | Some("user") + ) || record.get("isSidechain").and_then(Value::as_bool) == Some(true) + || !record.get("parent_tool_use_id").is_none_or(Value::is_null) + { + continue; + } + if let Some(uuid) = record + .get("uuid") + .and_then(Value::as_str) + .filter(|uuid| !uuid.is_empty()) + { + last = Some(uuid.to_owned()); + } + } + last + }) + .await + .unwrap_or(None) +} + /// The `--model` value: the 1M context window is selected via a model-id /// suffix (`sonnet[1m]`), exactly how the CLI itself does it. fn model_arg(request: &RunRequest) -> Option { @@ -152,6 +321,11 @@ pub struct ClaudeHarness { initialize: discovery::InitializeCache, models_cache: crate::catalog::Catalog, workspace_commands: crate::skills::CommandDiscovery, + /// Test override for the CLI's config root — the directory whose + /// `projects/` holds the session store the fork launch reads (a + /// missing source session and the latest-turn boundary). Unset in + /// production: `CLAUDE_CONFIG_DIR` (or `~/.claude`) applies. + config_root: Option, } impl Default for ClaudeHarness { @@ -163,6 +337,7 @@ impl Default for ClaudeHarness { initialize: discovery::InitializeCache::default(), models_cache: crate::catalog::Catalog::default(), workspace_commands: crate::skills::CommandDiscovery::default(), + config_root: None, } } } @@ -178,6 +353,15 @@ impl ClaudeHarness { self } + /// Point the session store at a fixed config root instead of + /// `CLAUDE_CONFIG_DIR` / `~/.claude` (tests drive the fork launch + /// against a crafted store; the env var is process-global and would + /// race parallel tests). + pub fn with_config_root(mut self, root: impl Into) -> Self { + self.config_root = Some(root.into()); + self + } + /// Tune the interrupt→SIGTERM→SIGKILL escalation timing. pub fn with_graces(mut self, interrupt_grace: Duration, kill_grace: Duration) -> Self { self.interrupt_grace = interrupt_grace; @@ -578,116 +762,296 @@ impl Harness for ClaudeHarness { impl ClaudeHarness { async fn run_with_mode( &self, - request: RunRequest, + mut request: RunRequest, controls: RunControls, title_only: bool, ) -> Result>, HarnessError> { let exe = self.resolve_executable()?; - let mut cmd = self.build_command(&exe, &request); - if title_only { - cmd.args([ - "--system-prompt", - crate::TITLE_INSTRUCTIONS, - "--tools", - "", - "--strict-mcp-config", - "--mcp-config", - "{\"mcpServers\":{}}", - "--setting-sources", - "", - ]); - } else if let Some(mcp) = &request.mcp { - // Roboco's own server rides beside the user's configured servers - // (no `--strict-mcp-config`): the CLI merges an inline JSON - // config with settings-sourced ones. - cmd.args(["--mcp-config", &mcp_config_arg(mcp)]); - } - let normalizer = if let Some(session_id) = &request.resume { - let config = crate::model_context::root( + let config_root = self.config_root.clone().unwrap_or_else(|| { + crate::model_context::root( "CLAUDE_CONFIG_DIR", crate::executable::home_or_current_dir().join(".claude"), + ) + }); + // A resume the CLI will REFUSE — its session store has no such id — + // starts a fresh session instead when the engine attached a + // fallback history (the account-failover continuation, ticket 09): + // the budgeted transcript rides in front of the prompt, so the + // fresh session still knows the conversation. Plain resumes carry + // no fallback and keep today's behavior: pass `--resume` and let + // the CLI report the bad id itself. + if let (Some(resume), Some(history)) = + (request.resume.clone(), request.resume_fallback_history.clone()) + && !session_known(&config_root, &resume).await + { + tracing::debug!( + target: "roboco_harness::claude", + "resume of {resume} has no session file; starting fresh with the budgeted history" ); - Normalizer::for_resume(&config, session_id).await - } else { - Normalizer::new() - }; - let mut child = cmd.spawn().map_err(|e| { - if e.kind() == std::io::ErrorKind::NotFound { - HarnessError::NotInstalled(crate::executable::binary_hint(&exe)) + request.resume = None; + request.prompt = crate::prompt_with_history(&history, &request.prompt); + } + // A native fork (t3-orchestration-v2 ticket 18): the CLI continues + // the SOURCE session as a NEW session id — `--resume + // --fork-session` — so the copied history rides the provider and + // nothing is pasted. A boundary at an earlier turn also passes the + // hidden `--resume-session-at ` (inclusive); the latest turn + // needs no cut, and the unverified flag combination is exactly + // what the fallback below covers. A source session the CLI's store + // no longer has skips the fork outright; a CLI that REJECTS the + // flags — it exits before it ever speaks — retries once as a fresh + // session with the budgeted transcript in front of the prompt. A + // fork never fails the send. + let mut fork_flags: Option> = None; + // Only set when the fork launch is ATTEMPTED — the budgeted + // transcript the retry pastes if the CLI rejects the flags. A + // missing session never attempts; its fallback pastes at once, + // exactly like the refused resume above. + let mut fork_fallback: Option = None; + if let Some(fork) = request.provider_fork.clone() { + if session_known(&config_root, &fork.session_id).await { + // The source session doubles as the launch's resume id, and + // the restored spawn chips (subagent transcript tabs) come + // from its history — the forked copy continues it. + request.resume = Some(fork.session_id.clone()); + let latest = session_last_message_uuid(&config_root, &fork.session_id).await; + let mut flags = vec!["--fork-session".to_string()]; + if latest.as_deref() != Some(fork.last_turn.as_str()) { + flags.push(format!("--resume-session-at={}", fork.last_turn)); + } + fork_flags = Some(flags); + fork_fallback = fork.fallback_history; } else { - HarnessError::Io(e) + tracing::debug!( + target: "roboco_harness::claude", + "fork source {} has no session file; starting fresh with the budgeted history", + fork.session_id + ); + request.provider_fork = None; + if let Some(history) = fork.fallback_history { + request.prompt = crate::prompt_with_history(&history, &request.prompt); + } + } + } + let images = load_image_blocks(&request.attachments).await; + // One launch — or two, when the CLI refuses the fork flags: the + // second drops every session flag and pastes the budgeted + // transcript (the fallback the engine attached for exactly this + // refusal). + for attempt in 0..2 { + if attempt == 1 { + request.resume = None; + request.provider_fork = None; + fork_flags = None; + if let Some(history) = fork_fallback.take() { + request.prompt = crate::prompt_with_history(&history, &request.prompt); + } + tracing::debug!( + target: "roboco_harness::claude", + "the CLI refused the fork launch; starting fresh with the budgeted history" + ); + } + let mut cmd = self.build_command(&exe, &request); + if let Some(flags) = &fork_flags { + cmd.args(flags.clone()); + } + if title_only { + cmd.args([ + "--system-prompt", + crate::TITLE_INSTRUCTIONS, + "--tools", + "", + "--strict-mcp-config", + "--mcp-config", + "{\"mcpServers\":{}}", + "--setting-sources", + "", + ]); + } else if let Some(mcp) = &request.mcp { + // Roboco's own server rides beside the user's configured servers + // (no `--strict-mcp-config`): the CLI merges an inline JSON + // config with settings-sourced ones. + cmd.args(["--mcp-config", &mcp_config_arg(mcp)]); + } + let normalizer = if let Some(session_id) = &request.resume { + Normalizer::for_resume(&config_root, session_id).await + } else { + Normalizer::new() + }; + let mut child = cmd.spawn().map_err(|e| { + if e.kind() == std::io::ErrorKind::NotFound { + HarnessError::NotInstalled(crate::executable::binary_hint(&exe)) + } else { + HarnessError::Io(e) + } + })?; + + let stdin = child + .stdin + .take() + .ok_or_else(|| HarnessError::Protocol("claude child has no stdin".into()))?; + let mut stdout = child + .stdout + .take() + .ok_or_else(|| HarnessError::Protocol("claude child has no stdout".into()))?; + let stderr_tail = crate::StderrTail::default(); + if let Some(stderr) = child.stderr.take() { + let tail = stderr_tail.clone(); + tokio::spawn(async move { + let mut lines = BufReader::new(stderr).lines(); + while let Ok(Some(line)) = lines.next_line().await { + tracing::debug!(target: "roboco_harness::claude", "stderr: {line}"); + tail.push(&line); + } + }); + } + + let (stdin_tx, stdin_rx) = mpsc::unbounded_channel::(); + tokio::spawn(stdin_writer(stdin, stdin_rx)); + if !title_only { + // The opening control request: `perTaskStopAffordance` tells the + // CLI this host stops tasks individually, so an interrupt ends + // the turn and spares background agents. + let _ = stdin_tx.send(StdinMsg::Line(wire::initialize_request_line( + "roboco_initialize", + ))); } - })?; - let stdin = child - .stdin - .take() - .ok_or_else(|| HarnessError::Protocol("claude child has no stdin".into()))?; - let stdout = child - .stdout - .take() - .ok_or_else(|| HarnessError::Protocol("claude child has no stdout".into()))?; - let stderr_tail = crate::StderrTail::default(); - if let Some(stderr) = child.stderr.take() { - let tail = stderr_tail.clone(); - tokio::spawn(async move { - let mut lines = BufReader::new(stderr).lines(); - while let Ok(Some(line)) = lines.next_line().await { - tracing::debug!(target: "roboco_harness::claude", "stderr: {line}"); - tail.push(&line); + // The first prompt as the first stdin user line (streaming-input + // mode) — sent BEFORE the launch probe below: a CLI that lazily + // validates its resume flags fails them only once a prompt + // arrives, so the probe must be reading when that result lands. + // Ultrathink rides every user message — steers included. + // Staged image attachments are inlined as base64 image content blocks + // ahead of the text (verified against the real CLI); their path refs + // also ride the prompt text, so a skipped/unreadable file degrades to + // the old-app behavior (the agent opens the path with its Read tool). + // The line carries a uuid: the CLI then reports its + // `command_lifecycle`, which decides when a `result` is a steer + // boundary instead of the turn's end. A refused launch's retry + // delivers the budgeted transcript here — it is already in front + // of the prompt. + let first_command = uuid::Uuid::new_v4().to_string(); + let first = wire::prompt_line( + &apply_ultrathink(request.reasoning, &request.prompt), + &first_command, + &images, + ); + let _ = stdin_tx.send(StdinMsg::Line(first)); + + // The launch's acceptance probe (ticket 18): the CLI proves the + // launch as it speaks — anything it writes is buffered in `head` + // and handed to the session loop unchanged, so nothing is lost. + // The probe reads until one of: + // - any ASSISTANT output: the launch is good — hand over. + // - an ERRORED first `result` BEFORE any assistant output: the + // CLI accepted the flags but then failed the resume — a bad + // `--resume-session-at` uuid it validates late (B11), or a + // resumed session the account can no longer read (B12: the + // failover continuation's cross-account resume). Same + // refusal as silence: the retry below starts fresh with the + // budgeted transcript, because a fork/resume never fails the + // send. + // - silence: EOF before the CLI ever spoke — the flags were + // rejected up front (the original ticket-18 shape). + // - Stop mid-probe: hand the child to the session loop, which + // sees the token and ends the run interrupted. + // Every launch the probe covers carries a fallback history the + // retry pastes: a fork (the engine's budgeted transcript), or a + // resume with `resume_fallback_history` (the failover + // continuation). Plain launches and titles are handed over + // unread, as before. + let probe_launch = !title_only + && (fork_flags.is_some() + || (request.resume.is_some() && request.resume_fallback_history.is_some())); + let mut head: Vec = Vec::new(); + if probe_launch { + enum Probe { + Accepted, + Silent, + Refused, + Interrupted, } - }); - } + let launch_fallback = fork_fallback + .take() + .or_else(|| request.resume_fallback_history.take()); + let mut probe = [0u8; 1024]; + let bound = + tokio::time::Instant::now() + LAUNCH_PROBE_BOUND; + let outcome = loop { + tokio::select! { + biased; + _ = controls.interrupt.cancelled() => break Probe::Interrupted, + _ = tokio::time::sleep_until(bound) => break Probe::Accepted, + read = stdout.read(&mut probe) => match read { + Ok(0) | Err(_) => { + break if head.is_empty() { + Probe::Silent + } else { + Probe::Accepted + } + } + Ok(n) => { + head.extend_from_slice(&probe[..n]); + match launch_reading(&head) { + LaunchReading::Refused => break Probe::Refused, + // Assistant output spoke for the launch: + // hand the buffered stream over. + LaunchReading::Accepted => break Probe::Accepted, + LaunchReading::Undecided => {} + } + } + }, + } + }; + match outcome { + // Interrupted: the fallback stays unspent; the session + // loop ends the run interrupted. + Probe::Accepted | Probe::Interrupted => {} + Probe::Silent | Probe::Refused => { + shutdown_child(&mut child, self.kill_grace).await; + request.resume = None; + request.provider_fork = None; + fork_flags = None; + if let Some(history) = launch_fallback { + request.prompt = crate::prompt_with_history(&history, &request.prompt); + } + tracing::debug!( + target: "roboco_harness::claude", + "the CLI refused the launch (silent, or an error before any \ + assistant output); starting fresh with the budgeted history" + ); + continue; + } + } + } - let (stdin_tx, stdin_rx) = mpsc::unbounded_channel::(); - tokio::spawn(stdin_writer(stdin, stdin_rx)); - if !title_only { - // The opening control request: `perTaskStopAffordance` tells the - // CLI this host stops tasks individually, so an interrupt ends - // the turn and spares background agents. - let _ = stdin_tx.send(StdinMsg::Line(wire::initialize_request_line( - "roboco_initialize", - ))); + // The probe's consumed byte rides ahead of the pipe (no output + // is lost); an empty head is every unprobed launch. + let stdout_lines = + BufReader::new(std::io::Cursor::new(head).chain(stdout)).lines(); + let (event_tx, event_rx) = mpsc::channel::>(256); + tokio::spawn(run_session(Session { + normalizer, + title_only, + child, + stdout_lines, + first_command, + stdin_tx, + event_tx, + controls, + config: request, + interrupt_grace: self.interrupt_grace, + kill_grace: self.kill_grace, + stderr_tail, + })); + + return Ok(futures::stream::unfold(event_rx, |mut rx| async move { + rx.recv().await.map(|ev| (ev, rx)) + }) + .boxed()); } - - // The initial prompt as the first stdin user line (streaming-input - // mode). Ultrathink rides every user message — steers included. - // Staged image attachments are inlined as base64 image content blocks - // ahead of the text (verified against the real CLI); their path refs - // also ride the prompt text, so a skipped/unreadable file degrades to - // the old-app behavior (the agent opens the path with its Read tool). - // The line carries a uuid: the CLI then reports its - // `command_lifecycle`, which decides when a `result` is a steer - // boundary instead of the turn's end. - let images = load_image_blocks(&request.attachments).await; - let first_command = uuid::Uuid::new_v4().to_string(); - let first = wire::prompt_line( - &apply_ultrathink(request.reasoning, &request.prompt), - &first_command, - &images, - ); - let _ = stdin_tx.send(StdinMsg::Line(first)); - - let (event_tx, event_rx) = mpsc::channel::>(256); - tokio::spawn(run_session(Session { - normalizer, - title_only, - child, - stdout_lines: BufReader::new(stdout).lines(), - first_command, - stdin_tx, - event_tx, - controls, - config: request, - interrupt_grace: self.interrupt_grace, - kill_grace: self.kill_grace, - stderr_tail, - })); - - Ok(futures::stream::unfold(event_rx, |mut rx| async move { - rx.recv().await.map(|ev| (ev, rx)) - }) - .boxed()) + unreachable!("the launch loop returns (or errors) on its final pass") } } @@ -795,11 +1159,19 @@ async fn stdin_writer(mut stdin: ChildStdin, mut rx: mpsc::UnboundedReceiver>, crate::process::ChildStdout>>, +>; + struct Session { normalizer: Normalizer, title_only: bool, child: Child, - stdout_lines: tokio::io::Lines>, + stdout_lines: StdoutLines, /// The uuid the opening prompt was written with. first_command: String, stdin_tx: mpsc::UnboundedSender, @@ -1484,6 +1856,50 @@ mod tests { // Original input is preserved alongside the answers. assert!(updated["questions"].is_array()); } + + /// The fork launch's latest-turn detection (ticket 18): the session + /// store's last TOP-LEVEL message record — tagged subagent records and + /// non-message records (summaries, tags) never count. + #[tokio::test] + async fn the_session_scan_takes_the_last_top_level_message_record() { + let dir = tempfile::tempdir().unwrap(); + let project = dir.path().join("projects/probe"); + std::fs::create_dir_all(&project).unwrap(); + std::fs::write( + project.join("sess-scan.jsonl"), + concat!( + r#"{"type":"user","uuid":"u-1","parent_tool_use_id":null,"message":{"role":"user","content":"first"}}"#, + "\n", + // A tagged subagent record and a non-message record. + r#"{"type":"assistant","uuid":"sub-1","parent_tool_use_id":"toolu_x","isSidechain":true,"message":{"role":"assistant","content":[]}}"#, + "\n", + r#"{"type":"summary","uuid":"trailing","summary":"s"}"#, + "\n", + r#"{"type":"assistant","uuid":"msg-last","parent_tool_use_id":null,"message":{"role":"assistant","content":[]}}"#, + "\n", + // Records after the last message never count either. + r#"{"type":"summary","uuid":"after","summary":"s"}"#, + "\n", + // A torn tail line is harmless. + "{torn", + ), + ) + .unwrap(); + assert_eq!( + session_last_message_uuid(dir.path(), "sess-scan") + .await + .as_deref(), + Some("msg-last") + ); + // No session file (and unreadable ids): no latest turn — the + // launch treats the boundary as an earlier one. + assert_eq!(session_last_message_uuid(dir.path(), "no-such").await, None); + assert_eq!(session_last_message_uuid(dir.path(), "").await, None); + assert_eq!( + session_last_message_uuid(dir.path(), "../escape").await, + None + ); + } } #[cfg(test)] diff --git a/crates/harness/src/claude/normalize.rs b/crates/harness/src/claude/normalize.rs index 29f7d48116..993e6715e3 100644 --- a/crates/harness/src/claude/normalize.rs +++ b/crates/harness/src/claude/normalize.rs @@ -1,7 +1,10 @@ //! Frame → [`AgentEvent`] normalization (init dedupe, subagent tagging, tool //! decoding, error-code mapping). -use roboco_proto::{AgentEvent, DoneStatus, HarnessId, TodoItem, TodoStatus, ToolCall}; +use roboco_proto::{ + AgentErrorKind, AgentEvent, DoneStatus, HarnessId, ProviderTurnRef, TodoItem, TodoStatus, + ToolCall, +}; use serde_json::Value; use super::wire::{ContentBlock, Frame}; @@ -38,6 +41,24 @@ fn rate_window_label(kind: &str) -> &'static str { } } +/// Structured classification for an assistant-level error code: the CLI's +/// terse `rate_limit` says a plan window blocked the turn but not which one +/// (the accompanying `rate_limit_event` usually does, and carries the reset); +/// `authentication_failed` means the login is gone. Everything else stays +/// unclassified. +fn assistant_error_kind(code: &str) -> Option { + match code { + "rate_limit" => Some(AgentErrorKind::UsageLimit { + // Which window was hit is unknown at this surface — the plain + // "usage" label (the fallback window wording). + window: "usage".into(), + resets_at: None, + }), + "authentication_failed" => Some(AgentErrorKind::SignedOut), + _ => None, + } +} + /// Fallback wording for a `result` error whose `errors` array is empty, so the /// turn never ends with a blank (and therefore invisible) error. fn result_error_text(subtype: &str) -> &'static str { @@ -208,6 +229,12 @@ pub(crate) struct Normalizer { assistant_message_id: String, /// Last session id seen (init or result) — used for synthetic Dones. pub session_id: Option, + /// The uuid of the last TOP-LEVEL (untagged) assistant/user frame seen + /// — the turn ref of the turn a `result` closes (t3-orchestration-v2 + /// ticket 18): a fork resumes the provider session at exactly that + /// message (`--resume-session-at`, inclusive). Tagged subagent frames + /// never count (their uuids name the subagent's own nested turns). + last_message_uuid: Option, } impl Normalizer { @@ -220,6 +247,7 @@ impl Normalizer { agent_spawn_tools: std::collections::HashSet::new(), assistant_message_id: new_message_id(), session_id: None, + last_message_uuid: None, } } @@ -333,6 +361,39 @@ impl Normalizer { (prev, self.assistant_message_id.clone()) } + /// Record the uuid of a top-level (untagged) message frame: the + /// candidate turn ref of the turn a later `result` closes — the LAST + /// one before the result wins (see [`Self::turn_ref`]). + fn note_message_uuid(&mut self, uuid: Option<&str>) { + if let Some(uuid) = uuid.filter(|u| !u.is_empty()) { + self.last_message_uuid = Some(uuid.to_owned()); + } + } + + /// The provider turn reference a `result` frame closes (ticket 18): + /// the session id plus the uuid of the last top-level assistant/user + /// frame before the result — what a fork resumes the provider session + /// at (`--resume --fork-session`, `--resume-session-at + /// ` for an earlier boundary, inclusive). `None` when either + /// half is unknown (no message frame carried a uuid, or no session id + /// ever arrived): the engine then records no ref and a later fork + /// falls back to the budgeted paste. + fn turn_ref(&self, result_session: &Option) -> Option { + let session_id = result_session + .clone() + .or_else(|| self.session_id.clone()) + .filter(|id| !id.is_empty())?; + let turn_id = self + .last_message_uuid + .clone() + .filter(|id| !id.is_empty())?; + Some(ProviderTurnRef { + harness: HarnessId::ClaudeCode, + session_id, + turn_id, + }) + } + /// Normalize one stdout frame into 0+ unified events. `interrupted` folds /// a post-interrupt `result` into `Done { status: Interrupted }`. pub fn normalize(&mut self, frame: Frame, interrupted: bool) -> Vec { @@ -501,11 +562,18 @@ impl Normalizer { parent, AgentEvent::Error { message: assistant_error_text(code), + kind: assistant_error_kind(code), }, )); } return out; } + // A top-level message frame: its uuid is the candidate turn + // ref (ticket 18) — the last one before the turn's result + // is what a fork resumes at. Recorded even when the frame + // yields no events (a tool-result-only user frame still + // names a turn boundary). + self.note_message_uuid(f.uuid.as_deref()); // Record spawn tool ids up front: `task_notification` keys on // them; a task may finish before a `task_started` arrives. for b in f.message.blocks() { @@ -593,6 +661,7 @@ impl Normalizer { if let Some(code) = &f.error { out.push(AgentEvent::Error { message: assistant_error_text(code), + kind: assistant_error_kind(code), }); } // The enclosing assistant frame closes the streamed message @@ -642,6 +711,10 @@ impl Normalizer { ); return out; } + // A top-level user frame (tool results, or the CLI's echo of + // a line we wrote): its uuid is the candidate turn ref + // (ticket 18), same as an assistant frame's. + self.note_message_uuid(f.uuid.as_deref()); f.message .blocks() .filter(|b: &ContentBlock| b.kind == "tool_result") @@ -667,6 +740,15 @@ impl Normalizer { message: format!( "Claude {window} limit reached — the turn was blocked. Try again after it resets." ), + kind: Some(AgentErrorKind::UsageLimit { + window: window.into(), + // The CLI reports the reset in unix seconds + // (live-verified 2.1.228); the kind carries millis. + resets_at: f + .rate_limit_info + .resets_at + .map(|secs| (secs as i64).saturating_mul(1000)), + }), }] } @@ -707,7 +789,7 @@ impl Normalizer { }, result: f.result, error: None, - session_id: f.session_id, + session_id: f.session_id.clone(), } } else { // Split the CLI's internal `[ede_diagnostic]` breadcrumbs @@ -756,10 +838,18 @@ impl Normalizer { }, result: None, error, - session_id: f.session_id, + session_id: f.session_id.clone(), } }; let mut out = Vec::new(); + // The provider turn ref (ticket 18) rides FIRST, before the + // result's own events: the engine holds it while the entry + // the turn streamed into finishes, and the Done below — + // Usage and ContextUsage never close a segment — closes it, + // so the ref is stamped on exactly that entry. + if let Some(reference) = self.turn_ref(&f.session_id) { + out.push(AgentEvent::TurnRef { reference }); + } if let Some(window) = window { out.push(AgentEvent::ContextUsage { tokens: None, @@ -1365,6 +1455,108 @@ mod tests { ); } + /// The provider turn ref (t3-orchestration-v2 ticket 18): every + /// `result` closes the turn with the session id plus the uuid of the + /// LAST top-level assistant/user frame before it — tagged subagent + /// frames never count — emitted ahead of the result's own events so + /// the engine stamps it on the entry the turn streamed into. + #[test] + fn result_frames_emit_the_last_top_level_message_uuid_as_the_turn_ref() { + let mut norm = Normalizer::new(); + let init = r#"{"type":"system","subtype":"init","model":"m","cwd":"/x","session_id":"sess-t"}"#; + norm.normalize(crate::claude::wire::parse_frame(init).unwrap(), false); + let frames = [ + r#"{"type":"assistant","uuid":"msg-a","message":{"content":[{"type":"text","text":"a"}]}}"#, + // Tagged (subagent) frames carry uuids too — they name the + // subagent's own nested turns and must never become the ref. + r#"{"type":"assistant","uuid":"msg-sub","parent_tool_use_id":"toolu_s","message":{"content":[{"type":"text","text":"sub"}]}}"#, + r#"{"type":"user","uuid":"msg-b","message":{"content":[{"type":"tool_result","tool_use_id":"t","is_error":false}]}}"#, + r#"{"type":"result","subtype":"success","result":"done","errors":[],"usage":{"input_tokens":1,"output_tokens":2},"session_id":"sess-t"}"#, + ]; + let mut events = Vec::new(); + for raw in frames { + events.extend(norm.normalize(crate::claude::wire::parse_frame(raw).unwrap(), false)); + } + assert_eq!( + events.iter().filter(|e| matches!(e, AgentEvent::TurnRef { .. })).count(), + 1, + "one ref per turn: {events:?}" + ); + assert!(events.contains(&AgentEvent::TurnRef { + reference: ProviderTurnRef { + harness: HarnessId::ClaudeCode, + session_id: "sess-t".into(), + turn_id: "msg-b".into(), + } + })); + // The ref rides ahead of the result's own events — the Done closes + // the entry it must be stamped on. + let done_at = events + .iter() + .rposition(|e| matches!(e, AgentEvent::Done { .. })) + .expect("done"); + let ref_at = events + .iter() + .position(|e| matches!(e, AgentEvent::TurnRef { .. })) + .expect("turn ref"); + assert!(ref_at < done_at); + } + + /// A result whose own frame carries no session id falls back to the + /// last one seen (the init's); frames without uuids, or a session id + /// that never arrived, record no ref at all — a later fork then falls + /// back to the budgeted paste instead. + #[test] + fn turn_refs_need_a_uuid_and_a_session_id() { + let mut norm = Normalizer::new(); + let init = r#"{"type":"system","subtype":"init","model":"m","cwd":"/x","session_id":"sess-fb"}"#; + norm.normalize(crate::claude::wire::parse_frame(init).unwrap(), false); + // No uuid on the message frames, and none on the result: no ref. + for raw in [ + r#"{"type":"assistant","message":{"content":[{"type":"text","text":"a"}]}}"#, + r#"{"type":"result","subtype":"success","usage":{"input_tokens":0,"output_tokens":0}}"#, + ] { + let events = norm.normalize(crate::claude::wire::parse_frame(raw).unwrap(), false); + assert!( + !events.iter().any(|e| matches!(e, AgentEvent::TurnRef { .. })), + "{events:?}" + ); + } + // A uuid arrives; the result still names no session, so the ref + // falls back to the init's — the session a fork must resume. + let frame = r#"{"type":"assistant","uuid":"msg-late","message":{"content":[]}}"#; + norm.normalize(crate::claude::wire::parse_frame(frame).unwrap(), false); + let events = norm.normalize( + crate::claude::wire::parse_frame( + r#"{"type":"result","subtype":"success","usage":{"input_tokens":0,"output_tokens":0}}"#, + ) + .unwrap(), + false, + ); + assert!(events.contains(&AgentEvent::TurnRef { + reference: ProviderTurnRef { + harness: HarnessId::ClaudeCode, + session_id: "sess-fb".into(), + turn_id: "msg-late".into(), + } + })); + // No session id ever seen: a uuid alone is not a ref. + let mut fresh = Normalizer::new(); + let frame = r#"{"type":"assistant","uuid":"msg-2","message":{"content":[]}}"#; + fresh.normalize(crate::claude::wire::parse_frame(frame).unwrap(), false); + let events = fresh.normalize( + crate::claude::wire::parse_frame( + r#"{"type":"result","subtype":"success","usage":{"input_tokens":0,"output_tokens":0}}"#, + ) + .unwrap(), + false, + ); + assert!( + !events.iter().any(|e| matches!(e, AgentEvent::TurnRef { .. })), + "{events:?}" + ); + } + #[test] fn ede_diagnostics_never_surface_as_errors() { // The CLI's internal turn-accounting breadcrumbs must not become @@ -1439,7 +1631,142 @@ mod tests { assert!(normalize_one(&raw).is_empty(), "{overage}"); } } + + #[test] + fn a_blocking_limit_event_carries_the_window_and_reset_time() { + // The blocking event yields kind usage-limit with the window label + // and the CLI's own reset time — unix seconds on the wire (the live + // 2.1.228 capture carries seconds), millis in the kind. The message + // is unchanged, so older clients render exactly as before. + let events = normalize_one( + r#"{"type":"rate_limit_event","rate_limit_info":{"status":"rejected","rateLimitType":"five_hour","resetsAt":1762675400}}"#, + ); + assert_eq!( + events, + vec![AgentEvent::Error { + message: "Claude 5-hour limit reached — the turn was blocked. Try again after it resets.".into(), + kind: Some(AgentErrorKind::UsageLimit { + window: "5-hour".into(), + resets_at: Some(1_762_675_400_000), + }), + }] + ); + // No reset time reported: the kind still lands, without one. + let events = normalize_one( + r#"{"type":"rate_limit_event","rate_limit_info":{"status":"rejected","rateLimitType":"seven_day"}}"#, + ); + assert_eq!( + events, + vec![AgentEvent::Error { + message: "Claude weekly limit reached — the turn was blocked. Try again after it resets.".into(), + kind: Some(AgentErrorKind::UsageLimit { + window: "weekly".into(), + resets_at: None, + }), + }] + ); + } + + #[test] + fn a_malformed_resets_at_never_fails_the_frame() { + // `resetsAt` in every shape the CLI has been seen to send: an + // integer, a float, a numeric string — all parse; anything else + // (null, words, an object) reads as unknown. None of them may fail + // the frame: the window classification rides the same event, and a + // failed deserialize would drop the whole limit report. + let shape = |resets_at: &str| { + let raw = format!( + r#"{{"type":"rate_limit_event","rate_limit_info":{{"status":"rejected","rateLimitType":"five_hour",{resets_at}}}}}"# + ); + normalize_one(&raw) + }; + for (field, expected) in [ + (r#""resetsAt":1762675400"#, Some(1_762_675_400_000i64)), + (r#""resetsAt":1762675400.0"#, Some(1_762_675_400_000)), + (r#""resetsAt":"1762675400""#, Some(1_762_675_400_000)), + (r#""resetsAt":"1762675400.5""#, Some(1_762_675_400_000)), + (r#""resetsAt":null"#, None), + (r#""resetsAt":"soon""#, None), + (r#""resetsAt":{}"#, None), + (r#""resetsAt":0"#, None), + ] { + let events = shape(field); + match events.as_slice() { + [AgentEvent::Error { kind: Some(AgentErrorKind::UsageLimit { resets_at, .. }), .. }] => { + assert_eq!(*resets_at, expected, "{field}"); + } + other => panic!("{field} must still classify the limit: {other:?}"), + } + } + } + + #[test] + fn assistant_error_codes_classify_limits_and_auth() { + // The error message also completes its (empty) assistant message; + // only the error event is under test here. + let errors = |line: &str| { + normalize_one(line) + .into_iter() + .filter(|event| matches!(event, AgentEvent::Error { .. })) + .collect::>() + }; + // The terse `rate_limit` code: a usage limit, window unknown (the + // accompanying rate_limit_event usually carries the details). + let events = errors( + r#"{"type":"assistant","parent_tool_use_id":null,"message":{"content":[]},"error":"rate_limit"}"#, + ); + assert_eq!( + events, + vec![AgentEvent::Error { + message: "Claude usage limit reached — try again after the limit resets.".into(), + kind: Some(AgentErrorKind::UsageLimit { + window: "usage".into(), + resets_at: None, + }), + }] + ); + // Authentication failed: signed out, distinct from a limit. + let events = errors( + r#"{"type":"assistant","parent_tool_use_id":null,"message":{"content":[]},"error":"authentication_failed"}"#, + ); + assert_eq!( + events, + vec![AgentEvent::Error { + message: "Authentication failed — sign in to Claude again.".into(), + kind: Some(AgentErrorKind::SignedOut), + }] + ); + // Other codes stay unclassified. + let events = errors( + r#"{"type":"assistant","parent_tool_use_id":null,"message":{"content":[]},"error":"billing_error"}"#, + ); + assert_eq!( + events, + vec![AgentEvent::Error { + message: "Billing error — check your Claude plan or payment method.".into(), + kind: None, + }] + ); + } + + #[test] + fn subagent_error_codes_carry_the_classification_too() { + let events = normalize_one( + r#"{"type":"assistant","parent_tool_use_id":"toolu_sub","message":{"content":[]},"error":"authentication_failed"}"#, + ); + assert_eq!( + events, + vec![AgentEvent::Subagent { + parent_tool_use_id: "toolu_sub".into(), + event: Box::new(AgentEvent::Error { + message: "Authentication failed — sign in to Claude again.".into(), + kind: Some(AgentErrorKind::SignedOut), + }), + }] + ); + } } + #[cfg(test)] mod context_tests { use super::*; diff --git a/crates/harness/src/claude/wire.rs b/crates/harness/src/claude/wire.rs index abf14adbd8..5c4d0fe3c4 100644 --- a/crates/harness/src/claude/wire.rs +++ b/crates/harness/src/claude/wire.rs @@ -152,6 +152,18 @@ pub(crate) struct RateLimitInfo { pub status: String, #[serde(rename = "rateLimitType", default)] pub rate_limit_type: Option, + /// Unix seconds when the rejected window resets — the CLI reports it on + /// the blocking event (live-verified 2.1.228); absent when unknown. + /// Parsed leniently ([`deserialize_resets_at]): some CLI builds send a + /// float or a numeric string here, and a malformed value must cost + /// the timestamp — never the whole frame (the window classification + /// rides the same event). + #[serde( + rename = "resetsAt", + default, + deserialize_with = "deserialize_resets_at" + )] + pub resets_at: Option, /// `allowed`/`allowed_warning` when the account may spend provisioned /// overage past a rejected window. #[serde(rename = "overageStatus", default)] @@ -162,6 +174,36 @@ pub(crate) struct RateLimitInfo { pub overage_in_use: bool, } +/// `resetsAt` in every shape the CLI has been seen to send: an integer, +/// a float (some builds serialize JS numbers), or a numeric string — +/// parsed to unix seconds. Any other shape (null, a non-numeric string, +/// an object) reads as "unknown", and deserializing never fails the +/// frame. +fn deserialize_resets_at<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + let Some(value) = value else { + return Ok(None); + }; + let seconds = match value { + serde_json::Value::Null => None, + serde_json::Value::Number(number) => number + .as_u64() + .or_else(|| number.as_f64().map(|value| value.trunc() as u64)), + serde_json::Value::String(text) => { + let text = text.trim(); + text.parse::() + .ok() + .or_else(|| text.parse::().ok().map(|value| value.trunc() as u64)) + } + // Any other shape is a CLI oddity: unknown reset, never an error. + _ => None, + }; + Ok(seconds.filter(|seconds| *seconds > 0)) +} + impl RateLimitInfo { /// A rejected window that actually blocks the turn: an account spending /// provisioned overage keeps running despite the reject. diff --git a/crates/harness/src/codex/mod.rs b/crates/harness/src/codex/mod.rs index ce4bebb636..974f22e23a 100644 --- a/crates/harness/src/codex/mod.rs +++ b/crates/harness/src/codex/mod.rs @@ -9,9 +9,13 @@ //! //! - `initialize` handshake (clientInfo + `capabilities.experimentalApi`) then //! the `initialized` notification; unknown notification methods tolerated. -//! - `thread/start` (or `thread/resume` with a fresh-start fallback) → +//! - `thread/start` (or `thread/resume` with a fresh-start fallback; or +//! `thread/fork` with a fresh-start + budgeted-history fallback when the +//! engine sends a native-fork directive, t3-orchestration-v2 ticket 17) → //! `SessionStarted`; `turn/start` carries the prompt, model, effort, -//! `sandboxPolicy`, and approval policy. +//! `sandboxPolicy`, and approval policy. Every adopted turn id also emits +//! an [`AgentEvent::TurnRef`] naming the thread and turn, for the +//! transcript's provider turn refs. //! - Notifications map to [`AgentEvent`]s: agentMessage/reasoning deltas (both //! `delta`/`textDelta` spellings), item lifecycles → typed ToolCall/ToolResult, //! `thread/tokenUsage/updated` → Usage, turn/completed|failed|aborted → Done. @@ -70,8 +74,9 @@ use crate::process::{Child, Command, Stdio}; use crate::{Harness, HarnessError, RunControls}; use catalog::{REASONING_LEVELS, sandbox_mode, sandbox_policy_value, static_models, to_effort}; use normalize::{ - ChildRoute, Phase, ReasoningStream, delta_text, item_id, item_type, notification_thread_id, - route_child_notification, turn_error_message, turn_id, usage_event, + ChildRoute, Phase, RateLimitReset, ReasoningStream, delta_text, item_id, item_type, + notification_thread_id, rate_limit_reset, route_child_notification, turn_error_kind, + turn_error_message, turn_id, usage_event, }; /// Locate the device's installed Codex CLI: `CODEX_EXECUTABLE`, then our own @@ -843,19 +848,24 @@ impl TurnRouter { self.completed.iter().any(|c| c == id) } - fn note_started(&mut self, id: String) { + /// Note a turn the app-server announced (`turn/started`), returning the + /// id when it became the turn in flight — a repeat announcement of the + /// running turn, or one already completed, reports nothing new. + fn note_started(&mut self, id: String) -> Option { if id.is_empty() || self.is_completed(&id) { - return; + return None; } // A replacement `turn/started` is authoritative evidence that a stale // active turn is over, even if its completion notification was lost. + let changed = self.active.as_deref() != Some(id.as_str()); if let Some(prev) = self.active.take() && prev != id { self.remember_completed(prev); } - self.active = Some(id); + self.active = Some(id.clone()); self.announced = true; + changed.then_some(id) } fn note_completed(&mut self, id: &str) { @@ -872,13 +882,15 @@ impl TurnRouter { /// Adopt a turn id from a `turn/start` RESPONSE (the notification is /// allowed to beat it). A turn already running keeps its id: a follow-up /// `turn/start` folds into it and answers with that same id anyway, and - /// an announced turn is authoritative. - fn adopt_started(&mut self, id: String) { + /// an announced turn is authoritative. Returns the id when it was newly + /// adopted, so the caller emits its turn ref exactly once. + fn adopt_started(&mut self, id: String) -> Option { if self.active.is_some() || id.is_empty() || self.is_completed(&id) { - return; + return None; } - self.active = Some(id); + self.active = Some(id.clone()); self.announced = false; + Some(id) } /// Whether a `turn/completed` (or failed/aborted) for `id` ends the turn @@ -908,6 +920,20 @@ fn rotate(id: &mut String) -> (String, String) { (prev, id.clone()) } +/// The provider turn reference for a turn that just started +/// (t3-orchestration-v2 ticket 17): the engine records it on the transcript +/// entry streaming into this turn, so a fork can continue the thread at +/// exactly that turn (`thread/fork {threadId, lastTurnId}`). +fn turn_ref_event(thread_id: &str, turn_id: &str) -> AgentEvent { + AgentEvent::TurnRef { + reference: roboco_proto::ProviderTurnRef { + harness: HarnessId::Codex, + session_id: thread_id.to_owned(), + turn_id: turn_id.to_owned(), + }, + } +} + async fn send(tx: &mpsc::Sender>, ev: AgentEvent) -> bool { tx.send(Ok(ev)).await.is_ok() } @@ -1090,6 +1116,13 @@ async fn run_session(session: Session) { }; // ---- handshake + thread + first turn (interruptible) ------------------ + // The budgeted transcript the engine attached for a refused fork; the + // interrupted-start path leaves it unset (nothing was attempted). + let mut fork_fallback: Option = None; + // The same for a REFUSED RESUME (account failover, ticket 09): the + // engine attaches the chat's budgeted history to the continuation so a + // fresh thread still knows the conversation. + let mut resume_fallback: Option = None; let setup = async { client .request( @@ -1123,7 +1156,33 @@ async fn run_session(session: Session) { } } } - let thread = if let Some(resume) = &request.resume { + // A native fork (t3-orchestration-v2 ticket 17): fork the source + // thread at the inclusive boundary instead of resuming it — the + // forked thread carries the copied history, so nothing is pasted + // into the prompt. A refused fork falls back to a fresh thread, and + // the engine's budgeted transcript then rides in front of the prompt + // (`fork_fallback`, applied to the first turn below). + let thread = if let Some(fork) = request.provider_fork.clone() { + let mut p = start_params.clone(); + p.insert("threadId".into(), Value::String(fork.session_id.clone())); + p.insert("lastTurnId".into(), Value::String(fork.last_turn.clone())); + match client.request("thread/fork", Value::Object(p)).await { + Ok(thread) => thread, + // A fork the app-server refuses (experimental API off, a + // boundary it no longer knows) is a fallback, not an error: + // start fresh and deliver the budgeted history. + Err(e) => { + tracing::debug!( + target: "roboco_harness::codex", + "thread/fork refused (starting fresh with the budgeted history): {e}" + ); + fork_fallback = fork.fallback_history; + client + .request("thread/start", Value::Object(start_params.clone())) + .await? + } + } + } else if let Some(resume) = &request.resume { let mut p = start_params.clone(); p.insert("threadId".into(), Value::String(resume.clone())); match client.request("thread/resume", Value::Object(p)).await { @@ -1137,6 +1196,10 @@ async fn run_session(session: Session) { target: "roboco_harness::codex", "thread/resume failed (starting fresh): {e}" ); + // A refused resume with the engine's budgeted fallback + // history (ticket 09's account-failover continuation): + // the fresh thread still owes the chat its context. + resume_fallback = request.resume_fallback_history.clone(); client .request("thread/start", Value::Object(start_params.clone())) .await? @@ -1182,6 +1245,18 @@ async fn run_session(session: Session) { } }; + // A refused fork's fresh thread still owes the chat its prior context: + // the budgeted transcript rides in front of the prompt, exactly as the + // engine would have pasted it for a fresh session. The prompt itself + // still governs native-command routing below (a native command never + // carries a fallback — the engine attaches none). A refused RESUME + // delivers the same way (ticket 09). + let delivered_prompt = match (&fork_fallback, &resume_fallback) { + (Some(history), _) | (_, Some(history)) => { + crate::prompt_with_history(history, &request.prompt) + } + (None, None) => request.prompt.clone(), + }; // Model, effort and service tier ride every `turn/start` (each sticks // for later turns): a steer's in-place `config` takes effect from the // next turn the runtime starts. @@ -1228,8 +1303,18 @@ async fn run_session(session: Session) { } let mut router = TurnRouter::default(); - match start_turn(&client, turn_params(&live, &request.prompt)).await { - Ok(id) => router.adopt_started(id), + match start_turn(&client, turn_params(&live, &delivered_prompt)).await { + Ok(id) => { + // Emit the first turn's ref right after SessionStarted and + // before any item deltas: the entry about to stream belongs to + // this turn. + if let Some(started) = router.adopt_started(id) + && !send(&event_tx, turn_ref_event(&thread_id, &started)).await + { + shutdown_child(&mut child, kill_grace).await; + return; + } + } Err(e) => { let _ = event_tx .send(Ok(AgentEvent::Done { @@ -1264,6 +1349,10 @@ async fn run_session(session: Session) { let mut accepted_steers: VecDeque = VecDeque::new(); // A turn stop waiting for the app-server to announce the turn. let mut stop_when_announced = false; + // The latest `account/rateLimits/updated` snapshot reduced to the window + // it reports exhausted. Turn-level usage-limit errors attach this reset + // time; the notifications themselves stay otherwise unconsumed. + let mut limit_reset: Option = None; // Commands the parent thread started and the app-server has not reported // finished, with their process id and the turn that started them. One // still open after its turn is background work (a dev server, a @@ -1343,7 +1432,15 @@ async fn run_session(session: Session) { break 'main; } } - router.note_started(id); + // The turn ref rides the boundary: the entry the + // app-server is about to stream into belongs to this + // turn, whatever started it (a steer, a native + // handoff, a background wake). + if let Some(started) = router.note_started(id) { + if !send(&event_tx, turn_ref_event(&thread_id, &started)).await { + break 'main; + } + } if std::mem::take(&mut stop_when_announced) && let Some(active) = router.active.clone() { @@ -1487,6 +1584,14 @@ async fn run_session(session: Session) { } } + // The account's usage meters, pushed on token-count + // events. Nothing is surfaced directly: the snapshot is + // held so a turn-level usage-limit error can carry the + // reset time the error itself omits. + "account/rateLimits/updated" => { + limit_reset = rate_limit_reset(¶ms); + } + // Codex's `update_plan` tool: the whole checklist, replaced // on every call, with an in-progress step. "turn/plan/updated" => { @@ -1539,6 +1644,24 @@ async fn run_session(session: Session) { == Some("failed")) .then(|| "Codex turn failed".to_owned()) }); + // A classified failure (usage limit, signed out) + // surfaces as an Error event ahead of the Done so the + // kind reaches engine and clients; its text moves off + // the Done so the transcript keeps exactly ONE error + // part. Unclassified failures keep their historical + // Done-only shape. + let kind = turn_error_kind(¶ms, limit_reset.as_ref()); + let mut text_rides_error_event = false; + if let (Some(message), Some(kind)) = (&error, &kind) { + let classified = AgentEvent::Error { + message: message.clone(), + kind: Some(kind.clone()), + }; + if !send(&event_tx, classified).await { + break 'main; + } + text_rides_error_event = true; + } let status = if interrupted || stopping { DoneStatus::Interrupted } else if error.is_some() { @@ -1560,7 +1683,7 @@ async fn run_session(session: Session) { AgentEvent::Done { status, result: None, - error, + error: if text_rides_error_event { None } else { error }, session_id: Some(thread_id.clone()), }, ) @@ -1584,6 +1707,7 @@ async fn run_session(session: Session) { command_request(&text, &thread_id).ok().flatten().is_some(); if !steer_as_new_turn( &client, + &thread_id, turn_params(&live, &text), &mut router, &event_tx, @@ -1634,6 +1758,22 @@ async fn run_session(session: Session) { if interrupted { done_after_interrupt = true; } + let error = + turn_error_message(¶ms).unwrap_or_else(|| "Codex turn failed".into()); + // Same classified-failure shape as turn/completed: the + // kind rides an Error event ahead of the Done, which + // then carries no error text of its own — one rendered + // error part either way. + let kind = turn_error_kind(¶ms, limit_reset.as_ref()); + if let Some(kind) = &kind { + let classified = AgentEvent::Error { + message: error.clone(), + kind: Some(kind.clone()), + }; + if !send(&event_tx, classified).await { + break 'main; + } + } let _ = send( &event_tx, AgentEvent::Done { @@ -1643,10 +1783,7 @@ async fn run_session(session: Session) { DoneStatus::Errored }, result: None, - error: Some( - turn_error_message(¶ms) - .unwrap_or_else(|| "Codex turn failed".into()), - ), + error: if kind.is_none() { Some(error) } else { None }, session_id: Some(thread_id.clone()), }, ) @@ -1707,7 +1844,8 @@ async fn run_session(session: Session) { .or_else(|| params.get("message").and_then(Value::as_str)) .unwrap_or("Codex error") .to_owned(); - if !send(&event_tx, AgentEvent::Error { message }).await { + let kind = turn_error_kind(¶ms, limit_reset.as_ref()); + if !send(&event_tx, AgentEvent::Error { message, kind }).await { break 'main; } } @@ -1792,7 +1930,7 @@ async fn run_session(session: Session) { } else { current_native = command_request(&text, &thread_id).ok().flatten().is_some(); if !steer_as_new_turn( - &client, turn_params(&live, &text), &mut router, &event_tx, + &client, &thread_id, turn_params(&live, &text), &mut router, &event_tx, &mut assistant_message_id, &mut done_current, ).await { break 'main; } } @@ -1801,7 +1939,7 @@ async fn run_session(session: Session) { } else { current_native = command_request(&text, &thread_id).ok().flatten().is_some(); if !steer_as_new_turn( - &client, turn_params(&live, &text), &mut router, &event_tx, + &client, &thread_id, turn_params(&live, &text), &mut router, &event_tx, &mut assistant_message_id, &mut done_current, ).await { break 'main; } } @@ -1982,6 +2120,7 @@ fn send_turn_interrupt(client: &RpcClient, thread_id: &str, turn: String) { /// when the loop should end (turn/start failed or the consumer hung up). async fn steer_as_new_turn( client: &RpcClient, + thread_id: &str, params: Value, router: &mut TurnRouter, event_tx: &mpsc::Sender>, @@ -1990,23 +2129,34 @@ async fn steer_as_new_turn( ) -> bool { match start_turn(client, params).await { Ok(id) => { - router.adopt_started(id); *done_current = false; let (prev, next) = rotate(assistant_message_id); - send( + // The boundary first (the entry rotates), then the new turn's + // ref for the entry that is about to stream. + let steered = send( event_tx, AgentEvent::Steered { assistant_message_id: Some(prev), next_assistant_message_id: Some(next), }, ) - .await + .await; + if !steered { + return false; + } + match router.adopt_started(id) { + Some(started) => { + send(event_tx, turn_ref_event(thread_id, &started)).await + } + None => steered, + } } Err(e) => { let _ = send( event_tx, AgentEvent::Error { message: format!("Steering failed: {e}"), + kind: None, }, ) .await; @@ -2308,19 +2458,25 @@ mod tests { let mut r = TurnRouter::default(); r.note_completed("t-1"); // The turn/start response arriving after turn/completed must not - // resurrect the turn. - r.adopt_started("t-1".into()); + // resurrect the turn — and adopts nothing. + assert_eq!(r.adopt_started("t-1".into()), None); assert_eq!(r.active, None); // Nor may a late turn/started notification. - r.note_started("t-1".into()); + assert_eq!(r.note_started("t-1".into()), None); assert_eq!(r.active, None); - r.note_started("t-2".into()); + // A first announcement reports the turn it started. + assert_eq!(r.note_started("t-2".into()), Some("t-2".into())); assert_eq!(r.active.as_deref(), Some("t-2")); + // Re-announcing the running turn reports nothing new. + assert_eq!(r.note_started("t-2".into()), None); // A replacement started turn retires the stale one. - r.note_started("t-3".into()); + assert_eq!(r.note_started("t-3".into()), Some("t-3".into())); assert_eq!(r.active.as_deref(), Some("t-3")); assert!(r.is_completed("t-2")); + // An adoption while a turn runs folds into it and adopts nothing. + assert_eq!(r.adopt_started("t-4".into()), None); + assert_eq!(r.active.as_deref(), Some("t-3")); } } diff --git a/crates/harness/src/codex/normalize.rs b/crates/harness/src/codex/normalize.rs index 5cb7c437d0..c56f585b00 100644 --- a/crates/harness/src/codex/normalize.rs +++ b/crates/harness/src/codex/normalize.rs @@ -5,7 +5,7 @@ //! (`delta`/`textDelta`, `exitCode`/`exit_code`, camelCase/snake_case item //! types) are accepted, and unknown item types map to nothing. -use roboco_proto::{AgentEvent, DoneStatus, TodoItem, TodoStatus, ToolCall}; +use roboco_proto::{AgentErrorKind, AgentEvent, DoneStatus, TodoItem, TodoStatus, ToolCall}; use serde_json::Value; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -122,6 +122,138 @@ pub(crate) fn turn_error_message(params: &Value) -> Option { }) } +/// The error object a turn-level error notification carries: `turn.error` +/// on turn/completed|failed, the bare `error` on the top-level `error` +/// notification. +fn turn_error_object(params: &Value) -> Option<&Value> { + params + .get("turn") + .and_then(|t| t.get("error")) + .filter(|e| !e.is_null()) + .or_else(|| params.get("error").filter(|e| !e.is_null())) +} + +/// The app-server's structured turn-error code (`codexErrorInfo`, both +/// spellings it has shipped): `"usageLimitExceeded"`, `"rateLimitExceeded"`, +/// `"unauthorized"`, … — absent on older builds and plain failures. +pub(crate) fn turn_error_code(params: &Value) -> Option<&str> { + let error = turn_error_object(params)?; + field(error, &["codexErrorInfo", "codex_error_info"]).and_then(Value::as_str) +} + +/// The latest `account/rateLimits/updated` snapshot reduced to the window it +/// reports as exhausted: `primary` is the ~5-hour window, `secondary` the +/// weekly one (the snapshot's own `windowDurationMins` picks the label). +/// When both are exhausted the LATER reset is the binding constraint. +/// `resets_at` arrives as unix seconds; the kind carries epoch millis. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RateLimitReset { + pub window: String, + /// Epoch millis. + pub resets_at: i64, +} + +/// Which window label a usage-window duration gets — the same 5-hour/weekly +/// vocabulary the Claude driver uses, so clients see one label scheme. +fn window_label(minutes: i64) -> &'static str { + match minutes { + 300 => "5-hour", + 10_080 => "weekly", + _ => "usage", + } +} + +/// `params.rateLimits` → the exhausted window's label and reset, when the +/// snapshot says one is exhausted. Both camelCase and snake_case field +/// spellings, and missing windows, are tolerated. +pub(crate) fn rate_limit_reset(params: &Value) -> Option { + let limits = field(params, &["rateLimits", "rate_limits"])?; + let mut binding: Option = None; + for (name, default_minutes) in [("primary", 300i64), ("secondary", 10_080)] { + let Some(window) = field(limits, &[name]) else { continue }; + let exhausted = field(window, &["usedPercent", "used_percent"]) + .and_then(Value::as_f64) + .unwrap_or(0.0) + >= 100.0; + if !exhausted { + continue; + } + let Some(resets_at) = field(window, &["resetsAt", "resets_at"]).and_then(Value::as_i64) + else { + continue; + }; + let minutes = field(window, &["windowDurationMins", "window_duration_mins"]) + .and_then(Value::as_i64) + .unwrap_or(default_minutes); + let candidate = RateLimitReset { + window: window_label(minutes).into(), + resets_at: resets_at.saturating_mul(1000), + }; + if binding + .as_ref() + .map_or(true, |b| candidate.resets_at > b.resets_at) + { + binding = Some(candidate); + } + } + binding +} + +/// Map a structured turn-error code onto the structured error kind: +/// usage limits become a usage limit carrying the latest known reset; +/// auth failures (`unauthorized`) become signed out. Everything else +/// stays unclassified. +/// +/// `usageLimitExceeded` is the plan limit itself — the error says the +/// limit is reached, so it is always classified (the snapshot only +/// refines the window and reset). `rateLimitExceeded` is AMBIGUOUS: a +/// plain 429 says nothing about the plan, and a transient one must stay +/// an unclassified error (retried like any failure), so it classifies +/// only when the snapshot proves a blocked window — an exhausted window +/// whose reset is still AHEAD. A reset already in the past is no block +/// at all (the window has since lifted), so it is ignored: the error +/// reads as transient. The same past reset is dropped from a plan limit +/// too: it would arm a continuation that fires at once into the wall. +pub(crate) fn error_kind(code: &str, limit: Option<&RateLimitReset>) -> Option { + let limit = limit.filter(|reset| reset.resets_at > now_ms()); + match code { + "usageLimitExceeded" | "usage_limit_exceeded" => Some(match limit { + Some(reset) => AgentErrorKind::UsageLimit { + window: reset.window.clone(), + resets_at: Some(reset.resets_at), + }, + None => AgentErrorKind::UsageLimit { + window: "usage".into(), + resets_at: None, + }, + }), + "rateLimitExceeded" | "rate_limit_exceeded" => limit + .map(|reset| AgentErrorKind::UsageLimit { + window: reset.window.clone(), + resets_at: Some(reset.resets_at), + }), + "unauthorized" => Some(AgentErrorKind::SignedOut), + _ => None, + } +} + +/// Epoch milliseconds now (the past-reset filter's clock). +fn now_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_millis() as i64) + .unwrap_or_default() +} + +/// The classification for a turn-level error notification: the code it +/// carries, mapped with the latest rate-limit snapshot when one arrived. +pub(crate) fn turn_error_kind( + params: &Value, + limit: Option<&RateLimitReset>, +) -> Option { + turn_error_code(params).and_then(|code| error_kind(code, limit)) +} + /// `thread/tokenUsage/updated` → a [`AgentEvent::Usage`] snapshot of the LAST /// turn's tokens (held by the session loop, emitted before `Done`). pub(crate) fn usage_event(params: &Value) -> Option { @@ -309,6 +441,9 @@ pub(crate) fn map_item(phase: Phase, item: &Value) -> Vec { if let Some(message) = error { events.push(AgentEvent::Error { message: message.into(), + // Image-generation quota is item-scoped, not a turn-level + // plan limit: stays unclassified. + kind: None, }); } else { let name = std::path::Path::new(&path) @@ -419,6 +554,8 @@ pub(crate) fn map_item(phase: Phase, item: &Value) -> Vec { } "error" => vec![AgentEvent::Error { message: str_field(item, &["message"]), + // Item-scoped transcript error: not a turn-level plan limit. + kind: None, }], "collabAgentToolCall" | "collab_agent_tool_call" => { let tool = str_field(item, &["tool"]); @@ -643,6 +780,10 @@ impl ChildStream { .or_else(|| params.get("message").and_then(Value::as_str)) .unwrap_or("Codex subagent error") .to_owned(), + // A subagent's failure carries the same code shape, but the + // account-wide rate-limit snapshot lives in the driver loop — + // classified here without a reset time. + kind: turn_error_kind(params, None), }], _ => Vec::new(), } @@ -1081,6 +1222,156 @@ mod tests { assert_eq!(turn_error_message(&json!({"turn": {"id": "t"}})), None); assert_eq!(turn_error_message(&json!({"turn": {"error": null}})), None); } + + #[test] + fn turn_error_code_reads_turn_and_bare_notification_shapes() { + // turn/failed|completed: under `turn.error`, both spellings. + assert_eq!( + turn_error_code(&json!({"turn": {"error": { + "message": "usage limit", "codexErrorInfo": "usageLimitExceeded" + }}})), + Some("usageLimitExceeded") + ); + assert_eq!( + turn_error_code(&json!({"turn": {"error": { + "message": "usage limit", "codex_error_info": "rate_limit_exceeded" + }}})), + Some("rate_limit_exceeded") + ); + // The bare `error` notification: under `error`. + assert_eq!( + turn_error_code(&json!({"error": { + "message": "unauthorized", "codexErrorInfo": "unauthorized" + }})), + Some("unauthorized") + ); + // Plain failures and older builds carry no code. + assert_eq!(turn_error_code(&json!({"turn": {"error": {"message": "boom"}}})), None); + assert_eq!(turn_error_code(&json!({"message": "flat"})), None); + } + + #[test] + fn error_codes_classify_limits_and_auth() { + // A reset still ahead (2100) and one already past (2020). + let snapshot = RateLimitReset { + window: "weekly".into(), + resets_at: 4_102_444_800_000, + }; + let lifted = RateLimitReset { + window: "weekly".into(), + resets_at: 1_577_836_800_000, + }; + // Both limit codes, both spellings, with a live exhausted window. + for code in [ + "usageLimitExceeded", + "usage_limit_exceeded", + "rateLimitExceeded", + "rate_limit_exceeded", + ] { + assert_eq!( + error_kind(code, Some(&snapshot)), + Some(AgentErrorKind::UsageLimit { + window: "weekly".into(), + resets_at: Some(4_102_444_800_000), + }), + "{code}" + ); + } + // The plan limit classifies on the error alone; a past reset is + // ignored, never carried. + for code in ["usageLimitExceeded", "usage_limit_exceeded"] { + for limit in [None, Some(&lifted)] { + assert_eq!( + error_kind(code, limit), + Some(AgentErrorKind::UsageLimit { + window: "usage".into(), + resets_at: None, + }), + "{code} without a live snapshot" + ); + } + } + // A bare 429 (no exhausted window, or one that already lifted) is + // transient: unclassified. + for code in ["rateLimitExceeded", "rate_limit_exceeded"] { + assert_eq!(error_kind(code, None), None, "{code}"); + assert_eq!(error_kind(code, Some(&lifted)), None, "{code} past reset"); + } + assert_eq!(error_kind("unauthorized", None), Some(AgentErrorKind::SignedOut)); + assert_eq!(error_kind("unauthorized", Some(&snapshot)), Some(AgentErrorKind::SignedOut)); + // Everything else stays unclassified. + for code in ["serverOverloaded", "streamError", ""] { + assert_eq!(error_kind(code, Some(&snapshot)), None, "{code}"); + } + } + + #[test] + fn rate_limit_reset_picks_the_exhausted_windows_binding_reset() { + // Primary exhausted: the 5-hour window, seconds → millis. + assert_eq!( + rate_limit_reset(&json!({"rateLimits": { + "primary": {"usedPercent": 100, "windowDurationMins": 300, "resetsAt": 1735689600}, + "secondary": {"usedPercent": 61, "windowDurationMins": 10080, "resetsAt": 1736208000} + }})), + Some(RateLimitReset { window: "5-hour".into(), resets_at: 1_735_689_600_000 }) + ); + // Both exhausted: the later reset binds (the weekly window). + assert_eq!( + rate_limit_reset(&json!({"rateLimits": { + "primary": {"usedPercent": 100, "windowDurationMins": 300, "resetsAt": 1735689600}, + "secondary": {"usedPercent": 100, "windowDurationMins": 10080, "resetsAt": 1736208000} + }})), + Some(RateLimitReset { window: "weekly".into(), resets_at: 1_736_208_000_000 }) + ); + // Only the weekly window is reported/exhausted; the label follows + // the snapshot's own duration, and snake_case is tolerated. + assert_eq!( + rate_limit_reset(&json!({"rate_limits": { + "secondary": {"used_percent": 100, "window_duration_mins": 10080, "resets_at": 1736208000} + }})), + Some(RateLimitReset { window: "weekly".into(), resets_at: 1_736_208_000_000 }) + ); + // Nothing exhausted, or no snapshot at all: no reset to report. + assert_eq!( + rate_limit_reset(&json!({"rateLimits": { + "primary": {"usedPercent": 42, "windowDurationMins": 300, "resetsAt": 1735689600}, + "secondary": {"usedPercent": 61, "windowDurationMins": 10080, "resetsAt": 1736208000} + }})), + None + ); + assert_eq!(rate_limit_reset(&json!({"message": "x"})), None); + // An exhausted window whose reset the snapshot omits is skipped, not + // fatal; a missing duration falls back to the slot's usual window. + assert_eq!( + rate_limit_reset(&json!({"rateLimits": { + "primary": {"usedPercent": 100}, + "secondary": {"usedPercent": 100, "resetsAt": 1736208000} + }})), + Some(RateLimitReset { window: "weekly".into(), resets_at: 1_736_208_000_000 }) + ); + } + + #[test] + fn subagent_error_notifications_carry_the_classification() { + let mut feed = ChildStream::default(); + let events = feed.map( + "child", + "error", + &json!({"threadId": "child", "error": { + "message": "You've hit your usage limit", "codexErrorInfo": "usageLimitExceeded" + }}), + ); + assert_eq!( + events, + vec![AgentEvent::Error { + message: "You've hit your usage limit".into(), + kind: Some(AgentErrorKind::UsageLimit { + window: "usage".into(), + resets_at: None, + }), + }] + ); + } } #[cfg(test)] @@ -1194,7 +1485,8 @@ mod generated_image_tests { assert_eq!( events[2], AgentEvent::Error { - message: expected.into() + message: expected.into(), + kind: None } ); } diff --git a/crates/harness/src/lib.rs b/crates/harness/src/lib.rs index f740ae52fe..c21065428f 100644 --- a/crates/harness/src/lib.rs +++ b/crates/harness/src/lib.rs @@ -41,7 +41,16 @@ pub enum HarnessError { Io(#[from] std::io::Error), } +/// The prompt a fresh session receives when a native fork or resume falls +/// back to the engine's budgeted transcript: the history in front of the +/// prompt, verbatim (the engine's transcript carries its own separator). +/// Every driver delivers the fallback through this one spelling. +pub(crate) fn prompt_with_history(history: &str, prompt: &str) -> String { + format!("{history}{prompt}") +} + /// A steer prompt pushed into a live run; delivered at the harness's steering boundary. +#[derive(Clone)] pub struct SteerMessage { pub prompt: String, pub message_id: Option, diff --git a/crates/harness/src/mock.rs b/crates/harness/src/mock.rs index c86c55ea29..766d03457c 100644 --- a/crates/harness/src/mock.rs +++ b/crates/harness/src/mock.rs @@ -5,8 +5,8 @@ use futures::StreamExt; use futures::stream::BoxStream; use roboco_proto::{ - AgentEvent, DoneStatus, HarnessId, Model, ReasoningLevel, RunRequest, SteeringMode, - UserInputQuestion, + AgentErrorKind, AgentEvent, DoneStatus, HarnessId, Model, ReasoningLevel, RunRequest, + SteeringMode, UserInputQuestion, }; use crate::{Harness, HarnessError, RunControls}; @@ -192,6 +192,13 @@ impl Harness for MockHarness { let (body, tail) = self.script.split_at(done_ix); let error_event = mock_error.then(|| AgentEvent::Error { message: "Claude usage limit reached — try again after the limit resets.".into(), + // The same classification the Claude driver attaches to its + // `rate_limit` error code, so engine-side tests can assert on the + // structured kind without a real CLI. + kind: Some(AgentErrorKind::UsageLimit { + window: "usage".into(), + resets_at: None, + }), }); // Dev/testing knob: `ROBOCO_MOCK_CODE=1` appends rust + ts code blocks // (keywords, strings, numbers, comments) plus inline code — for diff --git a/crates/harness/src/opencode/mod.rs b/crates/harness/src/opencode/mod.rs index d4f3965b91..df07a9092b 100644 --- a/crates/harness/src/opencode/mod.rs +++ b/crates/harness/src/opencode/mod.rs @@ -35,6 +35,18 @@ //! THAT session — exactly what the desktop's working-predicate keys on. //! Busy is re-asserted at the top of every agent-loop iteration, so idle //! after busy is authoritative, not a lull. +//! - every prompt carries a client-chosen user message id (1.x `messageID`, +//! 2.x `id`, `msg_`-prefixed) and emits an [`AgentEvent::TurnRef`] naming +//! it: the engine records the ref on the turn's transcript entry, so a +//! fork can continue the provider session at exactly that turn +//! (t3-orchestration-v2 ticket 20). +//! - a native fork (ticket 20) forks the SOURCE provider session before +//! prompting — 1.x `POST /session/{id}/fork {messageID}`, 2.x +//! `POST /api/session/{id}/fork {before}` — both EXCLUSIVE at a message +//! id, so the cut is the NEXT turn's user message, resolved from the +//! session's message list; with no next turn the whole session is +//! copied. A refused or unresolvable fork falls back to a fresh session +//! with the engine's budgeted transcript pasted in front of the prompt. //! - reasoning streams as reasoning parts on both wires → //! [`AgentEvent::ReasoningDelta`], the thinking feed. //! @@ -961,6 +973,27 @@ impl Server { .await .map(|_| ()) } + + /// 2.x: the agent rides the session too (a create carries it in the + /// body; a resumed or forked session re-applies it here). A no-op on + /// 1.x, which has no session-scoped agent route — the run's agent + /// rides the prompt wire there or stays the server default. + async fn adopt_agent( + &self, + session_id: &str, + agent: Option<&str>, + directory: Option<&str>, + ) -> Result<(), HarnessError> { + if self.protocol().await == Protocol::V2 + && let Some(agent) = agent + { + let path = format!("/api/session/{session_id}/agent"); + self.post_json(&path, directory, &json!({ "agent": agent })) + .await + .map(|_| ())?; + } + Ok(()) + } } fn http_client() -> reqwest::Client { @@ -1330,6 +1363,30 @@ fn new_message_id() -> String { uuid::Uuid::new_v4().to_string() } +/// A client-chosen user message id for a posted turn +/// (t3-orchestration-v2 ticket 20): both wires take one on the prompt (1.x +/// `messageID`, 2.x `id`), which becomes the message's server-side id — +/// the turn's provider ref, and the anchor a later fork cuts the session +/// at. 2.x requires the `msg_` prefix; a fresh uuid keeps it unique across +/// the server (2.x answers a repeat id in another session with a 409). +fn new_user_message_id() -> String { + format!("msg_{}", uuid::Uuid::new_v4().simple()) +} + +/// The provider turn reference for a prompt just posted (ticket 20): the +/// client-chosen user message id is the turn's ref — the engine stamps it +/// on the transcript entry the turn streams, so a fork can continue the +/// provider session at exactly that turn. +fn turn_ref_event(session_id: &str, user_message_id: &str) -> AgentEvent { + AgentEvent::TurnRef { + reference: roboco_proto::ProviderTurnRef { + harness: HarnessId::Opencode, + session_id: session_id.to_owned(), + turn_id: user_message_id.to_owned(), + }, + } +} + /// Rotate the assistant message id; returns (previous, next). fn rotate(id: &mut String) -> (String, String) { let prev = std::mem::replace(id, new_message_id()); @@ -1510,41 +1567,57 @@ async fn run_session(session: Session) { .and_then(Value::as_str) .filter(|a| !a.is_empty()); - // ---- session create/resume ------------------------------------------- + // ---- session create/fork/resume -------------------------------------- + // The budgeted transcript a refused native fork falls back to (ticket + // 20): set when the fork fails, applied in front of the first prompt + // below. The interrupted-start path leaves it unset (nothing was + // attempted). + let mut fork_fallback: Option = None; let setup = async { - let session_id = match &request.resume { - Some(resume) => { - // Sessions are durable server-side: resume = reuse the id. - match server.session_info(resume, dir).await { - Ok(info) => { - let id = info - .get("id") - .and_then(Value::as_str) - .unwrap_or(resume) - .to_owned(); - if server.protocol().await == Protocol::V2 - && let Some(agent) = agent - { - server - .post_json( - &format!("/api/session/{id}/agent"), - dir, - &json!({"agent": agent}), - ) - .await?; - } - id - } - Err(e) => { - tracing::debug!( - target: "roboco_harness::opencode", - "session resume failed (starting fresh): {e}" - ); - create_session(&server, dir, agent).await? - } + let session_id = if let Some(fork) = request.provider_fork.clone() { + // A native fork (t3-orchestration-v2 ticket 20): fork the source + // provider session at the boundary turn instead of starting + // fresh — the copied history rides the provider session, so + // nothing is pasted into the prompt. A fork the server refuses + // (an unknown boundary, a call this build can't make) is a + // fallback, not an error: start fresh and deliver the budgeted + // history the engine attached. + match fork_session(&server, dir, &fork).await { + Ok(forked) => { + server.adopt_agent(&forked, agent, dir).await?; + forked + } + Err(e) => { + tracing::debug!( + target: "roboco_harness::opencode", + "session fork refused (starting fresh with the budgeted history): {e}" + ); + fork_fallback = fork.fallback_history; + create_session(&server, dir, agent).await? } } - None => create_session(&server, dir, agent).await?, + } else if let Some(resume) = &request.resume { + // Sessions are durable server-side: resume = reuse the id. + match server.session_info(resume, dir).await { + Ok(info) => { + let id = info + .get("id") + .and_then(Value::as_str) + .unwrap_or(resume) + .to_owned(); + server.adopt_agent(&id, agent, dir).await?; + id + } + Err(e) => { + tracing::debug!( + target: "roboco_harness::opencode", + "session resume failed (starting fresh): {e}" + ); + create_session(&server, dir, agent).await? + } + } + } else { + create_session(&server, dir, agent).await? }; // Provider catalog: resolves the model's advertised reasoning @@ -1593,6 +1666,21 @@ async fn run_session(session: Session) { } }; + // A refused fork's fresh session still owes the chat its prior context: + // the budgeted transcript rides in front of the prompt, byte-identical + // to the paste a non-native fork delivers (the engine leaves OpenCode's + // canonical conversion to this driver, so the history converts here + // exactly as the prompt did in `run`). The prompt itself still governs + // native-command routing below — the engine attaches no fallback to a + // native command, so the concatenation can never hide one. + let delivered_prompt = match &fork_fallback { + Some(history) => crate::prompt_with_history( + &roboco_proto::invocation::harness_prompt(history, HarnessId::Opencode), + &request.prompt, + ), + None => request.prompt.clone(), + }; + let (mut model, mut variant) = model_selection(&providers, &request); let context_windows: HashMap = providers .all @@ -1686,13 +1774,15 @@ async fn run_session(session: Session) { let stall = stall_bound(); let (command_failure_tx, mut command_failure_rx) = mpsc::unbounded_channel(); let mut turn_generation = 0_u64; + let first_user_message_id = new_user_message_id(); if let Err(e) = post_prompt( &server, &bus_tx, + &event_tx, &session_id, dir, &commands, - &request.prompt, + &delivered_prompt, initial_native_command_selected, turn_generation, &command_failure_tx, @@ -1700,6 +1790,7 @@ async fn run_session(session: Session) { model: model.as_ref(), variant: variant.as_deref(), attachments: &request.attachments, + user_message_id: &first_user_message_id, }, ) .await @@ -1708,6 +1799,7 @@ async fn run_session(session: Session) { &event_tx, AgentEvent::Error { message: e.to_string(), + kind: None, }, ) .await; @@ -1818,6 +1910,9 @@ async fn run_session(session: Session) { break $label; } let steer = texts.join("\n\n"); + // One posted prompt = one user message, whatever queued + // texts it merges — one turn, one provider ref (ticket 20). + let steer_user_message_id = new_user_message_id(); let adopted = match config { Some(next) => adopt_config(&server, &session_id, dir, &next) .await @@ -1828,6 +1923,7 @@ async fn run_session(session: Session) { Ok(()) => post_prompt( &server, &bus_tx, + &event_tx, &session_id, dir, &commands, @@ -1839,6 +1935,7 @@ async fn run_session(session: Session) { model: model.as_ref(), variant: variant.as_deref(), attachments: &attachments, + user_message_id: &steer_user_message_id, }, ) .await, @@ -1852,6 +1949,7 @@ async fn run_session(session: Session) { Err(e) => { let _ = send(&event_tx, AgentEvent::Error { message: e.to_string(), + kind: None, }).await; turn.error = Some(e.to_string()); turn.aborted_for_retry = true; @@ -2031,6 +2129,7 @@ async fn run_session(session: Session) { &event_tx, AgentEvent::Error { message: failure.message.clone(), + kind: None, }, ) .await; @@ -2074,6 +2173,9 @@ async fn run_session(session: Session) { assistant_message_id: Some(prev), next_assistant_message_id: Some(next), }).await; + // One posted prompt — one user message, one + // provider ref (ticket 20). + let steer_user_message_id = new_user_message_id(); let adopted = match &steer.config { Some(next) => adopt_config(&server, &session_id, dir, next) .await @@ -2084,6 +2186,7 @@ async fn run_session(session: Session) { Ok(()) => post_prompt( &server, &bus_tx, + &event_tx, &session_id, dir, &commands, @@ -2095,6 +2198,7 @@ async fn run_session(session: Session) { model: model.as_ref(), variant: variant.as_deref(), attachments: &steer.attachments, + user_message_id: &steer_user_message_id, }, ) .await, @@ -2108,6 +2212,7 @@ async fn run_session(session: Session) { &event_tx, AgentEvent::Error { message: message.clone(), + kind: None, }, ) .await; @@ -2161,7 +2266,7 @@ async fn run_session(session: Session) { "opencode made no progress for {}s after the prompt. {STALL_HINT}", stall.unwrap_or(DEFAULT_STALL_BOUND).as_secs() ); - let _ = send(&event_tx, AgentEvent::Error { message: message.clone() }).await; + let _ = send(&event_tx, AgentEvent::Error { message: message.clone(), kind: None }).await; let _ = server.abort_session(&session_id, dir).await; settle_children(&mut children, &event_tx, DoneStatus::Interrupted).await; let _ = send(&event_tx, AgentEvent::Done { @@ -2223,7 +2328,7 @@ async fn run_session(session: Session) { &server.stderr_tail, ); if turn.active { - let _ = send(&event_tx, AgentEvent::Error { message: message.clone() }).await; + let _ = send(&event_tx, AgentEvent::Error { message: message.clone(), kind: None }).await; } settle_children(&mut children, &event_tx, DoneStatus::Interrupted).await; let _ = send(&event_tx, AgentEvent::Done { @@ -2380,6 +2485,151 @@ async fn create_session( unreachable!("create_session retry loop returns from every path") } +/// Fork the source provider session for a native fork +/// (t3-orchestration-v2 ticket 20): OpenCode's fork is EXCLUSIVE at a +/// message id — 1.x `POST /session/{id}/fork {messageID}`, 2.x +/// `POST /api/session/{id}/fork {before}` (both shapes checked against +/// 1.18.31 and 2.0.11) — so keeping everything through the boundary turn +/// means cutting before the NEXT turn's user message; with no next turn +/// the whole session is copied (no boundary). `fork.last_turn` is the +/// boundary turn's own user message id (the provider turn ref recorded on +/// the boundary entry); the next one is resolved from the session's +/// message list. Any failure — the list, the boundary, the call — is the +/// caller's fallback (fresh session + the budgeted paste), never an error +/// the turn dies on. +async fn fork_session( + server: &Server, + dir: Option<&str>, + fork: &roboco_proto::ProviderFork, +) -> Result { + let messages = session_messages(server, dir, &fork.session_id).await?; + let boundary = fork_cut(&messages, &fork.last_turn)?; + let (path, body) = match server.protocol().await { + Protocol::V1 => ( + format!("/session/{}/fork", fork.session_id), + boundary + .as_ref() + .map(|id| json!({ "messageID": id })) + .unwrap_or_else(|| json!({})), + ), + Protocol::V2 => ( + format!("/api/session/{}/fork", fork.session_id), + boundary + .as_ref() + .map(|id| json!({ "before": id })) + .unwrap_or_else(|| json!({})), + ), + }; + let created = server.post_json(&path, dir, &body).await?; + created + .pointer("/data/id") + .or_else(|| created.get("id")) + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| { + HarnessError::Protocol("opencode session fork returned no id".into()) + }) +} + +/// One message-list entry reduced to what a fork cut needs: (id, role). +/// 1.x wraps the message in `{info: {id, role}, parts}`; 2.x items are +/// flat `{id, type, …}` — the role lives under either key. +fn message_id_and_role(item: &Value) -> Option<(String, String)> { + let info = item.get("info").unwrap_or(item); + let id = info.get("id").and_then(Value::as_str)?; + let role = info + .get("role") + .or_else(|| info.get("type")) + .and_then(Value::as_str)?; + Some((id.to_owned(), role.to_owned())) +} + +/// Whether a message starts a turn: a user prompt, or a 2.x synthetic +/// report the session answers as its next user input (both begin a new +/// execution; 1.x only ever has `user`/`assistant`). +fn starts_turn(role: &str) -> bool { + role == "user" || role == "synthetic" +} + +/// Resolve the exclusive fork cut from a session's (id, role) message list +/// in creation order: the id of the next turn-starting message after +/// `last_turn` (the boundary turn's own user message), `None` when nothing +/// follows (the whole session is copied). `Err` when the boundary message +/// is gone from the list — the session can no longer be cut there. +fn fork_cut( + messages: &[(String, String)], + last_turn: &str, +) -> Result, HarnessError> { + let pos = messages + .iter() + .position(|(id, _)| id == last_turn) + .ok_or_else(|| { + HarnessError::Protocol(format!( + "opencode session no longer holds message {last_turn}" + )) + })?; + Ok(messages[pos + 1..] + .iter() + .find(|(_, role)| starts_turn(role)) + .map(|(id, _)| id.clone())) +} + +/// The session's message list as (id, role) pairs in creation order — +/// 1.x `GET /session/{id}/message` (a raw array of `{info, parts}` +/// entries), 2.x `GET /api/session/{id}/message` (cursor-paginated +/// `{data, cursor}`; `order=asc` fixes oldest-first and the cursor alone +/// pages on — the spec forbids combining it with `order`). The cursor is +/// opaque, so it is percent-encoded into the query. +async fn session_messages( + server: &Server, + dir: Option<&str>, + session_id: &str, +) -> Result, HarnessError> { + /// A page cap so a server that keeps feeding cursors cannot hang the + /// fork; hitting it refuses the fork (the budgeted paste) rather than + /// risking a cut resolved against an incomplete list. + const MAX_MESSAGE_PAGES: usize = 1024; + + let protocol = server.protocol().await; + let mut messages = Vec::new(); + let mut cursor: Option = None; + for page in 0..MAX_MESSAGE_PAGES { + let path = match (protocol, cursor.as_deref()) { + (Protocol::V1, _) => format!("/session/{session_id}/message"), + (Protocol::V2, None) => format!("/api/session/{session_id}/message?order=asc"), + (Protocol::V2, Some(cursor)) => { + // `encode_directory` is a plain percent-encoder; it serves + // opaque query values as well as the header. + format!( + "/api/session/{session_id}/message?cursor={}", + encode_directory(cursor) + ) + } + }; + let response = server.get_json(&path, dir).await?; + let next = response + .pointer("/cursor/next") + .and_then(Value::as_str) + .filter(|cursor| !cursor.is_empty()) + .map(str::to_owned); + for item in unwrap_data(response).as_array().cloned().unwrap_or_default() { + if let Some(entry) = message_id_and_role(&item) { + messages.push(entry); + } + } + match (protocol, next) { + (Protocol::V1, _) | (Protocol::V2, None) => return Ok(messages), + (Protocol::V2, Some(_)) if page + 1 == MAX_MESSAGE_PAGES => { + return Err(HarnessError::Protocol( + "opencode message list never reached its end".into(), + )) + } + (Protocol::V2, Some(next)) => cursor = Some(next), + } + } + unreachable!("the loop returns or errors on its last page") +} + fn context_usage_event(info: &Value, context_windows: &HashMap) -> Option { let tokens = info.get("tokens")?; let counts: Vec = [ @@ -2486,12 +2736,15 @@ fn pick_variant( .map(str::to_owned) } -/// Build a `prompt_async` body: text part + attachment file parts. +/// Build a `prompt_async` body: text part + attachment file parts. The +/// client-chosen `messageID` rides the top level (1.x `PromptInput` takes +/// it; the server adopts it as the user message's id — ticket 20). fn prompt_body( prompt: &str, model: Option<(&str, &str)>, variant: Option<&str>, attachments: &[String], + message_id: &str, ) -> Value { let mut parts = vec![json!({ "type": "text", "text": prompt })]; for path in attachments { @@ -2507,6 +2760,10 @@ fn prompt_body( } let mut body = serde_json::Map::new(); body.insert("parts".into(), Value::Array(parts)); + body.insert( + "messageID".into(), + Value::String(message_id.to_owned()), + ); if let Some((provider, model)) = model { body.insert( "model".into(), @@ -2519,11 +2776,10 @@ fn prompt_body( Value::Object(body) } -/// 2.x prompt body: plain text plus `{uri, name}` file attachments. Model -/// and variant do NOT ride here — they were set on the session at run -/// start (`POST /api/session/{id}/model`). -fn prompt_body_v2(prompt: &str, attachments: &[String]) -> Value { - let files: Vec = attachments +/// 2.x file attachments: `{uri, name}` pairs shared by the prompt and +/// command bodies. +fn prompt_files_v2(attachments: &[String]) -> Vec { + attachments .iter() .map(|path| { json!({ @@ -2534,8 +2790,15 @@ fn prompt_body_v2(prompt: &str, attachments: &[String]) -> Value { .unwrap_or_default(), }) }) - .collect(); - json!({ "text": prompt, "files": files }) + .collect() +} + +/// 2.x prompt body: plain text plus file attachments, under the +/// client-chosen message `id` (the turn's provider ref — ticket 20; 2.x +/// requires the `msg_` prefix). Model and variant do NOT ride here — they +/// were set on the session at run start (`POST /api/session/{id}/model`). +fn prompt_body_v2(prompt: &str, attachments: &[String], message_id: &str) -> Value { + json!({ "id": message_id, "text": prompt, "files": prompt_files_v2(attachments) }) } fn mime_for(path: &str) -> &'static str { @@ -2556,10 +2819,14 @@ fn mime_for(path: &str) -> &'static str { /// What a posted turn carries besides its text (1.x folds model/variant /// into the prompt body; 2.x ignores them there — set on the session). +/// `user_message_id` is the client-chosen message id the turn travels +/// under (ticket 20): the turn's provider ref, and the anchor a later +/// fork cuts the provider session at. struct TurnSpec<'a> { model: Option<&'a (String, String)>, variant: Option<&'a str>, attachments: &'a [String], + user_message_id: &'a str, } /// Send a turn: a leading `/command` known to the agent routes through the @@ -2580,7 +2847,7 @@ fn command_body_v2( { let mut body = json!({"name": name, "text": text}); if !attachments.is_empty() { - body["files"] = prompt_body_v2(text, attachments)["files"].clone(); + body["files"] = Value::Array(prompt_files_v2(attachments)); } body } else { @@ -2591,6 +2858,7 @@ fn command_body_v2( async fn post_prompt( server: &Server, bus_tx: &mpsc::Sender, + event_tx: &mpsc::Sender>, session_id: &str, dir: Option<&str>, commands: &[SlashCommand], @@ -2604,6 +2872,7 @@ async fn post_prompt( model, variant, attachments, + user_message_id, } = spec; let protocol = server.protocol().await; if let Some((name, arguments)) = @@ -2615,15 +2884,21 @@ async fn post_prompt( .into(), )); } - // 1.x names the args `arguments`; 2.x `text`. - let (path, cmd_body) = match protocol { + // 1.x names the args `arguments` and takes the client message id + // (`messageID`, like a prompt — T3 sends it there too), so the + // command turn still gets its provider ref; 2.x `text`, and its + // command endpoint takes NO id (checked against the 2.0.11 + // schema), so that turn carries no ref. + let (path, cmd_body, id_carried) = match protocol { Protocol::V1 => ( format!("/session/{session_id}/command"), - json!({ "command": name, "arguments": arguments }), + json!({ "command": name, "arguments": arguments, "messageID": user_message_id }), + true, ), Protocol::V2 => ( format!("/api/session/{session_id}/command"), command_body_v2(server.version.get(), name, &arguments, attachments), + false, ), }; let server_base = server.base.clone(); @@ -2632,6 +2907,16 @@ async fn post_prompt( let path_owned = path.clone(); let protocol = server.protocol.clone(); let command_failure_tx = command_failure_tx.clone(); + // The command turn's provider ref (ticket 20), emitted before the + // endpoint's own request so it precedes every event the turn + // streams. 2.x commands carry no client id — no ref for them. + if id_carried + && !send(event_tx, turn_ref_event(session_id, user_message_id)).await + { + return Err(HarnessError::Protocol( + "opencode event consumer closed".into(), + )); + } tokio::spawn(async move { let server = Server { child: None, @@ -2675,13 +2960,23 @@ async fn post_prompt( model.map(|(provider, model)| (provider.as_str(), model.as_str())), variant, attachments, + user_message_id, ), ), Protocol::V2 => ( format!("/api/session/{session_id}/prompt"), - prompt_body_v2(prompt, attachments), + prompt_body_v2(prompt, attachments, user_message_id), ), }; + // The turn's provider ref (ticket 20): emitted before the prompt's own + // POST so it precedes every event the turn streams — the engine stamps + // it on the transcript entry the turn writes, and a fork of the chat + // cuts the provider session at it. + if !send(event_tx, turn_ref_event(session_id, user_message_id)).await { + return Err(HarnessError::Protocol( + "opencode event consumer closed".into(), + )); + } let server = Server { child: None, base: server.base.clone(), @@ -2890,7 +3185,7 @@ async fn handle_bus_event(ctx: BusCtx<'_>) -> BusOutcome { let msg = format!( "Giving up after {attempt} provider retries: {message}. {STALL_HINT}" ); - if !send(event_tx, AgentEvent::Error { message: msg }).await { + if !send(event_tx, AgentEvent::Error { message: msg, kind: None }).await { return BusOutcome::ConsumerGone; } let _ = server.abort_session(session_id, dir).await; @@ -2900,7 +3195,7 @@ async fn handle_bus_event(ctx: BusCtx<'_>) -> BusOutcome { "The provider is failing and opencode is retrying (attempt \ {attempt}): {message}" ); - if !send(event_tx, AgentEvent::Error { message: msg }).await { + if !send(event_tx, AgentEvent::Error { message: msg, kind: None }).await { return BusOutcome::ConsumerGone; } } @@ -2945,7 +3240,7 @@ async fn handle_bus_event(ctx: BusCtx<'_>) -> BusOutcome { if kind == "session.error" { turn.error = Some(message.clone()); } - if !duplicate && !send(event_tx, AgentEvent::Error { message }).await { + if !duplicate && !send(event_tx, AgentEvent::Error { message, kind: None }).await { return BusOutcome::ConsumerGone; } BusOutcome::Continue diff --git a/crates/harness/src/opencode/tests.rs b/crates/harness/src/opencode/tests.rs index 4025389658..f46a7f5408 100644 --- a/crates/harness/src/opencode/tests.rs +++ b/crates/harness/src/opencode/tests.rs @@ -474,7 +474,7 @@ async fn native_command_http_failures_settle_the_current_turn() { let mut surfaced = None; loop { match wire.events.recv().await.unwrap().unwrap() { - AgentEvent::Error { message } => surfaced = Some(message), + AgentEvent::Error { message, .. } => surfaced = Some(message), AgentEvent::Done { status, error, .. } => { return (status, error.or(surfaced)); } @@ -759,7 +759,7 @@ async fn v2_execution_failure_and_interrupt_settle_the_turn() { let mut error = None; loop { match wire.events.recv().await.unwrap().unwrap() { - AgentEvent::Error { message } => error = Some(message), + AgentEvent::Error { message, .. } => error = Some(message), AgentEvent::Done { status, error: e, .. } => return (status, e.or(error)), @@ -1057,16 +1057,19 @@ fn variants_only_ride_models_that_advertise_them() { } #[test] -fn prompt_body_carries_model_variant_and_attachments() { +fn prompt_body_carries_model_variant_attachments_and_message_id() { let body = prompt_body( "hello", Some(("anthropic", "claude-opus-5")), Some("high"), &["/tmp/shot.png".to_owned()], + "msg_unit", ); assert_eq!(body["model"]["providerID"], "anthropic"); assert_eq!(body["model"]["modelID"], "claude-opus-5"); assert_eq!(body["variant"], "high"); + // The client-chosen message id (ticket 20): the turn's provider ref. + assert_eq!(body["messageID"], "msg_unit"); assert_eq!(body["parts"][0]["type"], "text"); assert_eq!(body["parts"][0]["text"], "hello"); assert_eq!(body["parts"][1]["type"], "file"); @@ -1074,6 +1077,73 @@ fn prompt_body_carries_model_variant_and_attachments() { assert_eq!(body["parts"][1]["url"], "file:///tmp/shot.png"); } +#[test] +fn new_user_message_ids_fit_both_wires() { + // 2.x requires the `msg_` prefix on prompt ids and fork boundaries. + let id = new_user_message_id(); + assert!(id.starts_with("msg_"), "{id}"); + assert_ne!(id, new_user_message_id()); +} + +#[test] +fn message_id_and_role_reads_both_list_shapes() { + // 1.x: `{info: {id, role}, parts}`. 2.x: flat `{id, type}`. + let v1 = json!({ "info": { "id": "msg_1", "role": "user" }, "parts": [] }); + let v2 = json!({ "id": "msg_2", "type": "assistant", "time": { "created": 1 } }); + assert_eq!( + message_id_and_role(&v1), + Some(("msg_1".to_owned(), "user".to_owned())) + ); + assert_eq!( + message_id_and_role(&v2), + Some(("msg_2".to_owned(), "assistant".to_owned())) + ); + // A synthetic 2.x report is a turn start the fork must cut at too. + let synthetic = json!({ "id": "msg_3", "type": "synthetic" }); + assert_eq!( + message_id_and_role(&synthetic), + Some(("msg_3".to_owned(), "synthetic".to_owned())) + ); + assert_eq!(message_id_and_role(&json!({ "parts": [] })), None); +} + +#[test] +fn fork_cut_addresses_the_next_turns_user_message() { + // The boundary turn's own message starts the list; the cut is the NEXT + // user message (OpenCode's fork boundary is exclusive). + let messages: Vec<(String, String)> = [ + ("msg_first", "user"), + ("msg_first_reply", "assistant"), + ("msg_second", "user"), + ("msg_second_reply", "assistant"), + ] + .into_iter() + .map(|(id, role)| (id.to_owned(), role.to_owned())) + .collect(); + assert_eq!( + fork_cut(&messages, "msg_first").unwrap(), + Some("msg_second".to_owned()) + ); + // The boundary turn is the latest: nothing follows → copy the whole + // session (no cut). + assert_eq!(fork_cut(&messages, "msg_second").unwrap(), None); + // A synthetic report the session answers starts the next turn too. + let with_report: Vec<(String, String)> = [ + ("msg_first", "user"), + ("msg_report", "synthetic"), + ] + .into_iter() + .map(|(id, role)| (id.to_owned(), role.to_owned())) + .collect(); + assert_eq!( + fork_cut(&with_report, "msg_first").unwrap(), + Some("msg_report".to_owned()) + ); + // A boundary the session no longer holds refuses the fork (the caller + // falls back to the budgeted paste). + assert!(fork_cut(&messages, "msg_gone").is_err()); +} + fn feed_with_assistant(message: &str) -> SessionFeed { let mut feed = SessionFeed::default(); feed.assistant_messages.insert(message.into(), true); @@ -1648,8 +1718,9 @@ fn v2_model_list_folds_into_provider_catalog() { } #[test] -fn prompt_body_v2_carries_text_and_files_only() { - let body = prompt_body_v2("hello", &["/tmp/shot.png".to_owned()]); +fn prompt_body_v2_carries_text_files_and_the_message_id() { + let body = prompt_body_v2("hello", &["/tmp/shot.png".to_owned()], "msg_unit"); + assert_eq!(body["id"], "msg_unit"); assert_eq!(body["text"], "hello"); assert_eq!(body["files"][0]["uri"], "file:///tmp/shot.png"); assert_eq!(body["files"][0]["name"], "shot.png"); @@ -1890,14 +1961,18 @@ async fn v2_model_selection_and_prompt_use_the_documented_bodies() { .1, json!({"model":{"providerID":"opencode","id":"muse","variant":"low"}}) ); - assert_eq!( - posts - .iter() - .find(|(p, _)| p == "/api/session/fixture/prompt") - .unwrap() - .1, - json!({"text":"first","files":[]}) - ); + let prompt = posts + .iter() + .find(|(p, _)| p == "/api/session/fixture/prompt") + .unwrap() + .1 + .clone(); + assert_eq!(prompt["text"], "first"); + assert_eq!(prompt["files"], json!([])); + // The client-chosen message id (ticket 20) rides `id` — a dynamic + // `msg_`-prefixed uuid, so only its shape is asserted here. + let message_id = prompt["id"].as_str().unwrap_or_default(); + assert!(message_id.starts_with("msg_"), "{message_id}"); } #[tokio::test] @@ -2078,7 +2153,7 @@ fn v2_command_bodies_follow_server_version() { let attachments = vec!["/workspace/image.png".into()]; assert_eq!( command_body_v2(Some(&version), "test", "args", &attachments)["files"], - prompt_body_v2("args", &attachments)["files"] + prompt_body_v2("args", &attachments, "msg_unit")["files"] ); } } diff --git a/crates/harness/src/pi/mod.rs b/crates/harness/src/pi/mod.rs index e6ff760a69..f47e2a30f5 100644 --- a/crates/harness/src/pi/mod.rs +++ b/crates/harness/src/pi/mod.rs @@ -20,20 +20,85 @@ //! `get_state.sessionFile` (a bare id can hit pi's global session search, //! which prompts interactively on stdin — the protocol channel). The //! resume pointer rides `SessionStarted/Done.session_id`. -//! - **Prompting**: `prompt` while idle (response = preflight acceptance); -//! a mid-run steer goes through pi's `steer` queue (delivered after the -//! current assistant turn's tool calls); `abort` interrupts, waiting for -//! idle before responding. `follow_up` stays unused: a parked session's -//! next message is a fresh `prompt` in roboco's model. -//! - **Turn settlement**: `agent_settled` is the authoritative end — every -//! turn shape ends with a deterministic `Done` (the engine retires its +//! - **Prompting**: every prompt — turn starts and mid-run steers alike — +//! goes out as `prompt` with `streamingBehavior: "steer"`: pi starts a +//! run when idle and queues the prompt when an extension run is +//! streaming, atomically against settlement, so a prompt can never strand +//! against the end of a run. The acknowledgement's `disposition` reports +//! what became of it (`started` / `queued` / `handled`; pi < 0.99 reports +//! none and the driver infers from whether a run starts). The acceptance +//! is unbounded ON PURPOSE — there is deliberately no acceptance timeout +//! (ticket 02 removed the fixed one): a healthy preflight can take +//! arbitrarily long (extensions, the Roboco MCP bridge, compaction), so +//! a fixed budget killed slow-but-fine starts. The cost is that a WEDGED +//! pi holds its turn open until the user stops it — only Stop interrupts +//! a slow acceptance; nothing else ever times the run out. A +//! rejected prompt never ends the session — the message returns to the +//! driver queue and the rejection surfaces. Delivery tracking keeps one +//! FIFO record per original input, created when the input goes out (the +//! acknowledgement is never a consumption receipt, upstream PROTOCOL.md): +//! a consumed user `message_start` echo pops the oldest record and emits +//! its `Steered` boundary — whichever of the echo or the +//! started/handled acknowledgement processes first consumes the record, +//! so a burst is never short a boundary. A mid-run steer carries its +//! staged images inline, exactly like a turn-start prompt. Pi is a +//! STEP-BOUNDARY harness (the engine registry descriptor agrees): the +//! steering mode is set to `"all"` at session start unless the user +//! already configured one (agent-dir or project `.pi/settings.json`), +//! so a burst of steers is delivered together, in order, at the next +//! step boundary. `abort` interrupts, waiting for idle before +//! responding. `follow_up` stays unused: a parked session's next message +//! is a fresh prompt in roboco's model. +//! - **Turn settlement**: `agent_settled`, then a state probe: the turn +//! ends only once pi reports it is not streaming, not compacting and +//! holds no pending messages, and the settle generation is unchanged +//! (`agent_start`, `compaction_start`, every prompt or steer, and every +//! settle bump it). Busy and stale probes re-arm; a probe that keeps +//! failing ends the turn errored — never a silent kill. Every turn shape +//! still ends with a deterministic `Done` (the engine retires its //! quiesce watchdogs). Turn errors reject the prompt: a final assistant //! error (`stopReason: "error"`), exhausted `auto_retry_*`, or a failed -//! `compaction_end` without `willRetry` produce `Done { Errored }`. +//! `compaction_end` without `willRetry` produce `Done { Errored }` — and +//! a successful compaction retry clears the error state, so a run that +//! recovered through compaction ends completed. The turn is never over +//! while an input is still awaiting its acknowledgement: a quiescent +//! probe re-arms until the preflight lands. A probe that keeps failing +//! ends the turn errored — never a silent kill — and drains the queued +//! mailbox messages like a clean settle (the next one starts the next +//! turn), so the mailbox never wedges behind the dead turn. A message pi +//! keeps REJECTING is capped, not retried forever: past two deliveries +//! it is dropped from the driver (the engine's at-least-once ledger +//! still owns it) and the turn ends errored — never +//! a loop of Error + Completed for a message that never ran. A steer +//! stranded in pi's queue at settle is recovered by `clear_queue` and +//! re-queued INTACT (its attachments and run config ride along). Stop +//! sends `clear_queue` before `abort` — and only takes the "already +//! settled" shortcut when the run truly settled, never while work is +//! still in flight: steers pi queued but never consumed are cancelled +//! with the turn (the engine's at-least-once ledger already owns their +//! fate), never delivered into the aborted run. //! - **Stats**: `get_session_stats.contextUsage` feeds the context meter //! after each settle; `message_update.usage` rides per-turn `Usage` //! passthrough — first-party replacements for the pi-acp fork's //! usage_update/turn-error patches. +//! - **Native forks** (orchestration v2 ticket 19): the forked chat's +//! first run continues the SOURCE's real session instead of pasting +//! the copied transcript. A short-lived `pi --mode rpc --fork ` process in the run's cwd performs the fork — the CLI copies +//! the source session into a new file for the current project, and, +//! when the boundary is not the source's latest turn, the RPC `fork` +//! re-roots that copy BEFORE the next turn's user entry (pi's fork is +//! exclusive) — then the main spawn resumes the new file with +//! `--session` and the copied history rides the provider: nothing is +//! pasted. Transcript entries record a provider turn ref (the session +//! file plus the turn's user entry id, `get_entries` after each +//! settle): for an unsteered turn that is its FIRST user entry; a turn +//! a steer landed in names its LAST user entry instead, because the +//! fork boundary is "the user entry after the ref" and naming the last +//! consumed entry makes that the next turn's start exactly. A missing +//! ref, a stale boundary, or a refused/cancelled fork falls back to +//! the budgeted transcript in front of the prompt — never a failed +//! send. //! - **Extension UI**: blocking dialogs (`select`/`confirm`/`input`/`editor`) //! map to roboco's question flow and are answered with //! `extension_ui_response`; fire-and-forget methods are tolerated. @@ -60,7 +125,7 @@ mod mcp; mod normalize; mod wire; -use std::collections::VecDeque; +use std::collections::{HashMap, VecDeque}; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -74,8 +139,8 @@ use tokio::io::AsyncBufReadExt; use tokio::sync::{mpsc, oneshot}; use roboco_proto::{ - AgentEvent, DoneStatus, HarnessId, Model, ReasoningLevel, RunRequest, SteeringMode, - UserInputAnswer, UserInputQuestion, + AgentEvent, DoneStatus, HarnessId, Model, ProviderFork, ProviderTurnRef, ReasoningLevel, + RunRequest, SteeringMode, UserInputAnswer, UserInputQuestion, }; use crate::acp::child; @@ -122,6 +187,37 @@ const INSTALL_HINT: &str = "pi (searched PATH, the login shell's PATH, npm globa `npm install -g @earendil-works/pi-coding-agent` — Node.js >= 24; set \ PI_EXECUTABLE to override)"; +/// pi's settings directory: an explicit override (tests), else +/// `PI_CODING_AGENT_DIR`, else `~/.pi/agent`. +fn pi_agent_dir(explicit: Option<&Path>) -> PathBuf { + explicit.map(Path::to_path_buf).unwrap_or_else(|| { + crate::model_context::root( + "PI_CODING_AGENT_DIR", + crate::executable::home_or_current_dir().join(".pi").join("agent"), + ) + }) +} + +/// Whether a steering mode is already set in the settings pi loads for +/// `cwd` (the agent directory's global settings, then the project's): +/// `set_steering_mode "all"` at session start must never overwrite an +/// explicit choice — the user's, or `/steering` picked inside a chat +/// (pi persists that in the same files). A missing, unreadable, or +/// malformed file simply means "not configured". +fn steering_mode_configured(agent_dir: Option<&Path>, cwd: &str) -> bool { + [ + pi_agent_dir(agent_dir).join("settings.json"), + Path::new(cwd).join(".pi").join("settings.json"), + ] + .into_iter() + .any(|settings| { + std::fs::read(&settings) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .is_some_and(|settings| settings.get("steeringMode").is_some()) + }) +} + /// pi accepts both spellings for its npm-global install locations. fn pi_install_paths() -> Vec { let mut dirs = Vec::new(); @@ -280,6 +376,11 @@ async fn apply_model(client: &PiClient, model: &str) -> Result<(), HarnessError> /// fixture RPC process with [`PiHarness::with_executable`]. pub struct PiHarness { executable: Option, + /// The pi agent directory the steering-mode default reads settings + /// from: `PI_CODING_AGENT_DIR` / `~/.pi/agent` unless overridden. Only + /// the steering-mode check consults it — model contexts keep hashing + /// `auth.json` alone (see [`PiHarness::model_context`]). + agent_dir: Option, /// Grace between `abort` and SIGTERM. interrupt_grace: Duration, /// Grace between SIGTERM and SIGKILL. @@ -299,6 +400,7 @@ impl Default for PiHarness { fn default() -> Self { Self { executable: None, + agent_dir: None, interrupt_grace: Duration::from_secs(2), kill_grace: Duration::from_secs(3), startup_timeout: Duration::from_secs(120), @@ -318,6 +420,14 @@ impl PiHarness { self } + /// Pin the pi agent directory the steering-mode default reads + /// `settings.json` from (tests: a temp dir makes the "user configured + /// a mode" check hermetic against the real `~/.pi/agent`). + pub fn with_agent_dir(mut self, path: impl Into) -> Self { + self.agent_dir = Some(path.into()); + self + } + /// Tune the interrupt→SIGTERM→SIGKILL escalation timing. pub fn with_graces(mut self, interrupt_grace: Duration, kill_grace: Duration) -> Self { self.interrupt_grace = interrupt_grace; @@ -404,7 +514,7 @@ impl PiHarness { /// command construction below). async fn run_with_mode( &self, - request: RunRequest, + mut request: RunRequest, controls: RunControls, title_only: bool, ) -> Result>, HarnessError> { @@ -415,11 +525,79 @@ impl PiHarness { // directory?" prompt — on stdin, the protocol channel. A pointer // that is not an existing file (a legacy ACP-era id, a moved // session) degrades to a fresh session with a visible notice. - let resume_path = request + let mut resume_path = request .resume .as_deref() .filter(|resume| Path::new(resume).is_file()) .map(str::to_owned); + // A native fork (t3-orchestration-v2 ticket 19): the forked chat's + // first run continues the SOURCE's real pi session instead of + // pasting the copied transcript into a fresh one. A short-lived + // `pi --mode rpc --fork ` process performs the fork + // (see [`fork_source_session`]) and the main spawn below resumes + // the forked session file with `--session`, so the history rides + // the provider and nothing is pasted. Any failure — a missing + // source file, a stale boundary ref, a refused or cancelled fork, + // a pi that answers with anything but a distinct new file — falls + // back to a fresh session with the engine's budgeted transcript + // in front of the prompt: the send never fails. The fork owns + // session continuity, so a plain resume never competes with it + // (codex parity: `thread/fork` instead of `thread/resume`). + let mut fork_notice: Option = None; + if !title_only + && let Some(fork) = request.provider_fork.clone() + { + request.resume = None; + resume_path = None; + let launch = if !Path::new(&fork.session_id).is_file() { + ForkLaunch::Failed("the source session file is gone".into()) + } else { + tokio::select! { + biased; + // Stop before the fork could start ends the run + // interrupted, exactly like a Stop during startup — a + // fork is not a fallback candidate after that. + _ = controls.interrupt.cancelled() => ForkLaunch::Interrupted, + launched = fork_source_session( + &exe, + &fork, + &request.cwd, + self.startup_timeout, + self.kill_grace, + ) => match launched { + Ok(file) => ForkLaunch::Session(file), + Err(reason) => ForkLaunch::Failed(reason), + }, + } + }; + match launch { + ForkLaunch::Session(file) => resume_path = Some(file), + ForkLaunch::Failed(reason) => { + // The budgeted transcript rides the prompt in front of + // the user message (codex parity: the refused-fork + // fallback), and a notice says what happened. + let history = fork.fallback_history.as_deref().unwrap_or(""); + request.prompt = crate::prompt_with_history(history, &request.prompt); + fork_notice = Some(format!( + "pi could not fork the source session ({reason}); starting a new \ + session with the chat transcript, trimmed to fit" + )); + } + ForkLaunch::Interrupted => { + let (event_tx, event_rx) = + mpsc::channel::>(256); + let _ = event_tx + .send(Ok(AgentEvent::Done { + status: DoneStatus::Interrupted, + result: None, + error: None, + session_id: None, + })) + .await; + return Ok(event_stream(event_rx)); + } + } + } let mut cmd = Command::new(&exe); cmd.arg("--mode").arg("rpc"); @@ -505,6 +683,12 @@ impl PiHarness { let (client, incoming) = PiClient::new(stdin, stdout); let (event_tx, event_rx) = mpsc::channel::>(256); + // Steering mode "all" at session start unless the user configured + // one: pi defaults to one-at-a-time, and a burst of roboco steers + // belongs together at the next step boundary, in order. Title runs + // never steer, so the one-shot sandbox never touches the setting. + let steering_mode_all = + !title_only && !steering_mode_configured(self.agent_dir.as_deref(), &request.cwd); tokio::spawn(run_session(Session { child, client, @@ -514,19 +698,29 @@ impl PiHarness { request, resume_path, mcp_scratch, + steering_mode_all, interrupt_grace: self.interrupt_grace, kill_grace: self.kill_grace, startup_timeout: self.startup_timeout, stderr_tail, + fork_notice, })); - Ok(futures::stream::unfold(event_rx, |mut rx| async move { - rx.recv().await.map(|ev| (ev, rx)) - }) - .boxed()) + Ok(event_stream(event_rx)) } } +/// The consumer's event stream: one channel receiver unfolded into a +/// boxed stream (the run task owns the sender side). +fn event_stream( + event_rx: mpsc::Receiver>, +) -> BoxStream<'static, Result> { + futures::stream::unfold(event_rx, |mut rx| async move { + rx.recv().await.map(|ev| (ev, rx)) + }) + .boxed() +} + #[async_trait] impl Harness for PiHarness { fn id(&self) -> HarnessId { @@ -538,10 +732,13 @@ impl Harness for PiHarness { fn supports_steering(&self) -> bool { true } - /// pi's `steer` queue delivers after the current assistant turn's tool - /// calls — turn boundaries, matching the registry descriptor. + /// pi's steer queue (reached through `prompt` + steer behavior, with the + /// mode set to "all") delivers after the current assistant turn's tool + /// calls — the next STEP boundary inside the live turn — matching the + /// registry descriptor exactly (both must flip together; orchestration + /// v2 ticket 03). fn steering_mode(&self) -> SteeringMode { - SteeringMode::TurnBoundary + SteeringMode::StepBoundary } fn reasoning_levels(&self) -> &[ReasoningLevel] { &REASONING_LEVELS @@ -623,6 +820,10 @@ impl Harness for PiHarness { request.mcp = None; request.model_options.clear(); request.auto_approve = false; + // A native-fork directive must never reach the title sandbox: the + // one-shot never continues a session, so a fork would only spawn a + // doomed subprocess (its history belongs to a chat, not a title). + request.provider_fork = None; self.run_with_mode(request, controls, true).await } } @@ -643,11 +844,19 @@ struct Session { /// The scratch dir holding the MCP bridge file for exactly this run /// (dropped after the pi child shuts down, when the file is closed). mcp_scratch: Option, + /// Send `set_steering_mode "all"` once pi answers `get_state` — armed + /// in `run_with_mode` only when no mode is configured (see there). + steering_mode_all: bool, interrupt_grace: Duration, kill_grace: Duration, startup_timeout: Duration, /// Rolling stderr tail for the crash message on an unexpected exit. stderr_tail: crate::StderrTail, + /// A refused native fork's notice (ticket 19): the run starts fresh + /// with the budgeted transcript pasted in front of the prompt, and + /// this surfaces what happened to the fold (parity with the + /// lost-resume notice). + fork_notice: Option, } fn new_message_id() -> String { @@ -668,11 +877,31 @@ type RequestInputFn = Box< dyn Fn(Vec) -> oneshot::Receiver> + Send + Sync, >; -/// Bound on the prompt/steer acceptance responses (pi answers at preflight — -/// fast); a hang past it is a wedged process, not a slow one. -const ACCEPT_TIMEOUT: Duration = Duration::from_secs(30); -/// Bound on the post-settle `clear_queue` + `get_session_stats` round trips. +/// Bound on the post-settle `get_session_stats` round trip (best-effort: +/// a failure skips the context meter, never the turn). const SETTLE_TIMEOUT: Duration = Duration::from_secs(15); +/// Bound on one settle-probe round (`get_state`, `clear_queue`): pi answers +/// in milliseconds; a hang past it is one failed attempt. +const PROBE_TIMEOUT: Duration = Duration::from_secs(5); +/// How many times one mailbox message may be delivered before repeated +/// rejection hands it back to the engine as a failed send (B1): the +/// original delivery plus ONE re-delivery. Past it, the turn ends errored +/// — never a loop of Error + Completed for a message that never ran. +const MESSAGE_DELIVERY_ATTEMPTS: u32 = 2; +/// Bound on the settle capture's `get_entries` round trip (B7): the +/// capture must never wedge the run loop for its full 2×15s budget — +/// an interrupt landing mid-capture cuts even this short, and a lost +/// capture costs the provider turn ref (a later fork falls back to the +/// budgeted paste), never the turn. +const REF_CAPTURE_BUDGET: Duration = Duration::from_secs(3); +/// Settle-probe attempts before the turn ends errored: a failed probe is +/// retried, and after this many the turn fails visibly instead of the +/// process being killed silently. +const PROBE_ATTEMPTS: u32 = 3; +/// Delay before a re-armed probe (a busy report, or the inference after an +/// acknowledgement that carries no disposition): the probe must never +/// hot-loop while pi streams. +const PROBE_REARM_DELAY: Duration = Duration::from_millis(500); /// Drain window for frames already written when the child exits: a crash /// right after the final response must still surface that response. const EXIT_DRAIN: Duration = Duration::from_millis(200); @@ -681,49 +910,478 @@ const EXIT_DRAIN: Duration = Duration::from_millis(200); /// see EOF first. const EOF_DRAIN: Duration = Duration::from_millis(50); -/// A steer request in flight: (text, already-retried-as-prompt, response). -type SteerCall = BoxFuture<'static, (String, bool, Result)>; +/// A mid-run steer in flight: the mailbox message and its `prompt` +/// acknowledgement. +type SteerCall = BoxFuture<'static, (u64, SteerMessage, Result)>; + +/// One armed prompt: its acceptance future and the mailbox message it +/// delivers (None = the run's own first prompt). The pair always travels +/// together — the message is owed a `Steered` boundary once pi accepts the +/// prompt, and returns to the queue if pi rejects it. `delivery` names the +/// [`DeliveryRecord`] the arming created, so the acknowledgement (or a +/// rejection) can retire it. +struct Armed { + acceptance: BoxFuture<'static, Result>, + message: Option, + delivery: Option, +} + +/// One FIFO delivery record per original input sent to pi (upstream +/// PROTOCOL.md: neither the prompt ACK nor `queue_update` is a consumption +/// receipt). The record is created when the input GOES OUT, so a user +/// message echo that races its acknowledgement still consumes it — the +/// driver's select may process the echo before the ready acceptance +/// future. Whoever consumes the record first — the echo, or the +/// Started/Handled acknowledgement that confirms delivery at acceptance — +/// emits the one `Steered` boundary the input owes; a record pi never +/// consumes (a steer stranded in its queue at settle) is retired by +/// clear_queue and re-delivered as the next turn's prompt. +struct DeliveryRecord { + id: u64, + text: String, + /// The original mailbox message for an input accepted into pi's steer + /// queue: a strand recovered by the settle probe's `clear_queue` + /// re-queues it INTACT — attachments and run config included — instead + /// of rebuilding it as bare text. + message: Option, +} -/// The post-`agent_settled` sequence: recover any steers stranded in pi's -/// queue (the settle race), then fetch the session stats for the context -/// meter. Resolves with (recovered steer texts, stats data). -type SettleOutcome = (Vec, Option); +/// Count one more rejection of `msg` (keyed by its engine message id, or +/// its text when it has none) and report whether the re-delivery cap is +/// reached (B1). A capped message's count is forgotten: the engine owns +/// its fate from here. +fn note_rejection(rejections: &mut HashMap, msg: &SteerMessage) -> Option { + let key = msg.message_id.clone().unwrap_or_else(|| msg.prompt.clone()); + let attempts = rejections.entry(key.clone()).or_insert(0); + *attempts += 1; + let attempts = *attempts; + if attempts >= MESSAGE_DELIVERY_ATTEMPTS { + rejections.remove(&key); + Some(attempts) + } else { + None + } +} -fn settle_future(client: &PiClient, pending_steers: usize) -> BoxFuture<'static, SettleOutcome> { +/// Retire `id`'s delivery record; `true` when it was still pending (the +/// boundary it owes has not been emitted at the echo yet). +fn retire_delivery(records: &mut VecDeque, id: u64) -> bool { + let before = records.len(); + records.retain(|record| record.id != id); + before != records.len() +} + +/// pi's answer to a `prompt`: what became of it. Pi ≥ 0.99 reports +/// `disposition` on the acknowledgement; older builds report nothing and +/// the driver infers (see [`Disposition::Unknown`]). +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +enum Disposition { + /// A run began with the prompt. + Started, + /// Accepted into pi's steer queue: delivered (consumed) when its user + /// `message_start` echo arrives. + Queued, + /// pi handled the prompt without starting a run (an extension + /// command): the turn ends once pi reports it is not streaming. + Handled, + /// pi < 0.99 reported nothing: infer from whether a run starts — the + /// settle probe ends the turn when nothing does. + Unknown, +} + +fn disposition_of(data: &Value) -> Disposition { + match data.get("disposition").and_then(Value::as_str) { + Some("started") => Disposition::Started, + Some("queued") => Disposition::Queued, + Some("handled") => Disposition::Handled, + _ => Disposition::Unknown, + } +} + +/// One settle-probe round: read pi's state and decide whether the turn may +/// end. Pi is at rest only when it is neither streaming nor compacting and +/// holds no pending messages; messages left in pi's queue while it is idle +/// (steers stranded by the settle race) are cleared and handed back — they +/// re-deliver as the next turn's prompts. +enum ProbeRound { + /// pi is streaming or compacting: the turn cannot end yet. + Busy, + /// pi is idle with messages still queued: they were cleared; the texts + /// return for re-delivery. + Recovered(Vec), + /// pi is at rest: the turn may end (stats for the context meter are + /// best-effort). + Quiescent(Option), + /// The probe itself failed (the request was rejected or timed out). + Failed(String), +} + +/// The texts pi's `clear_queue` handed back (its steering + followUp +/// queues). +fn queue_texts(data: &Value) -> Vec { + ["steering", "followUp"] + .iter() + .flat_map(|key| { + data.get(*key) + .and_then(Value::as_array) + .cloned() + .unwrap_or_default() + }) + .filter_map(|value| value.as_str().map(str::to_owned)) + .collect() +} + +/// One probe round. Bounded: a request pi never answers counts as one +/// failed attempt. +fn probe_round(client: &PiClient) -> BoxFuture<'static, ProbeRound> { let client = client.clone(); Box::pin(async move { - // A steer accepted while the run was active can land in pi's queue - // during the settle window (the queue check happened first): the - // queue would then sit undelivered while the session parks. - // `clear_queue` hands the texts back; they redeliver as prompts. - let recovered = if pending_steers > 0 { - match client.request("clear_queue", json!({})).await { - Ok(data) => ["steering", "followUp"] - .iter() - .flat_map(|key| { - data.get(*key) - .and_then(Value::as_array) - .cloned() - .unwrap_or_default() - }) - .filter_map(|value| value.as_str().map(str::to_owned)) - .collect(), - Err(e) => { - tracing::debug!( - target: "roboco_harness::pi", - "clear_queue failed (stranded steers ride the next prompt): {e}" - ); - Vec::new() - } + let state = match tokio::time::timeout( + PROBE_TIMEOUT, + client.request("get_state", json!({})), + ) + .await + { + Ok(Ok(state)) => state, + Ok(Err(e)) => return ProbeRound::Failed(e.to_string()), + Err(_) => { + return ProbeRound::Failed( + "get_state did not answer within the probe budget".into(), + ) } - } else { - Vec::new() }; - let stats = client.request("get_session_stats", json!({})).await.ok(); - (recovered, stats) + let busy = |key: &str| state.get(key).and_then(Value::as_bool).unwrap_or(false); + if busy("isStreaming") || busy("isCompacting") { + return ProbeRound::Busy; + } + let pending = state + .get("pendingMessageCount") + .and_then(Value::as_u64) + .unwrap_or(0); + if pending > 0 { + return match tokio::time::timeout( + PROBE_TIMEOUT, + client.request("clear_queue", json!({})), + ) + .await + { + Ok(Ok(data)) => ProbeRound::Recovered(queue_texts(&data)), + Ok(Err(e)) => ProbeRound::Failed(e.to_string()), + Err(_) => ProbeRound::Failed( + "clear_queue did not answer within the probe budget".into(), + ), + }; + } + let stats = match tokio::time::timeout( + SETTLE_TIMEOUT, + client.request("get_session_stats", json!({})), + ) + .await + { + Ok(Ok(stats)) => Some(stats), + // Best-effort: a failed read skips the context meter, never + // the turn. + _ => None, + }; + ProbeRound::Quiescent(stats) }) } +/// Arm one more probe round, optionally after the re-arm delay: a busy +/// report must not hot-loop the probe while pi streams. +fn rearm_probe(client: &PiClient, delay: bool) -> BoxFuture<'static, ProbeRound> { + let client = client.clone(); + Box::pin(async move { + if delay { + tokio::time::sleep(PROBE_REARM_DELAY).await; + } + probe_round(&client).await + }) +} + +/// One `get_entries` listing parsed for the session-tree walks (ticket +/// 19): the entry map (id → parent + whether the entry is a user message) +/// and the current leaf id. Entries without an id are skipped — the walks +/// simply cannot pass through them; a listing without the entries array +/// fails outright. Only `type: "message"` entries carry a role; every +/// other entry type (compaction, model changes, extension entries) walks +/// as a pass-through node. +struct SessionEntries { + by_id: HashMap, bool)>, + leaf: Option, +} + +fn parse_entries(data: &Value) -> Option { + let entries = data.get("entries")?.as_array()?; + let mut by_id = HashMap::with_capacity(entries.len()); + for entry in entries { + let Some(id) = entry.get("id").and_then(Value::as_str) else { + continue; + }; + let parent = entry + .get("parentId") + .and_then(Value::as_str) + .map(str::to_owned); + let is_user = entry.pointer("/message/role").and_then(Value::as_str) == Some("user"); + by_id.insert(id.to_owned(), (parent, is_user)); + } + let leaf = data.get("leafId").and_then(Value::as_str).map(str::to_owned); + Some(SessionEntries { by_id, leaf }) +} + +/// The newest user entry on the active branch: walking the branch from +/// the leaf, the FIRST user entry encountered is the LAST user entry the +/// settling turn consumed — the turn's steer when one landed mid-run, +/// else its prompt (for an unsteered turn that is also the turn's FIRST +/// user entry, the ticket's spelling). That is the turn's provider turn +/// ref, because pi's exclusive fork boundary is "the user entry after the +/// ref" and naming the last consumed entry makes that the next turn's +/// start exactly — see [`fork_boundary`]. A walk that leaves the listing +/// (a since-bounded stretch with no user entry of its own — an +/// extension-initiated run) or reaches the root names nothing. +fn newest_user_entry(entries: &SessionEntries) -> Option { + // The first entry of the branch walk (newest first) is exactly the + // newest user entry; the shared walk carries the cycle guard. + branch_user_entries(entries).into_iter().next() +} + +/// The user entry ids on the active branch, NEWEST first. The fork +/// boundary search walks the whole branch: the fork subprocess reads a +/// full `get_entries` listing, so the walk always reaches the root (and +/// the same cycle guard bounds a corrupt tree). A walk that leaves the +/// listing stops — the entries after a `since` cursor bound it to one +/// turn's stretch. +fn branch_user_entries(entries: &SessionEntries) -> Vec { + let mut users = Vec::new(); + let mut cursor = entries.leaf.clone(); + let mut steps = entries.by_id.len().saturating_add(1); + while let Some(current) = cursor { + let Some((parent, is_user)) = entries.by_id.get(¤t).cloned() else { + break; + }; + if is_user { + users.push(current); + } + if steps == 0 { + break; + } + steps -= 1; + cursor = parent; + } + users +} + +/// The resolved fork boundary for a native fork at `last_turn` (the +/// provider turn ref recorded on the boundary entry). pi's RPC `fork` +/// re-roots the session BEFORE the named user message, so the boundary +/// names the user entry of the turn AFTER the boundary turn — keeping +/// everything through it, exactly the prefix the fork copied. +#[derive(Debug, PartialEq, Eq)] +enum ForkBoundary { + /// Re-root the session before this user entry (one turn newer than + /// the boundary). + RootAt(String), + /// The boundary IS the branch's newest turn: the `--fork` copy + /// already ends there — fork the whole session, no RPC. + NewestTurn, + /// The ref names no entry on the active branch (a stale ref, or one + /// from a re-rooted/compacted-away branch) — the fork falls back to + /// the budgeted paste rather than silently keeping turns the + /// boundary excluded. + OffBranch, +} + +fn fork_boundary(users: &[String], last_turn: &str) -> ForkBoundary { + let Some(index) = users.iter().position(|id| id == last_turn) else { + return ForkBoundary::OffBranch; + }; + // `users` is newest-first: index 0 is the branch's newest user entry, + // and one step towards the head is one turn newer. + match index.checked_sub(1).and_then(|newer| users.get(newer)) { + Some(entry_id) => ForkBoundary::RootAt(entry_id.clone()), + None => ForkBoundary::NewestTurn, + } +} + +/// One bounded `get_entries` round: a request pi never answers counts as a +/// failed capture (the ref is skipped, never the turn). +async fn read_entries(client: &PiClient, params: Value) -> Result { + match tokio::time::timeout(SETTLE_TIMEOUT, client.request("get_entries", params)).await { + Ok(Ok(data)) => Ok(data), + Ok(Err(e)) => Err(e.to_string()), + Err(_) => Err("get_entries did not answer within the settle budget".into()), + } +} + +/// One settle's provider turn ref capture (ticket 19): read the session +/// tree and name the LAST user entry the turn consumed — the session file +/// plus that entry id (see [`newest_user_entry`] for why the last, not +/// the first). The previous capture's leaf rides as `since`, bounding the +/// listing to the turn's entries and the walk to the turn's stretch; a +/// cursor pi no longer knows (a re-rooted branch) or a slow read falls +/// back to one full listing — the walk's answer is the same either way, +/// so a failed bounded read costs only the round trip. Best-effort: any +/// failed read costs the ref, never the turn. Advances the cursor on +/// every successful listing, so the next settle bounds itself to the next +/// turn. +async fn capture_turn_ref( + client: &PiClient, + session_pointer: &str, + cursor: &mut Option, +) -> Option { + let params = match cursor.as_deref() { + Some(since) => json!({ "since": since }), + None => json!({}), + }; + let data = match read_entries(client, params).await { + Ok(data) => data, + Err(_) => read_entries(client, json!({})).await.ok()?, + }; + let listing = parse_entries(&data)?; + let reference = + newest_user_entry(&listing).map(|turn_id| ProviderTurnRef { + harness: HarnessId::Pi, + session_id: session_pointer.to_owned(), + turn_id, + }); + *cursor = listing.leaf.clone(); + reference +} + +/// The outcome of the fork-launch phase in [`PiHarness::run_with_mode`]: +/// the forked session file the main spawn resumes, the reason a fork +/// could not be performed (the budgeted-paste fallback), or a Stop that +/// landed before the fork could start. +enum ForkLaunch { + Session(String), + Failed(String), + Interrupted, +} + +/// Perform one native fork of a source pi session (ticket 19) in a +/// short-lived `pi --mode rpc --fork ` process running in the +/// run's cwd: `--fork` files the new session under the CURRENT project — +/// the same place the main spawn below resumes from (an RPC +/// clone/switch_session alone would keep the source's cwd), and the +/// process never runs a prompt, so it needs no MCP bridge and no model +/// selection. The sequence is pi's own fork surface: the CLI copies the +/// source session into a new file; `get_entries` walks the copy's active +/// branch; when the boundary is not the source's latest turn, the RPC +/// `fork` re-roots the copy BEFORE the next turn's user entry (pi's fork +/// is exclusive); `get_state` reports the new session file. Returns that +/// file — never the source itself (a pi answering with the source would +/// have the main run CONTINUE the source session, mutating it) — or the +/// reason the fork failed (the caller falls back to the budgeted paste). +async fn fork_source_session( + exe: &Path, + fork: &ProviderFork, + cwd: &str, + startup_timeout: Duration, + kill_grace: Duration, +) -> Result { + let source = Path::new(&fork.session_id); + let mut cmd = Command::new(exe); + cmd.arg("--mode").arg("rpc").arg("--fork").arg(source); + if !cwd.is_empty() { + cmd.current_dir(cwd); + } + child::configure(&mut cmd); + crate::compose_child_path(&mut cmd, exe); + cmd.stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .kill_on_drop(true); + let child = cmd + .spawn() + .map_err(|e| format!("could not spawn the pi fork process: {e}"))?; + let mut child = child::Child::new(child); + let stdin = child + .stdin + .take() + .ok_or_else(|| "the pi fork process has no stdin".to_owned())?; + let stdout = child + .stdout + .take() + .ok_or_else(|| "the pi fork process has no stdout".to_owned())?; + let (client, mut incoming) = PiClient::new(stdin, stdout); + // Events (notify noise) are drained so a chatty extension can never + // stall the reader. + tokio::spawn(async move { + while incoming.recv().await.is_some() {} + }); + let work = async { + // The forked copy's session file: must exist and be a DIFFERENT + // file from the source, or the main spawn would continue the + // source session itself. + let state = client + .request("get_state", json!({})) + .await + .map_err(|e| e.to_string())?; + let mut file = state + .pointer("/sessionFile") + .and_then(Value::as_str) + .filter(|file| !file.is_empty()) + .map(str::to_owned) + .ok_or_else(|| "pi reported no session file for the fork".to_owned())?; + if Path::new(&file) == source { + return Err("pi forked onto the source session file".to_owned()); + } + // The boundary: the copy's active branch tells whether the fork is + // already at the right place (the source's latest turn) or needs + // the exclusive re-root at the turn after the boundary. + let data = client + .request("get_entries", json!({})) + .await + .map_err(|e| e.to_string())?; + let listing = parse_entries(&data) + .ok_or_else(|| "pi returned an unreadable session tree".to_owned())?; + let users = branch_user_entries(&listing); + let boundary = match fork_boundary(&users, &fork.last_turn) { + ForkBoundary::OffBranch => { + return Err(format!( + "the boundary entry {} is no longer on the session branch", + fork.last_turn + )) + } + ForkBoundary::NewestTurn => None, + ForkBoundary::RootAt(entry_id) => Some(entry_id), + }; + if let Some(entry_id) = boundary { + let data = client + .request("fork", json!({ "entryId": entry_id })) + .await + .map_err(|e| e.to_string())?; + if data + .get("cancelled") + .and_then(Value::as_bool) + .unwrap_or(false) + { + return Err("a pi extension cancelled the session fork".to_owned()); + } + // The re-rooted session file, re-read from `get_state` (the + // fork happens inside the copy, so this is the file the main + // spawn resumes): a failed re-read keeps the copy's file — the + // fork itself already succeeded. + if let Ok(state) = client.request("get_state", json!({})).await + && let Some(after) = state + .pointer("/sessionFile") + .and_then(Value::as_str) + .filter(|file| !file.is_empty()) + && Path::new(after) != source + { + file = after.to_owned(); + } + } + Ok(file) + }; + let result = tokio::time::timeout(startup_timeout, work) + .await + .map_err(|_| "the pi fork process did not answer in time".to_owned()) + .and_then(|result| result); + child.shutdown(kill_grace).await; + result +} + /// The per-run event loop: one task multiplexing pi events, the prompt /// acceptance, the settle sequence, the steering mailbox, the interrupt /// token, and consumer liveness. @@ -739,10 +1397,12 @@ async fn run_session(session: Session) { // Held for the whole run: dropping it removes the bridge file, so it // must outlive the pi child (shut down before this function ends). mcp_scratch: _mcp_scratch, + steering_mode_all, interrupt_grace, kill_grace, startup_timeout, stderr_tail, + fork_notice, } = session; let RunControls { execution_lease: _execution_lease, @@ -756,6 +1416,21 @@ async fn run_session(session: Session) { // ---- startup: get_state (interruptible, bounded) ---------------------- let setup = async { let state = client.request("get_state", json!({})).await?; + // The steering mode rides the same startup: pi defaults to + // one-at-a-time and roboco's steers belong together. Best-effort — + // a pi old enough to lack the command logs and runs on its own + // default (the mode is a delivery-batching preference, never a + // run-shape contract). + if steering_mode_all + && let Err(e) = client + .request("set_steering_mode", json!({ "mode": "all" })) + .await + { + tracing::debug!( + target: "roboco_harness::pi", + "set_steering_mode all rejected (pi default runs): {e}" + ); + } // Apply the run's model and thinking level (pi clamps to the model's // own ladder; a rejected switch logs and the agent default runs, // pi-acp parity). Best-effort: an unavailable model or level is not @@ -871,6 +1546,21 @@ async fn run_session(session: Session) { without the previous context", request.resume.as_deref().unwrap_or_default() ), + kind: None, + }, + ) + .await; + } + if let Some(notice) = &fork_notice { + // A refused native fork (ticket 19): the fresh session carries the + // budgeted transcript in front of the prompt (pasted in + // run_with_mode), so the notice says the transcript rides along + // (trimmed), not lost — parity with the lost-resume notice above. + let _ = send( + &event_tx, + AgentEvent::Error { + message: notice.clone(), + kind: None, }, ) .await; @@ -880,17 +1570,45 @@ async fn run_session(session: Session) { // Agent activity since the last settle (extension-initiated runs deliver // output with no prompt of ours; their settle still emits a Done). let mut saw_activity = false; - // Steers accepted into pi's queue (or retried as steer-behavior prompts) - // whose delivery pi confirms with a user message. - let mut pending_steer_deliveries: usize = 0; + // FIFO delivery records for the mailbox inputs sent to pi — the + // consumption ledger behind the `Steered` boundaries. One record per + // original input, created when it goes out (not when pi acknowledges + // it), so an echo racing its acknowledgement still consumes it. + let mut sent_deliveries: VecDeque = VecDeque::new(); + // The delivery-record id sequence. + let mut delivery_seq: u64 = 0; let mut steer_calls: FuturesUnordered = FuturesUnordered::new(); - // Steers the driver itself owns (stranded in pi's queue at settle, or - // queued while the settle sequence runs) — delivered as fresh prompts - // once the current turn's Done is out, carrying their attachments and - // run config. + // Messages the driver itself owns — rejected by pi, stranded in pi's + // queue at settle, or queued while the settle probe winds down — + // delivered as fresh prompts once the current turn's Done is out, + // carrying their attachments and run config. let mut driver_steers: VecDeque = VecDeque::new(); - // The post-settle sequence (clear_queue + stats), then the Done. - let mut settle: Option> = None; + // Rejections per message (B1's re-delivery cap), keyed by the message + // id (the engine's, unique per send) or, for id-less messages, the + // prompt text: a pi that keeps rejecting one message must be capped, + // not retried forever. + let mut rejections: HashMap = HashMap::new(); + // The settle probe: armed at `agent_settled` (and after a handled or + // undispositioned acknowledgement, or a rejection), re-armed while pi + // reports busy. The turn ends only when a round reports pi at rest with + // an unchanged generation. + let mut probe: Option> = None; + // The generation the in-flight probe was armed at: `agent_start`, + // `compaction_start`, every prompt or steer sent, and every settle bump + // the counter; a probe result armed at a superseded generation is + // stale (a prompt acknowledgement raced it) and re-arms. + let mut probe_generation: u64 = 0; + // The session-tree cursor for provider turn refs (ticket 19): the leaf + // entry id of the last settle capture. Each settle's `get_entries` runs + // with it as `since`, bounding the listing — and the ref walk — to the + // entries the settling turn appended; a cursor pi no longer knows (a + // re-rooted branch) falls back to one full listing. `None` until the + // first capture: a fresh session's first walk simply runs to the root, + // and the first user entry it meets is this run's own prompt. + let mut ref_cursor: Option = None; + // Consecutive failed probe rounds (any success resets it). + let mut probe_failures: u32 = 0; + let mut generation: u64 = 0; // Per-turn token usage: the latest cumulative provider usage, emitted // at each turn boundary (codex parity). let mut last_usage: Option = None; @@ -907,15 +1625,24 @@ async fn run_session(session: Session) { let mut child_exit: Option> = None; let mut exit_drain_deadline: Option = None; - // Arm the first prompt. Prompt acceptance (preflight) — set when a - // prompt is armed, cleared when its response lands (Ok = run started, - // Err = rejected). A prompt sent while idle makes pi emit its own user - // message first — the echo, not a steer delivery. A prompt-owned run is - // between acceptance and its Done. + // Arm the first prompt. The acceptance — pi's preflight — is set when a + // prompt is armed and cleared when its response lands (Ok = accepted, + // read by disposition; Err = rejected). A prompt that starts a run + // makes pi emit its own user message first — the echo, not a steer + // delivery; a prompt pi queues delivers at that echo instead. A + // prompt-owned run is between acceptance and its Done. let images = load_image_contents(&request.attachments).await; - let mut acceptance: Option>> = Some( - prompt_acceptance(&client, &request.prompt, images, None), - ); + let mut armed = Some(Armed { + acceptance: prompt_acceptance(&client, &request.prompt, images), + message: None, + delivery: None, + }); + // Whether the current acceptance cycle has already spent its one + // immediate at-rest re-delivery of a rejected message: a pi that keeps + // rejecting falls back to the probe path (bounded by its re-arm delay) + // instead of hot-looping round trips. + let mut retried_at_rest = false; + generation += 1; // the prompt went out let mut awaiting_prompt_echo = true; let mut run_active = true; let mut done_current = false; @@ -933,138 +1660,608 @@ async fn run_session(session: Session) { exit_drain_deadline.unwrap_or_else(tokio::time::Instant::now) ), if exit_drain_deadline.is_some() => break 'main, - res = async { acceptance.as_mut().expect("guarded by if").await }, if acceptance.is_some() => { - acceptance = None; + res = async { (&mut armed.as_mut().expect("guarded by if").acceptance).await }, if armed.is_some() => { + let Armed { message: mailbox, delivery, .. } = + armed.take().expect("guarded by if"); + // The delivery record this prompt created (None = the run's + // own first prompt, whose echo is the awaiting one). + // Started/Handled retire it — the boundary fires here only + // when the echo has not consumed the record yet: the record + // is the single owner of the one `Steered` each input owes. match res { - Ok(_) => { /* accepted: the run streams until agent_settled */ } - Err(e) => { - // Pre-acceptance rejection: the prompt never ran — - // the turn errors with pi's own reason (an - // interrupt in flight still wins, ACP parity). - let (status, error) = if interrupted { - (DoneStatus::Interrupted, None) - } else { - (DoneStatus::Errored, Some(e.to_string())) - }; - done_current = true; - if interrupted { - done_after_interrupt = true; + Ok(data) => { + // A prompt acknowledgement bumps the settle + // generation: a probe whose reading predates it must + // re-arm — the steer it accepted may have landed in + // pi queue after the probe read it (T3 bug 16846). + generation += 1; + // A new acceptance cycle earns a fresh immediate + // re-delivery budget. + retried_at_rest = false; + match disposition_of(&data) { + Disposition::Started => { + // A run began: it streams until + // agent_settled, where the settle probe + // takes over. A mailbox message is delivered + // at acceptance — unless its echo already + // consumed the record (the processing race). + let retired = delivery + .is_some_and(|id| retire_delivery(&mut sent_deliveries, id)); + if retired + && !confirm_delivery( + &event_tx, + &mut assistant_message_id, + ) + .await + { + break 'main; + } + // Only when the boundary fired here does the + // run's own user echo become the awaiting + // one; otherwise the echo already delivered. + awaiting_prompt_echo |= retired; + run_active = true; + } + Disposition::Queued => { + // Accepted into pi steer queue: the delivery + // record stays — the user message echo + // consumes it (the delivery signal), in + // whatever order the acknowledgement + // processes. A strand is recovered by the + // settle probe clear_queue — the record now + // keeps the ORIGINAL message so that + // recovery re-queues it intact, attachments + // and run config included. + if let Some(id) = delivery + && let Some(record) = + sent_deliveries.iter_mut().find(|r| r.id == id) + { + record.message = mailbox.clone(); + } + // + // The run own first prompt queued behind an + // extension run: awaiting_prompt_echo + // already covers its echo. + run_active = true; + } + Disposition::Handled => { + // pi handled the prompt without a run (an + // extension command): the boundary is owed + // now, and the turn ends once pi reports it + // is not streaming. + let retired = delivery + .is_some_and(|id| retire_delivery(&mut sent_deliveries, id)); + if retired + && !confirm_delivery( + &event_tx, + &mut assistant_message_id, + ) + .await + { + break 'main; + } + awaiting_prompt_echo = false; + run_active = false; + if probe.is_none() { + probe_generation = generation; + probe = Some(rearm_probe(&client, false)); + } + } + Disposition::Unknown => { + // pi < 0.99 reports no disposition: + // acceptance still confirms delivery, but + // whether a run started is inferred — the + // probe (after a grace, so a starting run + // can flip isStreaming) ends the turn if + // nothing runs. + let retired = delivery + .is_some_and(|id| retire_delivery(&mut sent_deliveries, id)); + if retired + && !confirm_delivery( + &event_tx, + &mut assistant_message_id, + ) + .await + { + break 'main; + } + awaiting_prompt_echo |= retired; + if probe.is_none() { + probe_generation = generation; + probe = Some(rearm_probe(&client, true)); + } + } } - if !send( - &event_tx, - AgentEvent::Done { - status, - result: None, - error, - session_id: Some(session_pointer.clone()), - }, - ) - .await - { - break 'main; + }, + Err(e) => match mailbox { + Some(msg) => { + // A rejected mailbox prompt never ends the + // session: the rejection surfaces, the message + // returns to the queue, and it re-delivers once + // the turn ends — immediately when the session + // is already at rest, else after the probe + // closes the turn. The delivery record retires + // without a boundary: pi never consumed this + // input, so the engine at-least-once ledger + // keeps its entry until the re-delivery. + if let Some(id) = delivery { + retire_delivery(&mut sent_deliveries, id); + } + let _ = send( + &event_tx, + AgentEvent::Error { + message: format!("pi rejected the message: {e}"), + kind: None, + }, + ) + .await; + // B1's re-delivery cap: past the attempts, the + // message is NOT re-armed (an endlessly + // rejecting pi would otherwise Error + Done-loop + // on it forever) — it is handed back to the + // engine as a failed send, the at-least-once + // ledger re-dispatches it on a fresh runtime, + // and this turn ends ERRORED: never Completed + // for a message that never ran. + if let Some(attempts) = note_rejection(&mut rejections, &msg) { + turn_error = Some(format!( + "pi rejected the message {attempts} times: {e}" + )); + run_active = false; + if probe.is_none() { + probe_generation = generation; + probe = Some(rearm_probe(&client, true)); + } + } else { + driver_steers.push_front(msg); + run_active = false; + let at_rest = !saw_activity + && probe.is_none() + && steer_calls.is_empty(); + if at_rest && !retried_at_rest { + // One immediate re-delivery per acceptance + // cycle; a pi that keeps rejecting an idle + // prompt falls back to the probe path below, + // whose re-arm delay bounds the retry rate. + retried_at_rest = true; + if let Some(parked) = driver_steers.pop_front() + && !deliver_parked_steer( + &client, + &mut armed, + &mut sent_deliveries, + &mut delivery_seq, + &mut awaiting_prompt_echo, + &mut run_active, + &mut done_current, + &mut generation, + parked, + ) + .await + { + break 'main; + } + } else if probe.is_none() { + probe_generation = generation; + probe = Some(rearm_probe(&client, true)); + } + } } - break 'main; - } + None => { + // The run's own first prompt: the turn errors + // with pi's own reason (an interrupt in flight + // still wins, ACP parity). The session itself + // parks — the consumer decides its fate. + let (status, error) = if interrupted { + (DoneStatus::Interrupted, None) + } else { + (DoneStatus::Errored, Some(e.to_string())) + }; + done_current = true; + if interrupted { + done_after_interrupt = true; + } + if !send( + &event_tx, + AgentEvent::Done { + status, + result: None, + error, + session_id: Some(session_pointer.clone()), + }, + ) + .await + { + break 'main; + } + run_active = false; + } + }, } }, - outcome = async { settle.as_mut().expect("guarded by if").await }, if settle.is_some() => { - settle = None; - let (recovered, stats) = outcome; - if !recovered.is_empty() { - let stranded = pending_steer_deliveries.saturating_sub(recovered.len()); - pending_steer_deliveries = stranded; - let mut queue: Vec = driver_steers.drain(..).collect(); - queue.extend(recovered.into_iter().map(SteerMessage::text)); - driver_steers = queue.into(); - } else { - pending_steer_deliveries = 0; - } - if let Some(stats) = stats.as_ref().and_then(|stats| context_usage_event(stats)) - && !send(&event_tx, stats).await - { - break 'main; - } - let status = if turn_error.is_some() { - DoneStatus::Errored - } else { - DoneStatus::Completed - }; - let error = turn_error.take(); - done_current = true; - if interrupted { - done_after_interrupt = true; - } - if !send( - &event_tx, - AgentEvent::Done { - status, - result: None, - error, - session_id: Some(session_pointer.clone()), - }, - ) - .await - { - break 'main; - } - if interrupted { - break 'main; - } - // Agent activity during the settle round-trip (an - // extension-initiated run) needs its own settle + Done. - if saw_activity { - saw_activity = false; - let settle_client = client.clone(); - settle = Some(Box::pin(async move { - match tokio::time::timeout(SETTLE_TIMEOUT, settle_future(&settle_client, 0)).await { - Ok(outcome) => outcome, - Err(_) => (Vec::new(), None), + round = async { probe.as_mut().expect("guarded by if").await }, if probe.is_some() => { + probe = None; + match round { + ProbeRound::Busy => { + // pi is streaming or compacting: re-arm after the + // delay (the probe must never hot-loop while pi + // streams). + probe_failures = 0; + probe_generation = generation; + probe = Some(rearm_probe(&client, true)); + } + ProbeRound::Recovered(texts) => { + // pi is idle yet still held messages (steers + // stranded by the settle race): they were cleared + // and re-queue, in send order, as the next turn's + // prompts — INTACT (B6): a stranded steer's record + // keeps the original mailbox message, so its + // re-delivery carries the attachments and run + // config it was sent with, and its delivery record + // retires (pi will never echo it) so the + // re-delivery's own echo consumes the fresh record + // it creates. A stranded text with no record (the + // run's own first prompt) re-queues as bare text. + probe_failures = 0; + let mut recovered: VecDeque = VecDeque::new(); + for text in texts { + let record = sent_deliveries + .iter() + .position(|record| record.text == text) + .and_then(|pos| sent_deliveries.remove(pos)); + let message = record + .and_then(|record| record.message) + .unwrap_or_else(|| SteerMessage::text(text)); + recovered.push_back(message); } - })); - } - // A stranded steer becomes the next turn; otherwise the - // session parks — the caller owns teardown. - if settle.is_none() - && let Some(msg) = driver_steers.pop_front() - && !deliver_parked_steer( - &client, - &event_tx, - &mut assistant_message_id, - &mut acceptance, - &mut awaiting_prompt_echo, - &mut run_active, - &mut done_current, - &msg, - ) - .await - { - break 'main; + for message in recovered.into_iter().rev() { + driver_steers.push_front(message); + } + probe_generation = generation; + probe = Some(rearm_probe(&client, false)); + } + ProbeRound::Quiescent(stats) => { + if generation != probe_generation { + // A prompt, steer or settle bumped since arming: + // this reading is stale — re-probe. This is what + // re-arms the probe after a prompt + // acknowledgement that raced it. + probe_failures = 0; + probe_generation = generation; + probe = Some(rearm_probe(&client, false)); + } else if armed.is_some() || !steer_calls.is_empty() { + // A prompt or steer is still awaiting pi's + // acknowledgement (B4): generation equality + // cannot see an input that is in preflight — the + // bump lands when its acknowledgement does — + // and the turn is not over while one is. Wait it + // out: re-arm (after the delay — a slow + // preflight must not hot-loop the probe) and + // re-check once the acceptance resolves. + probe_failures = 0; + probe_generation = generation; + probe = Some(rearm_probe(&client, true)); + } else { + probe_failures = 0; + if let Some(usage) = stats + .as_ref() + .and_then(|stats| context_usage_event(stats)) + && !send(&event_tx, usage).await + { + break 'main; + } + // The provider turn ref (ticket 19): `get_entries` + // now that the turn has settled — the session + // file plus the LAST user entry the turn + // consumed (its steer when one landed mid-run, + // else its prompt — for an unsteered turn that + // is also its first user entry, the ticket's + // spelling). The engine holds it for the stamp + // when the entry's segment finishes at the + // `Done` below. Best-effort: a failed read costs + // the ref — a later fork falls back to the + // budgeted paste — never the turn. The capture + // is BOUNDED and interruptible (B7): its round + // trips must never wedge the run loop for their + // full 2×15s budget — a Stop landing mid-capture + // is read at once and cuts the wait short (the + // ref is lost, never the turn), and the Done + // below waits at most the short budget. + let capture = + capture_turn_ref(&client, &session_pointer, &mut ref_cursor); + tokio::pin!(capture); + let mut stopped = false; + let captured = tokio::select! { + reference = &mut capture => reference, + _ = interrupt.cancelled(), if !interrupt_sent => { + stopped = true; + None + } + _ = tokio::time::sleep(REF_CAPTURE_BUDGET) => None, + }; + if stopped { + // Stop landed mid-capture: the turn already + // settled with nothing in flight, so it closes + // out as interrupted at once (the "already + // settled" shortcut of the interrupt arm). + interrupted = true; + done_current = true; + done_after_interrupt = true; + let _ = send( + &event_tx, + AgentEvent::Done { + status: DoneStatus::Interrupted, + result: None, + error: None, + session_id: Some(session_pointer.clone()), + }, + ) + .await; + break 'main; + } + if let Some(reference) = captured + && !send(&event_tx, AgentEvent::TurnRef { reference }).await + { + break 'main; + } + let status = if turn_error.is_some() { + DoneStatus::Errored + } else { + DoneStatus::Completed + }; + let error = turn_error.take(); + done_current = true; + if interrupted { + done_after_interrupt = true; + } + if !send( + &event_tx, + AgentEvent::Done { + status, + result: None, + error, + session_id: Some(session_pointer.clone()), + }, + ) + .await + { + break 'main; + } + if interrupted { + break 'main; + } + // A queued message becomes the next turn; + // otherwise the session parks — the caller owns + // teardown. + if let Some(parked) = driver_steers.pop_front() + && !deliver_parked_steer( + &client, + &mut armed, + &mut sent_deliveries, + &mut delivery_seq, + &mut awaiting_prompt_echo, + &mut run_active, + &mut done_current, + &mut generation, + parked, + ) + .await + { + break 'main; + } + } + } + ProbeRound::Failed(reason) => { + probe_failures += 1; + if probe_failures >= PROBE_ATTEMPTS { + // The probe failed repeatedly: end the turn + // errored — visibly, with the reason — never a + // silent kill. The session parks; the queued + // messages' ledger entries survive for the + // engine's re-dispatch. The counter resets so + // the next turn's probe starts a fresh budget. + // + // The mailbox must not wedge behind the dead + // turn (B2): the delivery records clear — no + // later turn's user echo may consume a stale + // one — and the driver-owned messages drain + // below exactly like after a clean settle (the + // next one becomes the next turn), so nothing + // lines up behind the dead turn forever and a + // later mailbox message still gets its turn. + probe_failures = 0; + sent_deliveries.clear(); + turn_error = None; + done_current = true; + if interrupted { + done_after_interrupt = true; + } + if !send( + &event_tx, + AgentEvent::Done { + status: DoneStatus::Errored, + result: None, + error: Some(format!( + "pi's settle probe failed after \ + {PROBE_ATTEMPTS} attempts: {reason}" + )), + session_id: Some(session_pointer.clone()), + }, + ) + .await + { + break 'main; + } + if interrupted { + break 'main; + } + if let Some(parked) = driver_steers.pop_front() + && !deliver_parked_steer( + &client, + &mut armed, + &mut sent_deliveries, + &mut delivery_seq, + &mut awaiting_prompt_echo, + &mut run_active, + &mut done_current, + &mut generation, + parked, + ) + .await + { + break 'main; + } + } else { + probe_generation = generation; + probe = Some(rearm_probe(&client, true)); + } + } } }, res = steer_calls.next(), if !steer_calls.is_empty() => match res { - Some((text, retried, res)) => match res { - Ok(_) => { /* queued in pi; delivery shows as a user message */ } + Some((delivery, msg, res)) => match res { + Ok(data) => { + // The steer acknowledgement bumps the settle + // generation: a probe whose reading predates it must + // re-arm — the steer may have landed in pi queue + // after the probe read it (T3 bug 16846). + generation += 1; + match disposition_of(&data) { + Disposition::Started => { + // The steer raced the run end and pi started + // a run with it: delivered now, and the run + // streams to its own settle — unless its + // echo already consumed the record (the + // processing race). + let retired = + retire_delivery(&mut sent_deliveries, delivery); + if retired + && !confirm_delivery( + &event_tx, + &mut assistant_message_id, + ) + .await + { + break 'main; + } + // The started run's own user echo is the + // awaiting one — it must not consume the + // next queued steer's record. + awaiting_prompt_echo |= retired; + run_active = true; + } + Disposition::Queued => { + // Accepted into pi steer queue: the delivery + // record stays — the user message echo + // consumes it, in whatever order this + // acknowledgement processes. The record keeps + // the message so a strand recovered by + // clear_queue re-queues it intact (B6). + if let Some(record) = + sent_deliveries.iter_mut().find(|r| r.id == delivery) + { + record.message = Some(msg.clone()); + } + } + Disposition::Handled => { + // pi handled it without starting a run: the + // message is delivered, at acceptance unless + // an echo already consumed the record. + if retire_delivery(&mut sent_deliveries, delivery) + && !confirm_delivery( + &event_tx, + &mut assistant_message_id, + ) + .await + { + break 'main; + } + } + Disposition::Unknown => { + // pi < 0.99: delivery lands at the echo, if + // one ever comes — the record stays for it. + } + } + }, Err(e) if client.is_closed() => { tracing::debug!(target: "roboco_harness::pi", "steer lost to exit: {e}"); } - Err(e) if !retried => { - // Rejection (pi errors on extension-command text in - // a steer): redeliver as a prompt with steer - // behavior — pi runs extension commands immediately. - tracing::debug!( - target: "roboco_harness::pi", - "steer rejected (retrying as a steer-behavior prompt): {e}" - ); - steer_calls.push(steer_request(&client, &text, true)); - } Err(e) => { + // Rejected: surface it and hand the message back to + // the queue — it re-delivers once the turn ends + // (immediately when the session is already at + // rest, else when the settle probe closes the + // turn). The delivery record retires without a + // boundary: pi never consumed this input, so the + // engine at-least-once ledger keeps its entry until + // the re-delivery. Past B1's re-delivery cap the + // message is handed back to the engine as a failed + // send (the turn ends errored — never a loop of + // Error + Completed for a message that never ran); + // a rejection landing after the turn already ended + // (a probe failure closed it) goes straight back too. + retire_delivery(&mut sent_deliveries, delivery); let _ = send( &event_tx, - AgentEvent::Error { message: format!("Steering failed: {e}") }, + AgentEvent::Error { + message: format!("pi rejected the steer: {e}"), + kind: None, + }, ) .await; + let at_rest = !run_active + && !saw_activity + && probe.is_none() + && armed.is_none() + && steer_calls.is_empty(); + if let Some(attempts) = note_rejection(&mut rejections, &msg) { + if done_current { + // The turn already ended (a probe failure + // closed it): there is no turn left to fail — + // the Error above surfaced the rejection, and + // the engine's ledger still owns the message. + } else { + turn_error = Some(format!( + "pi rejected the message {attempts} times: {e}" + )); + if at_rest { + // Nothing else will settle the turn: the + // probe closes it errored. + probe_generation = generation; + probe = Some(rearm_probe(&client, true)); + } + } + } else { + driver_steers.push_front(msg); + if at_rest && (done_current || !retried_at_rest) { + // Same one-shot immediate re-delivery budget as a + // rejected parked prompt: bounded by the probe + // re-arm delay if pi keeps rejecting. + retried_at_rest = true; + if let Some(parked) = driver_steers.pop_front() + && !deliver_parked_steer( + &client, + &mut armed, + &mut sent_deliveries, + &mut delivery_seq, + &mut awaiting_prompt_echo, + &mut run_active, + &mut done_current, + &mut generation, + parked, + ) + .await + { + break 'main; + } + } else if at_rest && probe.is_none() { + // The immediate budget is spent: the probe ends + // the turn and its Done pops the message back. + probe_generation = generation; + probe = Some(rearm_probe(&client, true)); + } + } } }, None => {} @@ -1076,6 +2273,10 @@ async fn run_session(session: Session) { let kind = event.get("type").and_then(Value::as_str).unwrap_or(""); match kind { "agent_start" => { + // Agent activity: bumps the settle generation + // (an in-flight probe would read stale) and + // marks the turn active until its settle. + generation += 1; saw_activity = true; } "message_start" => { @@ -1083,19 +2284,22 @@ async fn run_session(session: Session) { if role == Some("assistant") { tracker.reset(); } else if role == Some("user") { - // Our idle prompt's own echo, or a steer - // delivery out of pi's queue. + // Our prompt's own echo, or a queued input's + // delivery out of pi's queue (upstream + // PROTOCOL.md: message_start for each + // consumed user input emits its own + // Steered; the ACK and queue_update are + // never consumption receipts). if awaiting_prompt_echo { awaiting_prompt_echo = false; - } else if pending_steer_deliveries > 0 { - pending_steer_deliveries -= 1; - let (prev, next) = rotate(&mut assistant_message_id); - if !send( + } else if sent_deliveries.pop_front().is_some() { + // The consumed signal: this echo + // delivers the oldest record — whether + // or not its acknowledgement has been + // processed yet. + if !confirm_delivery( &event_tx, - AgentEvent::Steered { - assistant_message_id: Some(prev), - next_assistant_message_id: Some(next), - }, + &mut assistant_message_id, ) .await { @@ -1128,6 +2332,7 @@ async fn run_session(session: Session) { if let Some(error) = message_end_error(&event) { let _ = send(&event_tx, AgentEvent::Error { message: error.clone(), + kind: None, }) .await; turn_error = Some(error); @@ -1206,6 +2411,7 @@ async fn run_session(session: Session) { "pi hit a transient error and is retrying \ (attempt {attempt}/{max}): {message}" ), + kind: None, }, ) .await; @@ -1227,8 +2433,13 @@ async fn run_session(session: Session) { } "compaction_end" => { // A failed compaction that will not retry ends - // the run as an error; success (manual, threshold, - // overflow-retry) just continues or re-prompts. + // the run as an error; success (manual, + // threshold, overflow-retry) just continues or + // re-prompts. A compaction retry that SUCCEEDED + // recovered the turn: the error that triggered + // it (the overflow error message, an exhausted + // auto-retry) no longer rejects the prompt — the + // retried run may still complete it. let will_retry = event.get("willRetry").and_then(Value::as_bool).unwrap_or(false); let aborted = event.get("aborted").and_then(Value::as_bool).unwrap_or(false); let failed = event.get("result").map(Value::is_null).unwrap_or(true) @@ -1241,6 +2452,8 @@ async fn run_session(session: Session) { .unwrap_or("pi's compaction failed.") .to_owned(), ); + } else if will_retry && !failed && !aborted { + turn_error = None; } } "extension_ui_request" => { @@ -1255,10 +2468,44 @@ async fn run_session(session: Session) { .unwrap_or_default(); tracing::debug!(target: "roboco_harness::pi", "pi extension error: {detail}"); } + "compaction_start" => { + // Compaction is agent activity too: it bumps + // the settle generation — the probe confirms + // rest only after compaction_end. + generation += 1; + } "agent_settled" => { + // Every settle bumps the generation. + generation += 1; if interrupted { // abort() landed: close out as Interrupted - // without waiting for anything else. + // without waiting for anything else. The + // settle capture still runs — pi is at rest + // (it just answered agent_settled), and the + // interrupted reply keeps its provider turn + // ref so a fork from it continues natively + // instead of pasting (spec story 59: + // interrupted replies are fork boundaries + // too). Best-effort, as at every settle — + // and bounded (B7): the round trip never + // delays the Interrupted Done by more than + // the short budget. + let reference = tokio::time::timeout( + REF_CAPTURE_BUDGET, + capture_turn_ref( + &client, + &session_pointer, + &mut ref_cursor, + ), + ) + .await + .ok() + .flatten(); + if let Some(reference) = reference + && !send(&event_tx, AgentEvent::TurnRef { reference }).await + { + break 'main; + } done_current = true; done_after_interrupt = true; let _ = send( @@ -1273,21 +2520,20 @@ async fn run_session(session: Session) { .await; break 'main; } - if (run_active || saw_activity) && settle.is_none() { + if run_active || saw_activity { run_active = false; saw_activity = false; - let settle_client = client.clone(); - let pending = pending_steer_deliveries; - settle = Some(Box::pin(async move { - match tokio::time::timeout(SETTLE_TIMEOUT, settle_future(&settle_client, pending)).await { - Ok(outcome) => outcome, - Err(_) => (Vec::new(), None), - } - })); + if probe.is_none() { + probe_generation = generation; + probe = Some(rearm_probe(&client, false)); + } + // A probe already in flight (armed after a + // handled or undispositioned acknowledgement, + // or a rejection): its generation check + // re-arms it for this settle. } - // A settle with no run behind it (spurious, a - // settle already in flight — its completion - // re-arms for any activity seen meanwhile — or + // A settle with no run behind it (spurious, one + // already covered by an in-flight probe, or // post-abort) is ignored. } // turn_start, queue_update, bash_execution_update @@ -1300,16 +2546,17 @@ async fn run_session(session: Session) { } } Some(PiIncoming::Eof) | None => { - // The settle sequence and the acceptance both race EOF - // through the pending map: a response that already + // The settle probe and the acceptance both race EOF + // through the pending map: a result that already // resolved must still close out the turn. Give each a // short drain before falling to crash bookkeeping. - if let Some(mut fut) = settle.take() { - if let Ok((recovered, stats)) = + if let Some(mut fut) = probe.take() { + if let Ok(ProbeRound::Quiescent(stats)) = tokio::time::timeout(EOF_DRAIN, &mut fut).await { - let _ = recovered; - if let Some(usage) = stats.as_ref().and_then(|s| context_usage_event(s)) { + if let Some(usage) = + stats.as_ref().and_then(|s| context_usage_event(s)) + { let _ = send(&event_tx, usage).await; } done_current = true; @@ -1329,18 +2576,30 @@ async fn run_session(session: Session) { .await; } } - if let Some(mut fut) = acceptance.take() { + if let Some(mut armed_prompt) = armed.take() { // A resolved REJECTION carries pi's own reason; a // resolved acceptance (or a request failed by the // EOF cleanup) falls through to the crash message. - if let Ok(Err(e)) = tokio::time::timeout(EOF_DRAIN, &mut fut).await { + // A Stop that killed pi before the answer still wins + // (ACP parity) — one Interrupted Done, never two. + if let Ok(Err(e)) = + tokio::time::timeout(EOF_DRAIN, &mut armed_prompt.acceptance).await + { done_current = true; + let (status, error) = if interrupted { + (DoneStatus::Interrupted, None) + } else { + (DoneStatus::Errored, Some(e.to_string())) + }; + if interrupted { + done_after_interrupt = true; + } let _ = send( &event_tx, AgentEvent::Done { - status: DoneStatus::Errored, + status, result: None, - error: Some(e.to_string()), + error, session_id: Some(session_pointer.clone()), }, ) @@ -1360,28 +2619,46 @@ async fn run_session(session: Session) { // current configuration runs (startup parity). let _ = configure_live(&client, next).await; } - let text = msg.prompt.clone(); - if run_active || acceptance.is_some() { - // Live turn: pi's own steer queue (delivered after - // the current assistant turn's tool calls). The - // attachments' path refs ride the text; the parked - // delivery inlines them as images. - steer_calls.push(steer_request(&client, &text, false)); - pending_steer_deliveries += 1; - } else if settle.is_some() || !steer_calls.is_empty() || !driver_steers.is_empty() { - // The settle sequence or a steer acknowledgement is + if run_active || armed.is_some() { + // Live turn: the prompt rides steer behavior — pi + // queues it while streaming (delivered after the + // current assistant turn's tool calls, all queued + // steers together in mode "all") and starts a run + // when idle, atomically against settlement. The + // staged attachments inline as images, exactly like + // a turn-start prompt. + generation += 1; // the prompt goes out + delivery_seq += 1; + let delivery = delivery_seq; + sent_deliveries.push_back(DeliveryRecord { + id: delivery, + text: msg.prompt.clone(), + message: None, + }); + // Images load BEFORE the call is queued (B5): the + // steer calls run concurrently, and a call that + // loads its images inside the future can be beaten + // to the wire by a later, text-only steer — pi would + // receive the burst out of order. Loading here keeps + // the FuturesUnordered's completion order and the + // wire's send order the same: mailbox order. + let images = load_image_contents(&msg.attachments).await; + steer_calls.push(steer_request(&client, msg, delivery, images)); + } else if probe.is_some() || !steer_calls.is_empty() || !driver_steers.is_empty() { + // The settle probe or a steer acknowledgement is // still winding down: FIFO behind it. driver_steers.push_back(msg); } else { if !deliver_parked_steer( &client, - &event_tx, - &mut assistant_message_id, - &mut acceptance, + &mut armed, + &mut sent_deliveries, + &mut delivery_seq, &mut awaiting_prompt_echo, &mut run_active, &mut done_current, - &msg, + &mut generation, + msg, ) .await { @@ -1394,8 +2671,8 @@ async fn run_session(session: Session) { // finish once nothing is in flight. steering_open = false; if !run_active - && acceptance.is_none() - && settle.is_none() + && armed.is_none() + && probe.is_none() && steer_calls.is_empty() && driver_steers.is_empty() { @@ -1407,10 +2684,21 @@ async fn run_session(session: Session) { _ = interrupt.cancelled(), if !interrupt_sent => { interrupt_sent = true; interrupted = true; - if settle.is_some() { - // The turn already settled; the stats round-trip was the - // only thing in flight. Close out as interrupted. - let _ = settle.take(); + // An armed probe does NOT mean the turn settled (B3): the + // probe also runs while work is live (a handled prompt + // during an extension run, a rejection mid-run). Only a + // probe with nothing else in flight is "already settled" — + // otherwise Stop must clear pi's queue and abort exactly + // like any live turn, or a steer pi never consumed would + // survive into the aborted run. + let settled = !run_active + && !saw_activity + && armed.is_none() + && steer_calls.is_empty(); + if probe.is_some() && settled { + // The turn already settled; the probe was the only thing + // in flight. Close out as interrupted. + let _ = probe.take(); done_current = true; done_after_interrupt = true; let _ = send( @@ -1425,13 +2713,24 @@ async fn run_session(session: Session) { .await; break 'main; } - let in_flight = run_active || acceptance.is_some() || saw_activity; + let in_flight = + run_active || armed.is_some() || saw_activity || !steer_calls.is_empty(); if in_flight { - // abort() waits for idle before responding; do not block - // the loop on it — the agent_settled it triggers (or the - // signal escalation below) ends the run. + // clear_queue FIRST: steers pi accepted but never + // consumed must not survive into the abort — they are + // cancelled with the turn (the engine's at-least-once + // ledger drops them on the interrupted exit, their user + // entries already sit in the transcript), never + // delivered into the aborted run. abort() waits for + // idle before responding; neither request blocks the + // loop — the agent_settled abort triggers (or the + // signal escalation below) ends the run. The wire keeps + // the order: both ride the client's single writer. let client = client.clone(); tokio::spawn(async move { + if let Err(e) = client.request("clear_queue", json!({})).await { + tracing::debug!(target: "roboco_harness::pi", "clear_queue failed (escalation will reap): {e}"); + } if let Err(e) = client.request("abort", json!({})).await { tracing::debug!(target: "roboco_harness::pi", "abort failed (escalation will reap): {e}"); } @@ -1497,69 +2796,66 @@ async fn run_session(session: Session) { } } -/// Arm a prompt and return its acceptance future. `streaming_behavior` is -/// set only for the steer-rejection retry (a prompt arriving while pi is -/// streaming must declare its queue behavior or pi rejects it). +/// Arm a prompt and return its acceptance future. Every prompt — turn +/// starts included — goes out with `streamingBehavior: "steer"`: pi starts +/// a run when idle and queues the prompt when an extension run is +/// streaming, atomically against settlement, so a prompt can never strand +/// against the end of a run. The acceptance is unbounded on purpose: pi's +/// preflight (input handlers, `before_agent_start` hooks including the +/// Roboco MCP bridge, and pre-prompt compaction) may take arbitrarily +/// long, and only Stop interrupts a slow acceptance. fn prompt_acceptance( client: &PiClient, text: &str, images: Vec, - streaming_behavior: Option<&str>, ) -> BoxFuture<'static, Result> { let client = client.clone(); let text = text.to_owned(); - let mut params = json!({ "message": text }); + let mut params = json!({ "message": text, "streamingBehavior": "steer" }); if !images.is_empty() { params["images"] = Value::Array(images); } - if let Some(behavior) = streaming_behavior { - params["streamingBehavior"] = Value::String(behavior.to_owned()); - } - Box::pin(async move { - match tokio::time::timeout(ACCEPT_TIMEOUT, client.request("prompt", params)).await { - Ok(result) => result, - Err(_) => Err(HarnessError::Protocol( - "pi did not accept the prompt within 30s (the process is likely wedged)".into(), - )), - } - }) + Box::pin(async move { client.request("prompt", params).await }) } -/// One steer request, optionally retrying a rejected steer as a -/// steer-behavior prompt (pi rejects extension-command text in `steer` -/// but runs it immediately as a prompt). -fn steer_request(client: &PiClient, text: &str, as_prompt: bool) -> SteerCall { +/// One mid-run steer: the same `prompt` with steer behavior (pi queues it +/// while streaming and starts a run when idle). Carries the mailbox +/// message so a rejection can hand it back to the queue verbatim, its +/// delivery-record id so the acknowledgement (or a rejection) can retire +/// it, and the message's attachments staged as inline images — a steer +/// delivers its images exactly like a turn-start prompt (the path refs +/// ride the text as a fallback either way). The images arrive LOADED (the +/// caller loads them before queueing, so a burst's wire order is its +/// mailbox order — see the steering arm). +fn steer_request( + client: &PiClient, + msg: SteerMessage, + delivery: u64, + images: Vec, +) -> SteerCall { let client = client.clone(); - let text = text.to_owned(); Box::pin(async move { - let result = if as_prompt { - let params = json!({ "message": text, "streamingBehavior": "steer" }); - client.request("prompt", params).await - } else { - client.request("steer", json!({ "message": text })).await - }; - (text, as_prompt, result) + let mut params = json!({ + "message": msg.prompt.clone(), + "streamingBehavior": "steer", + }); + if !images.is_empty() { + params["images"] = Value::Array(images); + } + let result = client.request("prompt", params).await; + (delivery, msg, result) }) } -/// A parked session's next message: confirm the boundary, then arm the -/// prompt (the mailbox's FIFO ledger retires on the Steered boundary) with -/// the message's attachments inlined and its run config applied. -#[allow(clippy::too_many_arguments)] -async fn deliver_parked_steer( - client: &PiClient, +/// Emit the `Steered` boundary that confirms a mailbox message's delivery: +/// the engine retires its at-least-once ledger entry here and splits the +/// assistant entry at the boundary. +async fn confirm_delivery( event_tx: &mpsc::Sender>, assistant_message_id: &mut String, - acceptance: &mut Option>>, - awaiting_prompt_echo: &mut bool, - run_active: &mut bool, - done_current: &mut bool, - msg: &SteerMessage, ) -> bool { - // Confirm the boundary FIRST (the mailbox ledger retires here), then - // arm the prompt — ACP parity for the between-turns delivery path. let (prev, next) = rotate(assistant_message_id); - if !send( + send( event_tx, AgentEvent::Steered { assistant_message_id: Some(prev), @@ -1567,17 +2863,53 @@ async fn deliver_parked_steer( }, ) .await - { - return false; - } +} + +/// A parked session's next message: apply its run config, then arm the +/// prompt with the message's attachments inlined. The delivery boundary +/// (`Steered`) is owed only once pi ACCEPTS the prompt — at the +/// acknowledgement for a run or a handled command, at the user message +/// echo for a queued steer — so a rejected prompt leaves the message +/// undelivered and recoverable. +#[allow(clippy::too_many_arguments)] +async fn deliver_parked_steer( + client: &PiClient, + armed: &mut Option, + sent_deliveries: &mut VecDeque, + delivery_seq: &mut u64, + awaiting_prompt_echo: &mut bool, + run_active: &mut bool, + done_current: &mut bool, + generation: &mut u64, + msg: SteerMessage, +) -> bool { if let Some(next) = &msg.config { + // Best-effort: a rejected switch logs inside and the current + // configuration runs (startup parity). let _ = configure_live(client, next).await; } let images = load_image_contents(&msg.attachments).await; - *acceptance = Some(prompt_acceptance(client, &msg.prompt, images, None)); - *awaiting_prompt_echo = true; + // The delivery record is created when the input goes out: pi's echo + // consumes it whether or not the acknowledgement has been processed + // yet (upstream PROTOCOL.md: the ACK is not a consumption receipt). + *delivery_seq += 1; + let delivery = *delivery_seq; + sent_deliveries.push_back(DeliveryRecord { + id: delivery, + text: msg.prompt.clone(), + message: None, + }); + *armed = Some(Armed { + acceptance: prompt_acceptance(client, &msg.prompt, images), + message: Some(msg), + delivery: Some(delivery), + }); + // Set per disposition once pi answers: a started run echoes its own + // user message; a queued acceptance delivers at the echo instead. + *awaiting_prompt_echo = false; *run_active = true; *done_current = false; + *generation += 1; // the prompt goes out true } @@ -1672,3 +3004,182 @@ fn image_mime_type(path: &std::path::Path, bytes: &[u8]) -> Option<&'static str> _ => None, }) } + +#[cfg(test)] +mod tests { + use super::steering_mode_configured; + use super::{ + ForkBoundary, SessionEntries, branch_user_entries, fork_boundary, newest_user_entry, + parse_entries, + }; + use std::path::Path; + + fn write_settings(dir: &Path, body: &str) { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(dir.join("settings.json"), body).unwrap(); + } + + /// One session tree as a get_entries payload: `ids` chains the entries + /// in order (the bool marks user entries). + fn tree(ids: &[(&str, bool)], leaf: Option<&str>) -> Option { + let mut parent: Option<&str> = None; + let mut entries = Vec::new(); + for (id, user) in ids { + entries.push(serde_json::json!({ + "type": "message", + "id": id, + "parentId": parent, + "message": { "role": if *user { "user" } else { "assistant" } }, + })); + parent = Some(id); + } + parse_entries(&serde_json::json!({ + "entries": entries, + "leafId": leaf, + })) + } + + /// The ref walk names the newest user entry on the branch: the turn's + /// steer when one landed mid-run, else its prompt. + #[test] + fn the_ref_walk_names_the_last_user_entry_of_the_turn() { + // u1 → a1 → s2 (a steer) → a2: the turn consumed s2 last. + let entries = tree( + &[("u1", true), ("a1", false), ("s2", true), ("a2", false)], + Some("a2"), + ) + .expect("parseable"); + assert_eq!(newest_user_entry(&entries).as_deref(), Some("s2")); + // Without the steer, the prompt IS the turn's last user entry. + let entries = tree(&[("u1", true), ("a1", false)], Some("a1")).expect("parseable"); + assert_eq!(newest_user_entry(&entries).as_deref(), Some("u1")); + // A full listing of a stretch with no user entry of its own (an + // extension-initiated run) degrades to the OLDER turn's entry — + // the since cursor is what scopes the walk to one turn. + let entries = tree( + &[("u1", true), ("a1", false), ("a2", false)], + Some("a2"), + ) + .expect("parseable"); + assert_eq!(newest_user_entry(&entries).as_deref(), Some("u1")); + // A since-bounded listing (the stretch ends at the cursor) names + // nothing: the walk leaves the listing before any user entry. + let entries = tree(&[("a2", false), ("a3", false)], Some("a3")).expect("parseable"); + assert_eq!(newest_user_entry(&entries), None); + // An empty session has no leaf to walk from. + let entries = tree(&[], None).expect("parseable"); + assert_eq!(newest_user_entry(&entries), None); + } + + /// A corrupt parent cycle cannot loop the walks: they terminate (the + /// guard bounds the pass) even though the parent chain never ends. + #[test] + fn corrupt_parent_cycles_terminate() { + let data = serde_json::json!({ + "entries": [ + { "type": "message", "id": "x", "parentId": "y", "message": { "role": "user" } }, + { "type": "message", "id": "y", "parentId": "x", "message": { "role": "assistant" } }, + ], + "leafId": "y", + }); + let entries = parse_entries(&data).expect("parseable"); + // The walk from the leaf meets the user entry before the cycle + // guard trips, and never loops past it. + assert_eq!(newest_user_entry(&entries).as_deref(), Some("x")); + assert!(branch_user_entries(&entries).len() <= 2); + } + + /// The branch walk lists the active branch's user entries, newest + /// first — the order the fork boundary search steps through. + #[test] + fn the_branch_walk_lists_user_entries_newest_first() { + let entries = tree( + &[ + ("u1", true), + ("a1", false), + ("u2", true), + ("a2", false), + ("u3", true), + ("a3", false), + ], + Some("a3"), + ) + .expect("parseable"); + assert_eq!( + branch_user_entries(&entries), + vec!["u3".to_owned(), "u2".to_owned(), "u1".to_owned()] + ); + } + + /// The fork boundary: the newest turn forks the whole session (no + /// RPC), an earlier one names the NEXT turn's user entry, and a ref + /// that is not on the branch refuses (the budgeted-paste fallback). + #[test] + fn the_fork_boundary_names_the_next_turn() { + let users: Vec = ["u3", "u2", "u1"] + .iter() + .map(|id| id.to_string()) + .collect(); + // The boundary is the latest turn: the --fork copy already ends + // there, no RPC fork. + assert_eq!(fork_boundary(&users, "u3"), ForkBoundary::NewestTurn); + // An earlier boundary names the turn after it. + assert_eq!(fork_boundary(&users, "u2"), ForkBoundary::RootAt("u3".to_owned())); + assert_eq!(fork_boundary(&users, "u1"), ForkBoundary::RootAt("u2".to_owned())); + // A stale ref refuses rather than forking somewhere wrong. + assert_eq!(fork_boundary(&users, "gone"), ForkBoundary::OffBranch); + } + + /// No settings anywhere: the driver selects the mode itself. + #[test] + fn steering_mode_is_unconfigured_without_settings() { + let home = tempfile::tempdir().unwrap(); + let project = tempfile::tempdir().unwrap(); + assert!(!steering_mode_configured( + Some(home.path()), + &project.path().display().to_string(), + )); + } + + /// A mode in the agent directory (the user's own, or `/steering` written + /// globally) is an explicit choice: never overwritten. + #[test] + fn agent_dir_settings_count_as_configured() { + let home = tempfile::tempdir().unwrap(); + let project = tempfile::tempdir().unwrap(); + write_settings(home.path(), r#"{"steeringMode": "one-at-a-time"}"#); + assert!(steering_mode_configured( + Some(home.path()), + &project.path().display().to_string(), + )); + } + + /// A mode in the project `.pi/settings.json` counts the same way. + #[test] + fn project_settings_count_as_configured() { + let home = tempfile::tempdir().unwrap(); + let project = tempfile::tempdir().unwrap(); + write_settings(&project.path().join(".pi"), r#"{"steeringMode": "all"}"#); + assert!(steering_mode_configured( + Some(home.path()), + &project.path().display().to_string(), + )); + } + + /// Settings without the key (or unreadable JSON) are not a choice. + #[test] + fn settings_without_the_key_are_not_configured() { + let home = tempfile::tempdir().unwrap(); + let project = tempfile::tempdir().unwrap(); + write_settings(home.path(), r#"{"defaultModel": "x/y"}"#); + assert!(!steering_mode_configured( + Some(home.path()), + &project.path().display().to_string(), + )); + write_settings(home.path(), "not json"); + assert!(!steering_mode_configured( + Some(home.path()), + &project.path().display().to_string(), + )); + } +} diff --git a/crates/harness/tests/acp.rs b/crates/harness/tests/acp.rs index d5943c1115..5e99de9a3d 100644 --- a/crates/harness/tests/acp.rs +++ b/crates/harness/tests/acp.rs @@ -48,6 +48,8 @@ fn request(prompt: &str) -> RunRequest { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } diff --git a/crates/harness/tests/acp_quiet.rs b/crates/harness/tests/acp_quiet.rs index 3b592d57bb..683a84993e 100644 --- a/crates/harness/tests/acp_quiet.rs +++ b/crates/harness/tests/acp_quiet.rs @@ -53,6 +53,8 @@ fn request(prompt: &str) -> RunRequest { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } diff --git a/crates/harness/tests/acp_stall.rs b/crates/harness/tests/acp_stall.rs index b9526c1037..14cc63b284 100644 --- a/crates/harness/tests/acp_stall.rs +++ b/crates/harness/tests/acp_stall.rs @@ -60,6 +60,8 @@ async fn silent_agent_errors_via_the_prompt_stall_watchdog() { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }; let harness = AcpHarness::grok().with_executable(fixture_path()); let stream = harness.run(request, controls).await.expect("run starts"); @@ -73,7 +75,7 @@ async fn silent_agent_errors_via_the_prompt_stall_watchdog() { let error = events .iter() .find_map(|e| match e { - AgentEvent::Error { message } => Some(message.clone()), + AgentEvent::Error { message, .. } => Some(message.clone()), _ => None, }) .expect("visible error chip"); diff --git a/crates/harness/tests/claude.rs b/crates/harness/tests/claude.rs index c60ee893d2..eebd8e3252 100644 --- a/crates/harness/tests/claude.rs +++ b/crates/harness/tests/claude.rs @@ -14,8 +14,8 @@ use tokio::sync::{mpsc, oneshot}; use roboco_harness::{ CancellationToken, ClaudeHarness, Harness, HarnessError, RunControls, SteerMessage, TurnControl}; use roboco_proto::{ - AgentEvent, DoneStatus, HarnessId, RunRequest, SandboxLevel, ToolCall, UserInputAnswer, - UserInputQuestion, + AgentErrorKind, AgentEvent, DoneStatus, HarnessId, RunRequest, SandboxLevel, ToolCall, + UserInputAnswer, UserInputQuestion, }; fn fixture_path() -> PathBuf { @@ -49,6 +49,8 @@ fn request(prompt: &str) -> RunRequest { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } @@ -93,6 +95,367 @@ async fn run_to_end( .expect("run finished in time") } +/// A resume whose session file the CLI store does not know, carrying the +/// engine's budgeted fallback (ticket 09's account-failover continuation): +/// the driver starts FRESH — no `--resume` reaches the CLI (the fake fails +/// the run if it does) — with the history in front of the prompt. +#[tokio::test] +async fn a_resume_without_a_session_file_starts_fresh_with_the_history() { + let (controls, _steer, _token) = controls("A"); + let mut req = request("scenario:resume-missing"); + // An id no real Claude session store knows (the check only READS the + // store — no login, no write). + req.resume = Some("roboco-missing-3f9c".into()); + req.resume_fallback_history = Some("HISTORY PINEAPPLE ".into()); + let events = run_to_end(&harness(), req, controls).await; + + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::TextDelta { text } + if text.contains("HISTORY PINEAPPLE") && text.contains("scenario:resume-missing") + )), + "the budgeted history rode in front of the prompt: {events:?}" + ); + // The fallback session's own final text rides the Done like any + // turn's (the driver forwards the result frame's text). + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + result: Some("echoed".into()), + error: None, + session_id: Some("sess-resume-missing".into()), + }) + ); +} + +/// Write one Claude session file into a temporary config root — the +/// records the CLI's own store would hold for `session_id`, with the given +/// top-level message uuids in order (the last one is the session's latest +/// turn). Records that must never count ride after them: a tagged +/// subagent frame and a trailing non-message record. +fn claude_session_store(root: &std::path::Path, session_id: &str, uuids: &[&str]) { + let project = root.join("projects").join("probe-cwd"); + std::fs::create_dir_all(&project).unwrap(); + let mut file = String::new(); + for uuid in uuids { + file.push_str(&format!( + "{{\"type\":\"assistant\",\"uuid\":\"{uuid}\",\"parent_tool_use_id\":null,\ + \"message\":{{\"role\":\"assistant\",\"content\":[{{\"type\":\"text\",\"text\":\"m\"}}]}}}}\n" + )); + } + file.push_str( + "{\"type\":\"assistant\",\"uuid\":\"sub-uuid\",\"parent_tool_use_id\":\"toolu_x\",\ + \"isSidechain\":true,\"message\":{\"role\":\"assistant\",\"content\":[]}}\n", + ); + file.push_str("{\"type\":\"summary\",\"uuid\":\"trailing\",\"summary\":\"s\"}\n"); + std::fs::write(project.join(format!("{session_id}.jsonl")), file).unwrap(); +} + +/// The budgeted transcript the engine attaches as the refused-fork +/// fallback (ticket 01's selection): what a fresh fallback session must +/// receive in front of the prompt, and a native fork must NEVER see. +const FORK_FALLBACK_HISTORY: &str = "Continue this side chat using the following prior transcript as context.\ +\nuser: Remember PINEAPPLE\n\n\n"; + +/// A fork at the session's LATEST turn (ticket 18): the launch resumes the +/// source session with `--fork-session` and NO `--resume-session-at`, the +/// copied history rides the provider (nothing is pasted — the fake fails +/// the turn if any of the fallback reaches it), and the new session id +/// arrives on the init frame with its own turn ref. +#[tokio::test] +async fn a_fork_at_the_latest_turn_resumes_the_source_session_with_fork_session() { + let dir = tempfile::tempdir().unwrap(); + // The boundary uuid is the session's LAST top-level message — the + // latest turn, so fork-session alone carries the whole session. + claude_session_store(dir.path(), "fork-latest-src", &["u-1", "msg-latest-uuid"]); + let harness = ClaudeHarness::new() + .with_executable(fixture_path()) + .with_config_root(dir.path()); + let (controls, _steer, _token) = controls("A"); + let mut req = request("scenario:fork-latest"); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: "fork-latest-src".into(), + last_turn: "msg-latest-uuid".into(), + fallback_history: Some(FORK_FALLBACK_HISTORY.into()), + }); + let events = run_to_end(&harness, req, controls).await; + + // The forked session: a NEW session id read from the init frame. + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::SessionStarted { session_id, .. } if session_id == "sess-fork-latest-new" + )), + "the run continues the forked session: {events:?}" + ); + assert!(events.iter().any(|e| matches!( + e, + AgentEvent::TextDelta { text } if text == "forked latest answer" + ))); + // The turn's ref names the forked session and its last top-level + // message uuid — tagged subagent frames never count. + assert!(events.contains(&AgentEvent::TurnRef { + reference: roboco_proto::ProviderTurnRef { + harness: HarnessId::ClaudeCode, + session_id: "sess-fork-latest-new".into(), + turn_id: "msg-fork-last".into(), + } + })); + // The forked turn's final text rides the Done like any turn's — the + // driver forwards the result frame's text (happy path: Some("done!")). + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + result: Some("forked latest".into()), + error: None, + session_id: Some("sess-fork-latest-new".into()), + }) + ); +} + +/// A fork at an EARLIER turn (ticket 21's boundary): the hidden +/// `--resume-session-at` rides with the boundary uuid (inclusive) beside +/// `--resume` + `--fork-session` — the combination the fallback exists for. +#[tokio::test] +async fn a_fork_at_an_earlier_turn_passes_the_boundary_uuid_to_resume_session_at() { + let dir = tempfile::tempdir().unwrap(); + // The boundary (msg-uuid-7) sits BEFORE the session's latest message + // (msg-newer-uuid) — an earlier turn, so the boundary uuid must cut it. + claude_session_store( + dir.path(), + "fork-earlier-src", + &["msg-uuid-7", "msg-newer-uuid"], + ); + let harness = ClaudeHarness::new() + .with_executable(fixture_path()) + .with_config_root(dir.path()); + let (controls, _steer, _token) = controls("A"); + let mut req = request("scenario:fork-earlier"); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: "fork-earlier-src".into(), + last_turn: "msg-uuid-7".into(), + fallback_history: Some(FORK_FALLBACK_HISTORY.into()), + }); + let events = run_to_end(&harness, req, controls).await; + + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::SessionStarted { session_id, .. } if session_id == "sess-fork-earlier-new" + )), + "the run continues the forked session: {events:?}" + ); + assert!(events.iter().any(|e| matches!( + e, + AgentEvent::TextDelta { text } if text == "forked earlier answer" + ))); + assert!(events.contains(&AgentEvent::TurnRef { + reference: roboco_proto::ProviderTurnRef { + harness: HarnessId::ClaudeCode, + session_id: "sess-fork-earlier-new".into(), + turn_id: "msg-fork-earlier-last".into(), + } + })); + // The forked turn's final text rides the Done like any turn's. + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + result: Some("forked earlier".into()), + error: None, + session_id: Some("sess-fork-earlier-new".into()), + }) + ); +} + +/// A CLI that REJECTS the fork flags — it exits before it ever speaks, so +/// the launch probe reads EOF — retries once as a fresh session with the +/// budgeted transcript in front of the prompt: a fork never fails the send. +#[tokio::test] +async fn a_fork_the_cli_refuses_falls_back_to_a_fresh_session_with_the_budgeted_history() { + let dir = tempfile::tempdir().unwrap(); + // An earlier-turn boundary, so the refused launch is the full flag + // combination (resume + fork-session + resume-session-at). + claude_session_store( + dir.path(), + "fork-refused-src", + &["msg-uuid-7", "msg-newer-uuid"], + ); + let harness = ClaudeHarness::new() + .with_executable(fixture_path()) + .with_config_root(dir.path()); + let (controls, _steer, _token) = controls("A"); + let mut req = request("scenario:fork-refused"); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: "fork-refused-src".into(), + last_turn: "msg-uuid-7".into(), + fallback_history: Some(FORK_FALLBACK_HISTORY.into()), + }); + let events = run_to_end(&harness, req, controls).await; + + // The fallback session served the turn (the fake fails it unless the + // prompt carries the budgeted history AND the user message, with no + // session flags at all). + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::SessionStarted { session_id, .. } if session_id == "sess-fork-fresh" + )), + "the refused fork fell back to a fresh session: {events:?}" + ); + assert!(events.iter().any(|e| matches!( + e, + AgentEvent::TextDelta { text } if text == "fresh fallback answer" + ))); + // The fallback turn's final text rides the Done like any turn's. + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + result: Some("fresh fallback".into()), + error: None, + session_id: Some("sess-fork-fresh".into()), + }) + ); +} + +/// A fork the CLI accepts and then FAILS (B11): a `--resume-session-at` +/// uuid it validates only once the first prompt arrives — the CLI answers +/// initialize, prints its init frame, and errors before any assistant +/// output. That errored first result is the same refusal as the silent +/// exit: the driver retries once as a fresh session with the budgeted +/// transcript pasted in front of the prompt, and the send never fails. +#[tokio::test] +async fn a_fork_that_errors_after_launch_retries_fresh_with_the_budget() { + let dir = tempfile::tempdir().unwrap(); + claude_session_store(dir.path(), "fork-bad-uuid-src", &["msg-uuid-7", "msg-newer-uuid"]); + let harness = ClaudeHarness::new() + .with_executable(fixture_path()) + .with_config_root(dir.path()); + let (controls, _steer, _token) = controls("A"); + let mut req = request("scenario:fork-bad-uuid"); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: "fork-bad-uuid-src".into(), + last_turn: "msg-uuid-7".into(), + fallback_history: Some(FORK_FALLBACK_HISTORY.into()), + }); + let events = run_to_end(&harness, req, controls).await; + + // The fallback session served the turn (the fake fails the retry + // unless the prompt carries the budgeted history AND the user message, + // with no session flags at all). + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::SessionStarted { session_id, .. } if session_id == "sess-fork-bad-fresh" + )), + "the late-refused fork fell back to a fresh session: {events:?}" + ); + assert!(events.iter().any(|e| matches!( + e, + AgentEvent::TextDelta { text } if text == "bad uuid fallback answer" + ))); + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + // The fallback turn's final text rides the Done like any turn's. + result: Some("bad uuid fallback".into()), + error: None, + session_id: Some("sess-fork-bad-fresh".into()), + }) + ); +} + +/// A resume the CLI accepts at launch and then REFUSES once the turn runs +/// (B12 — the account-failover continuation's cross-account resume, the +/// research-03 risk): the errored first result, before any assistant +/// output, retries fresh with the fallback history. The continuation never +/// fails the send. +#[tokio::test] +async fn a_resume_refused_after_launch_retries_fresh_with_the_fallback_history() { + let dir = tempfile::tempdir().unwrap(); + // The session exists in the store, so the pre-launch check passes — + // only the running turn reveals the refusal. + claude_session_store(dir.path(), "resume-refused-src", &["msg-uuid-7"]); + let harness = ClaudeHarness::new() + .with_executable(fixture_path()) + .with_config_root(dir.path()); + let (controls, _steer, _token) = controls("A"); + let mut req = request("scenario:resume-refused"); + req.resume = Some("resume-refused-src".into()); + req.resume_fallback_history = Some(FORK_FALLBACK_HISTORY.into()); + let events = run_to_end(&harness, req, controls).await; + + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::SessionStarted { session_id, .. } if session_id == "sess-resume-fresh" + )), + "the refused resume fell back to a fresh session: {events:?}" + ); + assert!(events.iter().any(|e| matches!( + e, + AgentEvent::TextDelta { text } if text == "resume fallback answer" + ))); + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + // The fallback turn's final text rides the Done like any turn's. + result: Some("resume fallback".into()), + error: None, + session_id: Some("sess-resume-fresh".into()), + }) + ); +} + +/// A source session the CLI's store no longer has: the fork launch is +/// skipped outright — no session flags — and the budgeted transcript rides +/// in front of the prompt on the fresh session. +#[tokio::test] +async fn a_fork_of_a_missing_session_falls_back_to_the_budgeted_history() { + let dir = tempfile::tempdir().unwrap(); + // No session file for fork-missing-src: the store cannot know it. + let harness = ClaudeHarness::new() + .with_executable(fixture_path()) + .with_config_root(dir.path()); + let (controls, _steer, _token) = controls("A"); + let mut req = request("scenario:fork-missing"); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: "fork-missing-src".into(), + last_turn: "msg-uuid-7".into(), + fallback_history: Some(FORK_FALLBACK_HISTORY.into()), + }); + let events = run_to_end(&harness, req, controls).await; + + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::SessionStarted { session_id, .. } + if session_id == "sess-fork-missing-fresh" + )), + "the missing session fell back to a fresh session: {events:?}" + ); + assert!(events.iter().any(|e| matches!( + e, + AgentEvent::TextDelta { text } if text == "missing session fallback answer" + ))); + // The fallback turn's final text rides the Done like any turn's. + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + result: Some("missing session fallback".into()), + error: None, + session_id: Some("sess-fork-missing-fresh".into()), + }) + ); +} + #[tokio::test] async fn happy_path_normalizes_events_and_tags_subagents() { let (controls, _steer, _token) = controls("A"); @@ -119,6 +482,23 @@ async fn happy_path_normalizes_events_and_tags_subagents() { assert_eq!(tools, &vec!["Bash".to_string(), "Read".to_string()]); assert_eq!(session_id, "sess-1"); + // The turn ref (t3-orchestration-v2 ticket 18): the session id plus + // the uuid of the LAST top-level message frame before the result — a + // USER frame here (the tool-result echo closes the turn) — while the + // tagged subagent frames' uuids never count. + assert!(events.contains(&AgentEvent::TurnRef { + reference: roboco_proto::ProviderTurnRef { + harness: HarnessId::ClaudeCode, + session_id: "sess-1".into(), + turn_id: "msg-happy-2".into(), + } + })); + assert_eq!( + events.iter().filter(|e| matches!(e, AgentEvent::TurnRef { .. })).count(), + 1, + "one ref per turn: {events:?}" + ); + assert!(events.contains(&AgentEvent::ReasoningDelta { text: "pondering".into() })); @@ -484,7 +864,7 @@ async fn error_codes_map_to_readable_messages() { let errors: Vec<&str> = events .iter() .filter_map(|e| match e { - AgentEvent::Error { message } => Some(message.as_str()), + AgentEvent::Error { message, .. } => Some(message.as_str()), _ => None, }) .collect(); @@ -499,6 +879,30 @@ async fn error_codes_map_to_readable_messages() { "rejected rate_limit_event not mapped: {errors:?}" ); + // Both limit surfaces carry kind usage-limit: the blocking event with + // the window and the CLI's own reset time, the terse assistant code + // without either. + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::Error { message, kind: Some(AgentErrorKind::UsageLimit { window, resets_at }) } + if message == "Claude 5-hour limit reached — the turn was blocked. Try again after it resets." + && window == "5-hour" + && *resets_at == Some(1_762_675_400_000) + )), + "blocking limit event not classified: {events:?}" + ); + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::Error { message, kind: Some(AgentErrorKind::UsageLimit { window, resets_at }) } + if message == "Claude usage limit reached — try again after the limit resets." + && window == "usage" + && resets_at.is_none() + )), + "assistant rate_limit code not classified: {events:?}" + ); + // Empty `errors` array on the result falls back to subtype wording. assert_eq!( events.last(), @@ -511,6 +915,33 @@ async fn error_codes_map_to_readable_messages() { ); } +#[tokio::test] +async fn an_authentication_failure_is_signed_out() { + let (controls, _steer, _token) = controls("A"); + let events = run_to_end(&harness(), request("scenario:auth"), controls).await; + + // Exactly one error: the message stays human-readable, and the kind says + // sign in again — distinct from waiting out a usage limit. + assert_eq!( + events + .iter() + .filter_map(|e| match e { + AgentEvent::Error { message, kind } => Some((message.clone(), kind.clone())), + _ => None, + }) + .collect::>(), + vec![( + "Authentication failed — sign in to Claude again.".to_owned(), + Some(AgentErrorKind::SignedOut), + )], + "exactly one error, and it is signed-out: {events:?}" + ); + assert!(matches!( + events.last(), + Some(AgentEvent::Done { status: DoneStatus::Errored, .. }) + )); +} + #[tokio::test] async fn missing_binary_is_not_installed() { let harness = ClaudeHarness::new().with_executable("/nonexistent/claude-nowhere"); @@ -579,6 +1010,36 @@ async fn captured_live_background_subagent_frames_replay_correctly() { .collect(); assert_eq!(dones.len(), 2, "eager done + wake done: {events:?}"); + // Turn refs (ticket 18): one per turn — the session id plus the uuid of + // the last top-level message frame before EACH result (the live + // capture's own uuids; the tagged subagent frames streaming between + // the two results never count). Each ref rides ahead of its turn's + // Done so the engine stamps it on the entry that turn streamed into. + let refs: Vec = events + .iter() + .enumerate() + .filter_map(|(i, e)| matches!(e, AgentEvent::TurnRef { .. }).then_some(i)) + .collect(); + assert_eq!(refs.len(), 2, "one ref per turn: {events:?}"); + assert!(events.contains(&AgentEvent::TurnRef { + reference: roboco_proto::ProviderTurnRef { + harness: HarnessId::ClaudeCode, + session_id: "a8fb1e09-b66e-4d02-8759-398c9d784261".into(), + turn_id: "6a1d5b71-3411-44e0-ac9b-742ff98f9aea".into(), + } + })); + assert!(events.contains(&AgentEvent::TurnRef { + reference: roboco_proto::ProviderTurnRef { + harness: HarnessId::ClaudeCode, + session_id: "a8fb1e09-b66e-4d02-8759-398c9d784261".into(), + turn_id: "86d6794c-69b5-4c12-a0bb-b7d3c2f04825".into(), + } + })); + assert!( + refs[0] < dones[0] && dones[0] < refs[1] && refs[1] < dones[1], + "each turn's ref precedes its own done: {events:?}" + ); + // The parent-feed Agent spawn is a plain tool call; the subagent's own // Bash call arrives tagged with the spawning tool-use id, between the // two dones, and never as a bare parent event. diff --git a/crates/harness/tests/codex.rs b/crates/harness/tests/codex.rs index df7885409f..824a9731a3 100644 --- a/crates/harness/tests/codex.rs +++ b/crates/harness/tests/codex.rs @@ -13,8 +13,8 @@ use tokio::sync::{mpsc, oneshot}; use roboco_harness::{ CancellationToken, CodexHarness, Harness, HarnessError, RunControls, SteerMessage, TurnControl}; use roboco_proto::{ - AgentEvent, DoneStatus, HarnessId, ReasoningLevel, RunRequest, SandboxLevel, TodoItem, - TodoStatus, ToolCall, UserInputAnswer, UserInputQuestion, + AgentErrorKind, AgentEvent, DoneStatus, HarnessId, ReasoningLevel, RunRequest, SandboxLevel, + TodoItem, TodoStatus, ToolCall, UserInputAnswer, UserInputQuestion, }; fn fixture_path() -> PathBuf { @@ -104,6 +104,8 @@ fn request(prompt: &str) -> RunRequest { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } @@ -204,6 +206,25 @@ async fn happy_path_maps_deltas_items_usage_and_done() { assert_eq!(cwd, "/tmp"); assert_eq!(session_id, "th-1"); + // The turn ref rides the stream once per turn (ticket 17): the engine + // stamps it on the transcript entry so a fork can continue this thread + // at exactly that turn. + assert_eq!( + events + .iter() + .filter(|e| matches!(e, AgentEvent::TurnRef { .. })) + .count(), + 1, + "turn ref emitted exactly once: {events:?}" + ); + assert!(events.contains(&AgentEvent::TurnRef { + reference: roboco_proto::ProviderTurnRef { + harness: HarnessId::Codex, + session_id: "th-1".into(), + turn_id: "t-1".into(), + } + })); + // Deltas — both wire spellings accepted. assert!(events.contains(&AgentEvent::TextDelta { text: "Hello".into() @@ -353,6 +374,92 @@ async fn happy_path_maps_deltas_items_usage_and_done() { ); } +#[tokio::test] +async fn a_native_fork_forks_the_source_thread_and_pastes_no_history() { + let (controls, _steer, _token) = controls("Yes"); + let mut req = request("scenario:fork-success"); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: "fork-source".into(), + last_turn: "turn-7".into(), + // What the engine would have pasted for a fresh session — the driver + // must NOT use it when the fork succeeds (the fake fails the turn if + // any of it reaches the input). + fallback_history: Some( + "Continue this side chat using the following prior transcript as context.\n\ + \nuser: Remember PINEAPPLE\n\n\n" + .into(), + ), + }); + let events = run_to_end(&harness(), req, controls).await; + // The run continues the FORKED thread (thread/fork's new id), and its + // turn ref names that thread and the turn that actually ran. + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::SessionStarted { session_id, .. } if session_id == "th-forked" + )), + "session continues the forked thread: {events:?}" + ); + assert!(events.contains(&AgentEvent::TurnRef { + reference: roboco_proto::ProviderTurnRef { + harness: HarnessId::Codex, + session_id: "th-forked".into(), + turn_id: "t-f1".into(), + } + })); + assert!(events.iter().any(|e| matches!( + e, + AgentEvent::TextDelta { text } if text == "forked thread answer" + ))); + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some("th-forked".into()), + }) + ); +} + +#[tokio::test] +async fn a_refused_fork_falls_back_to_a_fresh_thread_with_the_budgeted_history() { + let (controls, _steer, _token) = controls("Yes"); + let mut req = request("scenario:fork-refused"); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: "fork-refused".into(), + last_turn: "turn-7".into(), + fallback_history: Some( + "Continue this side chat using the following prior transcript as context.\n\ + \nuser: Remember PINEAPPLE\n\n\n" + .into(), + ), + }); + let events = run_to_end(&harness(), req, controls).await; + // The fresh fallback thread serves the turn (the fake fails it unless + // the input carries the budgeted history AND the user message). + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::SessionStarted { session_id, .. } if session_id == "th-fresh" + )), + "refused fork fell back to a fresh thread: {events:?}" + ); + assert!(events.iter().any(|e| matches!( + e, + AgentEvent::TextDelta { text } if text == "fresh thread answer" + ))); + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some("th-fresh".into()), + }) + ); +} + #[tokio::test] async fn steering_uses_turn_steer_with_expected_turn_id() { let (controls, steer, _token) = controls("Yes"); @@ -615,6 +722,124 @@ async fn turn_failed_maps_to_errored_done() { session_id: Some("th-1".into()), }) ); + // An unclassified failure keeps its historical shape: the text rides + // the Done and no Error event (and no kind) is minted for it. + assert!( + !events.iter().any(|e| matches!(e, AgentEvent::Error { .. })), + "unclassified failure minted an error event: {events:?}" + ); +} + +#[tokio::test] +async fn a_usage_limited_turn_carries_the_reset_from_the_latest_notification() { + let (controls, _steer, _token) = controls("Yes"); + let events = run_to_end(&harness(), request("scenario:usage-limit"), controls).await; + + // The turn-level usage-limit error classifies as a usage limit and + // carries the reset time of the exhausted window from the rate-limit + // notification that preceded it (seconds on the wire, millis in the + // kind). + assert_eq!( + events + .iter() + .filter_map(|e| match e { + AgentEvent::Error { message, kind } => Some((message.clone(), kind.clone())), + _ => None, + }) + .collect::>(), + vec![ + ( + "You have exceeded your usage limit".to_owned(), + Some(AgentErrorKind::UsageLimit { + window: "5-hour".into(), + resets_at: Some(4_102_444_800_000), + }), + ) + ], + "one classified error: {events:?}" + ); + // The failure text rides the Error event; the Done keeps the errored + // status without duplicating the text. + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Errored, + result: None, + error: None, + session_id: Some("th-1".into()), + }) + ); +} + +#[tokio::test] +async fn a_rate_limited_turn_completing_with_the_error_also_classifies() { + let (controls, _steer, _token) = controls("Yes"); + let events = run_to_end(&harness(), request("scenario:rate-limit-completed"), controls).await; + + // turn/completed (rather than turn/failed) carrying a rate-limit error: + // the weekly window is the exhausted one, so its reset is reported. + assert_eq!( + events + .iter() + .filter_map(|e| match e { + AgentEvent::Error { message, kind } => Some((message.clone(), kind.clone())), + _ => None, + }) + .collect::>(), + vec![ + ( + "rate limited, retry later".to_owned(), + Some(AgentErrorKind::UsageLimit { + window: "weekly".into(), + resets_at: Some(4_102_444_801_000), + }), + ) + ], + "one classified error: {events:?}" + ); + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Errored, + result: None, + error: None, + session_id: Some("th-1".into()), + }) + ); +} + +#[tokio::test] +async fn an_auth_failure_is_signed_out() { + let (controls, _steer, _token) = controls("Yes"); + let events = run_to_end(&harness(), request("scenario:auth-fail"), controls).await; + + // A dead login is signed out — distinct from a usage limit, so the user + // knows to sign in rather than wait for a reset. + assert_eq!( + events + .iter() + .filter_map(|e| match e { + AgentEvent::Error { message, kind } => Some((message.clone(), kind.clone())), + _ => None, + }) + .collect::>(), + vec![ + ( + "Token is expired or invalid".to_owned(), + Some(AgentErrorKind::SignedOut), + ) + ], + "one classified error: {events:?}" + ); + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Errored, + result: None, + error: None, + session_id: Some("th-1".into()), + }) + ); } #[tokio::test] @@ -642,6 +867,44 @@ async fn resume_falls_back_to_fresh_thread() { ); } +/// A refused resume with the engine's budgeted fallback (ticket 09's +/// account-failover continuation): the fresh thread's first turn carries +/// the history in front of the user message — the fake fails the turn +/// unless both reach the input. +#[tokio::test] +async fn a_refused_resume_falls_back_with_the_budgeted_history() { + let (controls, _steer, _token) = controls("Yes"); + let mut req = request("scenario:resume-fallback"); + req.resume = Some("resume-fail".into()); + req.resume_fallback_history = Some( + "Continue this side chat using the following prior transcript as context.\n\ + \nuser: Remember PINEAPPLE\n\n\n" + .into(), + ); + let events = run_to_end(&harness(), req, controls).await; + + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::SessionStarted { session_id, .. } if session_id == "th-fresh" + )), + "refused resume fell back to a fresh thread: {events:?}" + ); + assert!(events.iter().any(|e| matches!( + e, + AgentEvent::TextDelta { text } if text == "resumed with history" + ))); + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some("th-fresh".into()), + }) + ); +} + #[tokio::test] async fn resume_reuses_the_existing_thread() { let (controls, _steer, _token) = controls("Yes"); @@ -1567,7 +1830,7 @@ async fn ordinary_followup_cannot_overtake_a_queued_native_command() { assert_eq!(status, DoneStatus::Completed, "{error:?}"); events.push("done".into()); } - AgentEvent::Error { message } => panic!("{message}"), + AgentEvent::Error { message, .. } => panic!("{message}"), _ => {} } } diff --git a/crates/harness/tests/cursor.rs b/crates/harness/tests/cursor.rs index d9a0c0e19b..34063875e8 100644 --- a/crates/harness/tests/cursor.rs +++ b/crates/harness/tests/cursor.rs @@ -43,6 +43,8 @@ fn request(prompt: &str) -> RunRequest { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } diff --git a/crates/harness/tests/fixtures/fake-claude.sh b/crates/harness/tests/fixtures/fake-claude.sh index 8a85d74821..96f8a682f5 100755 --- a/crates/harness/tests/fixtures/fake-claude.sh +++ b/crates/harness/tests/fixtures/fake-claude.sh @@ -8,6 +8,46 @@ # 2.1.286). Driven by crates/harness/tests/claude.rs. read -r first || exit 1 + +# Fork launches (t3-orchestration-v2 ticket 18): the driver resumes the +# source session with --fork-session (and --resume-session-at for a +# boundary at an earlier turn). The source session id picks the behavior: +# acceptance, or a CLI that REJECTS the flags outright — exiting before it +# ever speaks, which is the refusal the driver's launch probe turns into +# the budgeted-paste fallback. Routed off argv because a refusing CLI never +# reads the prompt line. +fork_src= +for arg in "$@"; do + case "$arg" in + --resume=fork-latest-src) fork_src=fork-latest-src ;; + --resume=fork-earlier-src) fork_src=fork-earlier-src ;; + --resume=fork-refused-src) fork_src=fork-refused-src ;; + --resume=fork-bad-uuid-src) fork_src=fork-bad-uuid-src ;; + esac +done +if [ -n "$fork_src" ]; then + case " $* " in + *" --fork-session "*) ;; + *) echo "fork launch without --fork-session" >&2; exit 21 ;; + esac + case "$fork_src" in + fork-refused-src) + # An unknown hidden flag: the CLI exits before answering anything + # (no stdout at all — the probe reads EOF). + echo "error: unknown option '--resume-session-at'" >&2 + exit 2 + ;; + fork-bad-uuid-src) + # The launch carries the boundary uuid the CLI validates only once + # the first prompt arrives (the scenario arm below errors then). + case " $* " in + *"--resume-session-at=msg-uuid-7"*) ;; + *) echo "bad-uuid launch missing resume-session-at: $*" >&2; exit 26 ;; + esac + ;; + esac +fi + # A run opens with an `initialize` control request (perTaskStopAffordance); # the real CLI answers it before the first user line. (Command discovery # sends its own initialize as the only line — matched by its scenario arm @@ -35,6 +75,152 @@ case "$first" in emit '{"type":"result","subtype":"success","result":"echoed","usage":{"input_tokens":1,"output_tokens":1},"session_id":"sess-command-echo"}' ;; +*scenario:resume-missing*) + # The account-failover fallback (ticket 09): the session file is unknown + # to the CLI store, so the driver must start FRESH -- no --resume flag -- + # with the budgeted history in front of the prompt. Seeing --resume here + # fails the run. + for arg in "$@"; do + case "$arg" in + --resume=*) echo "refused resume still passed --resume" >&2; exit 1 ;; + esac + done + content=$(printf '%s\n' "$first" | sed 's/.*"content":"\([^"]*\)".*/\1/') + emit "{\"type\":\"stream_event\",\"event\":{\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"$content\"}}}" + emit '{"type":"result","subtype":"success","result":"echoed","usage":{"input_tokens":1,"output_tokens":1},"session_id":"sess-resume-missing"}' + ;; + +*scenario:fork-latest*) + # A native fork at the LATEST turn (ticket 18): --resume + --fork-session, + # no --resume-session-at, and a BARE prompt — the copied history rides + # the forked session (any pasted transcript fails the turn). + [ -n "$fork_src" ] || { echo "scenario reached without a fork launch" >&2; exit 22; } + case " $* " in + *"--resume-session-at"*) echo "resume-session-at on a latest-turn fork" >&2; exit 22 ;; + esac + for bad in 'PINEAPPLE' ''; do + case "$first" in *"$bad"*) echo "history pasted into a native fork: $bad" >&2; exit 22 ;; esac + done + fid=$(uuid_of "$first") + emit "{\"type\":\"command_lifecycle\",\"command_uuid\":\"$fid\",\"state\":\"queued\"}" + emit "{\"type\":\"command_lifecycle\",\"command_uuid\":\"$fid\",\"state\":\"started\"}" + emit '{"type":"system","subtype":"init","model":"claude-fable-5","tools":[],"cwd":"/tmp","session_id":"sess-fork-latest-new"}' + emit '{"type":"stream_event","parent_tool_use_id":null,"event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"forked latest answer"}}}' + # Top-level frames with uuids (the forked session's own turn): the LAST + # one before the result is the turn ref, tagged frames never count. + emit '{"type":"assistant","uuid":"msg-fork-thinking","parent_tool_use_id":null,"message":{"content":[{"type":"thinking","thinking":"hmm"}]}}' + emit '{"type":"assistant","uuid":"msg-fork-sub","parent_tool_use_id":"sub-x","message":{"content":[{"type":"text","text":"sub interior"}]}}' + emit '{"type":"assistant","uuid":"msg-fork-last","parent_tool_use_id":null,"message":{"content":[{"type":"text","text":"forked latest answer"}]}}' + emit '{"type":"result","subtype":"success","result":"forked latest","errors":[],"usage":{"input_tokens":1,"output_tokens":1},"session_id":"sess-fork-latest-new"}' + emit "{\"type\":\"command_lifecycle\",\"command_uuid\":\"$fid\",\"state\":\"completed\"}" + ;; + +*scenario:fork-earlier*) + # A native fork at an EARLIER turn (ticket 21's boundary): the hidden + # --resume-session-at with the boundary uuid, inclusive, and a bare + # prompt like every native fork. + [ -n "$fork_src" ] || { echo "scenario reached without a fork launch" >&2; exit 23; } + case " $* " in + *"--resume-session-at=msg-uuid-7"*) ;; + *) echo "resume-session-at missing or mismatched: $*" >&2; exit 23 ;; + esac + for bad in 'PINEAPPLE' ''; do + case "$first" in *"$bad"*) echo "history pasted into a native fork: $bad" >&2; exit 23 ;; esac + done + emit '{"type":"system","subtype":"init","model":"claude-fable-5","tools":[],"cwd":"/tmp","session_id":"sess-fork-earlier-new"}' + emit '{"type":"stream_event","parent_tool_use_id":null,"event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"forked earlier answer"}}}' + emit '{"type":"assistant","uuid":"msg-fork-earlier-last","parent_tool_use_id":null,"message":{"content":[{"type":"text","text":"forked earlier answer"}]}}' + emit '{"type":"result","subtype":"success","result":"forked earlier","errors":[],"usage":{"input_tokens":1,"output_tokens":1},"session_id":"sess-fork-earlier-new"}' + ;; + +*scenario:fork-refused*) + # The CLI REFUSED the fork flags (it exited before speaking), so the + # driver fell back to a fresh session: no session flags at all, and the + # budgeted history rides in front of the prompt. + for arg in "$@"; do + case "$arg" in + --resume=*|--fork-session|--resume-session-at=*) echo "fallback still carries session flags: $arg" >&2; exit 24 ;; + esac + done + for want in 'PINEAPPLE' 'scenario:fork-refused'; do + case "$first" in *"$want"*) ;; *) echo "fallback prompt missing: $want" >&2; exit 24 ;; esac + done + emit '{"type":"system","subtype":"init","model":"claude-fable-5","tools":[],"cwd":"/tmp","session_id":"sess-fork-fresh"}' + emit '{"type":"stream_event","parent_tool_use_id":null,"event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"fresh fallback answer"}}}' + emit '{"type":"assistant","uuid":"msg-fork-fresh","parent_tool_use_id":null,"message":{"content":[{"type":"text","text":"fresh fallback answer"}]}}' + emit '{"type":"result","subtype":"success","result":"fresh fallback","errors":[],"usage":{"input_tokens":1,"output_tokens":1},"session_id":"sess-fork-fresh"}' + ;; + +*scenario:fork-bad-uuid*) + # B11: the CLI ACCEPTS the fork flags (it answers initialize and prints + # its init frame) and then fails the resumed session when the first + # prompt arrives — a --resume-session-at uuid it validates late. The + # errored first result, before any assistant output, is the same + # refusal as the silent exit: the driver retries fresh with the + # budgeted history, no session flags at all. + if [ -n "$fork_src" ]; then + emit '{"type":"system","subtype":"init","model":"claude-fable-5","tools":[],"cwd":"/tmp","session_id":"sess-fork-bad-uuid-new"}' + emit '{"type":"result","subtype":"error_during_execution","result":"invalid resume point: msg-uuid-7","errors":["invalid resume point"],"is_error":true,"usage":{"input_tokens":0,"output_tokens":0},"session_id":"sess-fork-bad-uuid-new"}' + else + for arg in "$@"; do + case "$arg" in + --resume=*|--fork-session|--resume-session-at=*) echo "bad-uuid fallback still carries session flags: $arg" >&2; exit 27 ;; + esac + done + for want in 'PINEAPPLE' 'scenario:fork-bad-uuid'; do + case "$first" in *"$want"*) ;; *) echo "bad-uuid fallback prompt missing: $want" >&2; exit 27 ;; esac + done + emit '{"type":"system","subtype":"init","model":"claude-fable-5","tools":[],"cwd":"/tmp","session_id":"sess-fork-bad-fresh"}' + emit '{"type":"stream_event","parent_tool_use_id":null,"event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"bad uuid fallback answer"}}}' + emit '{"type":"assistant","uuid":"msg-fork-bad-fresh","parent_tool_use_id":null,"message":{"content":[{"type":"text","text":"bad uuid fallback answer"}]}}' + emit '{"type":"result","subtype":"success","result":"bad uuid fallback","errors":[],"usage":{"input_tokens":1,"output_tokens":1},"session_id":"sess-fork-bad-fresh"}' + fi + ;; + +*scenario:resume-refused*) + # B12: a resume the CLI accepts at launch and then REFUSES once the turn + # runs (research 03: a cross-account resume the login can no longer + # read). The errored first result, before any assistant output, refuses + # the launch: the driver retries fresh with the fallback history — a + # failover continuation never fails the send. + has_resume=false + for arg in "$@"; do + case "$arg" in + --resume=*) has_resume=true ;; + esac + done + if $has_resume; then + emit '{"type":"system","subtype":"init","model":"claude-fable-5","tools":[],"cwd":"/tmp","session_id":"sess-resume-refused"}' + emit '{"type":"result","subtype":"error_during_execution","result":"session unreadable under this login","errors":["session unreadable"],"is_error":true,"usage":{"input_tokens":0,"output_tokens":0},"session_id":"sess-resume-refused"}' + else + for want in 'PINEAPPLE' 'scenario:resume-refused'; do + case "$first" in *"$want"*) ;; *) echo "resume-refused fallback prompt missing: $want" >&2; exit 28 ;; esac + done + emit '{"type":"system","subtype":"init","model":"claude-fable-5","tools":[],"cwd":"/tmp","session_id":"sess-resume-fresh"}' + emit '{"type":"stream_event","parent_tool_use_id":null,"event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"resume fallback answer"}}}' + emit '{"type":"assistant","uuid":"msg-resume-fresh","parent_tool_use_id":null,"message":{"content":[{"type":"text","text":"resume fallback answer"}]}}' + emit '{"type":"result","subtype":"success","result":"resume fallback","errors":[],"usage":{"input_tokens":1,"output_tokens":1},"session_id":"sess-resume-fresh"}' + fi + ;; + +*scenario:fork-missing*) + # The source session's file is gone from the CLI's store: the driver + # skips the fork launch outright — no session flags, the budgeted history + # in front of the prompt (the fallback is mandatory, never a failed send). + for arg in "$@"; do + case "$arg" in + --resume=*|--fork-session|--resume-session-at=*) echo "missing-session fallback still carries session flags: $arg" >&2; exit 25 ;; + esac + done + for want in 'PINEAPPLE' 'scenario:fork-missing'; do + case "$first" in *"$want"*) ;; *) echo "missing-session prompt missing: $want" >&2; exit 25 ;; esac + done + emit '{"type":"system","subtype":"init","model":"claude-fable-5","tools":[],"cwd":"/tmp","session_id":"sess-fork-missing-fresh"}' + emit '{"type":"stream_event","parent_tool_use_id":null,"event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"missing session fallback answer"}}}' + emit '{"type":"assistant","uuid":"msg-fork-missing","parent_tool_use_id":null,"message":{"content":[{"type":"text","text":"missing session fallback answer"}]}}' + emit '{"type":"result","subtype":"success","result":"missing session fallback","errors":[],"usage":{"input_tokens":1,"output_tokens":1},"session_id":"sess-fork-missing-fresh"}' + ;; + *scenario:title*) tools_off=false system_set=false @@ -68,16 +254,69 @@ case "$first" in emit '{"type":"stream_event","parent_tool_use_id":null,"event":{"type":"content_block_delta","delta":{"type":"thinking_delta","thinking":"pondering"}}}' emit '{"type":"stream_event","parent_tool_use_id":null,"event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"Hello"}}}' # Subagent frames (parent_tool_use_id set): tagged, never in the parent feed. + # The tagged frames carry uuids too — they name the subagent's own nested + # turns and must never become the parent's turn ref. emit '{"type":"stream_event","parent_tool_use_id":"sub-1","event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"SUBAGENT"}}}' - emit '{"type":"assistant","parent_tool_use_id":"sub-1","message":{"content":[{"type":"tool_use","id":"sub-tool","name":"Bash","input":{"command":"echo sub"}}]}}' - emit '{"type":"user","parent_tool_use_id":"sub-1","message":{"content":[{"type":"tool_result","tool_use_id":"sub-tool","is_error":false}]}}' - emit '{"type":"assistant","parent_tool_use_id":null,"message":{"content":[{"type":"text","text":"Hello"},{"type":"tool_use","id":"tool-1","name":"Bash","input":{"command":"ls -la"}},{"type":"tool_use","id":"tool-2","name":"mcp__linear__search","input":{"q":"bug"}}]}}' - emit '{"type":"user","parent_tool_use_id":null,"message":{"content":[{"type":"tool_result","tool_use_id":"tool-1","is_error":false},{"type":"tool_result","tool_use_id":"tool-2","is_error":true}]}}' + emit '{"type":"assistant","uuid":"msg-sub","parent_tool_use_id":"sub-1","message":{"content":[{"type":"tool_use","id":"sub-tool","name":"Bash","input":{"command":"echo sub"}}]}}' + emit '{"type":"user","uuid":"msg-sub-user","parent_tool_use_id":"sub-1","message":{"content":[{"type":"tool_result","tool_use_id":"sub-tool","is_error":false}]}}' + emit '{"type":"assistant","uuid":"msg-happy-1","parent_tool_use_id":null,"message":{"content":[{"type":"text","text":"Hello"},{"type":"tool_use","id":"tool-1","name":"Bash","input":{"command":"ls -la"}},{"type":"tool_use","id":"tool-2","name":"mcp__linear__search","input":{"q":"bug"}}]}}' + emit '{"type":"user","uuid":"msg-happy-2","parent_tool_use_id":null,"message":{"content":[{"type":"tool_result","tool_use_id":"tool-1","is_error":false},{"type":"tool_result","tool_use_id":"tool-2","is_error":true}]}}' # Informational rate-limit status: stays quiet. emit '{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"}}' emit '{"type":"result","subtype":"success","result":"done!","errors":[],"usage":{"input_tokens":10,"output_tokens":20},"session_id":"sess-1","total_cost_usd":0.01}' ;; +*scenario:wake*) + # Eager-done + wake, the live-verified 2.1.228 background-subagent shape: + # the parent turn settles with result #1 while the subagent still runs; + # tagged subagent traffic continues; the CLI then wakes with a second init + # (SAME session id) and settles the wake turn with result #2. + emit '{"type":"system","subtype":"init","model":"claude-fable-5","tools":["Bash"],"cwd":"/tmp","session_id":"sess-wake"}' + emit '{"type":"assistant","parent_tool_use_id":null,"message":{"content":[{"type":"tool_use","id":"toolu_agent","name":"Agent","input":{"description":"background task","run_in_background":true}}]}}' + emit '{"type":"stream_event","parent_tool_use_id":null,"event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"LAUNCHED"}}}' + emit '{"type":"result","subtype":"success","result":"LAUNCHED","errors":[],"usage":{"input_tokens":5,"output_tokens":5},"session_id":"sess-wake"}' + # Background subagent interior, after the eager done. + emit '{"type":"stream_event","parent_tool_use_id":"toolu_agent","event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"sub working"}}}' + emit '{"type":"assistant","parent_tool_use_id":"toolu_agent","message":{"content":[{"type":"tool_use","id":"sub-t1","name":"Bash","input":{"command":"sleep 1"}}]}}' + emit '{"type":"user","parent_tool_use_id":"toolu_agent","message":{"content":[{"type":"tool_result","tool_use_id":"sub-t1","is_error":false}]}}' + # The wake turn: re-init (deduped), untagged output, second result. + emit '{"type":"system","subtype":"init","model":"claude-fable-5","tools":["Bash"],"cwd":"/tmp","session_id":"sess-wake"}' + emit '{"type":"stream_event","parent_tool_use_id":null,"event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"subagent finished"}}}' + emit '{"type":"result","subtype":"success","result":"wrapped up","errors":[],"usage":{"input_tokens":3,"output_tokens":3},"session_id":"sess-wake"}' + ;; + +*scenario:askuser*) + emit '{"type":"system","subtype":"init","model":"claude-fable-5","tools":["Bash"],"cwd":"/tmp","session_id":"sess-ask"}' + # A plain tool permission request: must be auto-allowed. + emit '{"type":"control_request","request_id":"cr-0","request":{"subtype":"can_use_tool","tool_name":"Bash","input":{"command":"ls"}}}' + read -r resp0 || exit 1 + case "$resp0" in + *'"request_id":"cr-0"'*'"behavior":"allow"'*) ;; + *) + emit '{"type":"result","subtype":"error_during_execution","errors":["bash tool was not allowed"],"usage":{"input_tokens":1,"output_tokens":1},"session_id":"sess-ask"}' + exit 0 + ;; + esac + # AskUserQuestion: must be intercepted and answered via updatedInput.answers. + emit '{"type":"control_request","request_id":"cr-1","request":{"subtype":"can_use_tool","tool_name":"AskUserQuestion","input":{"questions":[{"header":"Choice","question":"Pick one","options":["A","B"],"multiSelect":false}]}}}' + read -r resp1 || exit 1 + case "$resp1" in + *'"behavior":"allow"'*) + case "$resp1" in + *'"Pick one":"B"'*) + emit '{"type":"result","subtype":"success","result":"answered","errors":[],"usage":{"input_tokens":1,"output_tokens":1},"session_id":"sess-ask"}' + ;; + *) + emit '{"type":"result","subtype":"error_during_execution","errors":["answers missing from updatedInput"],"usage":{"input_tokens":1,"output_tokens":1},"session_id":"sess-ask"}' + ;; + esac + ;; + *) + emit '{"type":"result","subtype":"error_during_execution","errors":["AskUserQuestion was denied"],"usage":{"input_tokens":1,"output_tokens":1},"session_id":"sess-ask"}' + ;; + esac + ;; + *scenario:steer*) fid=$(uuid_of "$first") emit "{\"type\":\"command_lifecycle\",\"command_uuid\":\"$fid\",\"state\":\"queued\"}" @@ -297,7 +536,12 @@ case "$first" in read -r steer || exit 1 emit '{"type":"stream_event","parent_tool_use_id":null,"event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"absorbed"}}}' emit '{"type":"result","subtype":"success","result":"absorbed","errors":[],"usage":{"input_tokens":1,"output_tokens":1},"session_id":"sess-abs"}' - exec sleep 30 + # Park long enough to prove the quiet fallback released the held result + # (UNREPORTED_SETTLE, 5 s) while the CLI is still alive — exiting sooner + # would release it from EOF instead, and the test's Steered count would + # catch that — but still inside run_to_end's 10 s drain budget, so the + # stream ends without timing the test out. + exec sleep 7 ;; *scenario:stop-in-place*) @@ -355,14 +599,26 @@ case "$first" in *scenario:error*) emit '{"type":"system","subtype":"init","model":"claude-fable-5","tools":[],"cwd":"/tmp","session_id":"sess-err"}' + # Hard-rejected claude.ai usage window, with the CLI's own reset time + # (unix seconds, as the live 2.1.228 capture carries it) — FIRST, matching + # the live capture's order: the event lands before the terse + # assistant-level error code below, so the engine's limit merge (the + # later, windowless `rate_limit` code must not overwrite this window and + # reset) is what the fixture exercises. + emit '{"type":"rate_limit_event","rate_limit_info":{"status":"rejected","rateLimitType":"five_hour","resetsAt":1762675400}}' # Terse assistant-level error code with no content. emit '{"type":"assistant","parent_tool_use_id":null,"message":{"content":[]},"error":"rate_limit"}' - # Hard-rejected claude.ai usage window. - emit '{"type":"rate_limit_event","rate_limit_info":{"status":"rejected","rateLimitType":"five_hour"}}' # Result error with an EMPTY errors array: needs fallback wording. emit '{"type":"result","subtype":"error_max_turns","errors":[],"usage":{"input_tokens":1,"output_tokens":2},"session_id":"sess-err"}' ;; +*scenario:auth*) + # A turn killed by an expired login: signed out, not a limit. + emit '{"type":"system","subtype":"init","model":"claude-fable-5","tools":[],"cwd":"/tmp","session_id":"sess-auth"}' + emit '{"type":"assistant","parent_tool_use_id":null,"message":{"content":[]},"error":"authentication_failed"}' + emit '{"type":"result","subtype":"error_during_execution","errors":[],"usage":{"input_tokens":0,"output_tokens":0},"session_id":"sess-auth"}' + ;; + *) emit '{"type":"result","subtype":"error_during_execution","errors":["unknown scenario"],"usage":{"input_tokens":0,"output_tokens":0},"session_id":"sess-x"}' ;; diff --git a/crates/harness/tests/fixtures/fake-codex.sh b/crates/harness/tests/fixtures/fake-codex.sh index d61a9fc88b..1a8ced0cb1 100755 --- a/crates/harness/tests/fixtures/fake-codex.sh +++ b/crates/harness/tests/fixtures/fake-codex.sh @@ -2,8 +2,8 @@ # Fake Codex app-server for roboco-harness tests. # # Speaks scripted JSON-RPC 2.0 over stdio: initialize handshake, thread -# start/resume, then a scenario picked from the turn/start prompt text. Driven -# by crates/harness/tests/codex.rs. +# start/resume/fork, then a scenario picked from the turn/start prompt text. +# Driven by crates/harness/tests/codex.rs. emit() { printf '%s\n' "$1"; } rid() { printf '%s' "$1" | sed 's/.*"id":\([0-9]*\).*/\1/'; } @@ -49,7 +49,22 @@ if has "$line" '"method":"model/list"'; then emit "{\"id\":$(rid "$line"),\"result\":{\"data\":[{\"id\":\"gpt-5.6-sol\",\"model\":\"gpt-5.6-sol\",\"displayName\":\"GPT-5.6-Sol\",\"description\":\"Reliable agentic workhorse for everyday tasks.\",\"hidden\":false,\"supportedReasoningEfforts\":[{\"reasoningEffort\":\"low\"},{\"reasoningEffort\":\"ultra\"}],\"additionalSpeedTiers\":[],\"serviceTiers\":[],\"defaultServiceTier\":null,\"isDefault\":false}],\"nextCursor\":null}}" exec sleep 30 fi -if has "$line" '"method":"thread/resume"'; then +if has "$line" '"method":"thread/fork"'; then + if has "$line" '"threadId":"fork-source"' && has "$line" '"lastTurnId":"turn-7"'; then + # Native fork (ticket 17): a new thread id; the copied history rides the + # provider, so the first turn's input must carry no transcript. + emit "{\"id\":$(rid "$line"),\"result\":{\"thread\":{\"id\":\"th-forked\"}}}" + elif has "$line" '"threadId":"fork-refused"'; then + # A refused fork: fall back to thread/start; the first turn's input must + # then carry the budgeted history in front of the prompt. + emit "{\"id\":$(rid "$line"),\"error\":{\"code\":-32600,\"message\":\"fork not allowed\"}}" + read -r line || exit 1 + has "$line" '"method":"thread/start"' || exit 1 + emit "{\"id\":$(rid "$line"),\"result\":{\"thread\":{\"id\":\"th-fresh\"}}}" + else + exit 1 + fi +elif has "$line" '"method":"thread/resume"'; then if has "$line" '"threadId":"resume-with-child-v1"'; then emit "{\"id\":$(rid "$line"),\"result\":{\"thread\":{\"id\":\"th-resumed\",\"turns\":[{\"items\":[{\"type\":\"collabAgentToolCall\",\"id\":\"spawn-alpha\",\"tool\":\"spawnAgent\",\"status\":\"completed\",\"receiverThreadIds\":[\"child-alpha\"]}]}]}}}" elif has "$line" '"threadId":"resume-with-child-v2"'; then @@ -360,6 +375,49 @@ case "$turnline" in cat >/dev/null ;; +*scenario:fork-success*) + # The native fork delivered: the turn runs on the forked thread, its input + # is the bare prompt (no transcript block, no copied history). + has "$turnline" '"threadId":"th-forked"' || { fail_turn "$tid" "turn not on the forked thread"; exit 0; } + for bad in '' 'PINEAPPLE'; do + if has "$turnline" "$bad"; then + fail_turn "$tid" "history pasted into a native fork: $bad"; exit 0 + fi + done + has "$turnline" 'scenario:fork-success' || { fail_turn "$tid" "prompt missing"; exit 0; } + emit "{\"id\":$tid,\"result\":{\"turn\":{\"id\":\"t-f1\"}}}" + emit '{"method":"turn/started","params":{"threadId":"th-forked","turn":{"id":"t-f1"}}}' + emit '{"method":"item/agentMessage/delta","params":{"threadId":"th-forked","itemId":"m1","delta":"forked thread answer"}}' + emit '{"method":"turn/completed","params":{"threadId":"th-forked","turn":{"id":"t-f1","status":"completed"}}}' + ;; + +*scenario:fork-refused*) + # The refused fork fell back to a fresh thread: the input must carry the + # budgeted history in front of the user message. + has "$turnline" '"threadId":"th-fresh"' || { fail_turn "$tid" "turn not on the fresh thread"; exit 0; } + for want in 'PINEAPPLE' 'scenario:fork-refused'; do + has "$turnline" "$want" || { fail_turn "$tid" "fallback prompt missing: $want"; exit 0; } + done + emit "{\"id\":$tid,\"result\":{\"turn\":{\"id\":\"t-f2\"}}}" + emit '{"method":"turn/started","params":{"threadId":"th-fresh","turn":{"id":"t-f2"}}}' + emit '{"method":"item/agentMessage/delta","params":{"threadId":"th-fresh","itemId":"m1","delta":"fresh thread answer"}}' + emit '{"method":"turn/completed","params":{"threadId":"th-fresh","turn":{"id":"t-f2","status":"completed"}}}' + ;; + +*scenario:resume-fallback*) + # A REFUSED RESUME (ticket 09's account-failover continuation) fell back + # to a fresh thread: the input must carry the engine's budgeted history + # in front of the user message, exactly like the refused fork. + has "$turnline" '"threadId":"th-fresh"' || { fail_turn "$tid" "turn not on the fresh thread"; exit 0; } + for want in 'PINEAPPLE' 'scenario:resume-fallback'; do + has "$turnline" "$want" || { fail_turn "$tid" "fallback prompt missing: $want"; exit 0; } + done + emit "{\"id\":$tid,\"result\":{\"turn\":{\"id\":\"t-rf\"}}}" + emit '{"method":"turn/started","params":{"threadId":"th-fresh","turn":{"id":"t-rf"}}}' + emit '{"method":"item/agentMessage/delta","params":{"threadId":"th-fresh","itemId":"m1","delta":"resumed with history"}}' + emit '{"method":"turn/completed","params":{"threadId":"th-fresh","turn":{"id":"t-rf","status":"completed"}}}' + ;; + # NOTE: steer-race before steer — `case` takes the first matching glob. *scenario:steer-race*) emit "{\"id\":$tid,\"result\":{\"turn\":{\"id\":\"t-1\"}}}" @@ -513,6 +571,32 @@ case "$turnline" in emit '{"method":"turn/failed","params":{"turn":{"id":"t-1","error":{"message":"boom"}}}}' ;; +*scenario:usage-limit*) + # A turn killed by the account's usage limit. The rate-limit snapshot + # (pushed before the failure) carries the reset the error itself omits; + # the 5-hour window is exhausted, the weekly one is not. + emit "{\"id\":$tid,\"result\":{\"turn\":{\"id\":\"t-1\"}}}" + emit '{"method":"turn/started","params":{"turn":{"id":"t-1"}}}' + emit '{"method":"account/rateLimits/updated","params":{"rateLimits":{"primary":{"usedPercent":100,"windowDurationMins":300,"resetsAt":4102444800},"secondary":{"usedPercent":61,"windowDurationMins":10080,"resetsAt":4102444801}}}}' + emit '{"method":"turn/failed","params":{"turn":{"id":"t-1","error":{"message":"You have exceeded your usage limit","codexErrorInfo":"usageLimitExceeded"}}}}' + ;; + +*scenario:rate-limit-completed*) + # Same classification on turn/completed (status failed + error), with the + # weekly window as the exhausted one and no primary window reported. + emit "{\"id\":$tid,\"result\":{\"turn\":{\"id\":\"t-1\"}}}" + emit '{"method":"turn/started","params":{"turn":{"id":"t-1"}}}' + emit '{"method":"account/rateLimits/updated","params":{"rateLimits":{"secondary":{"usedPercent":100,"windowDurationMins":10080,"resetsAt":4102444801}}}}' + emit '{"method":"turn/completed","params":{"turn":{"id":"t-1","status":"failed","error":{"message":"rate limited, retry later","codexErrorInfo":"rateLimitExceeded"}}}}' + ;; + +*scenario:auth-fail*) + # A turn killed by an expired login: signed out, not a limit. + emit "{\"id\":$tid,\"result\":{\"turn\":{\"id\":\"t-1\"}}}" + emit '{"method":"turn/started","params":{"turn":{"id":"t-1"}}}' + emit '{"method":"turn/failed","params":{"turn":{"id":"t-1","error":{"message":"Token is expired or invalid","codexErrorInfo":"unauthorized"}}}}' + ;; + *scenario:resumed-child*) emit "{\"id\":$tid,\"result\":{\"turn\":{\"id\":\"t-2\"}}}" emit '{"method":"turn/started","params":{"threadId":"child-alpha","turn":{"id":"alpha-resumed"}}}' diff --git a/crates/harness/tests/fixtures/fake-pi-rpc.sh b/crates/harness/tests/fixtures/fake-pi-rpc.sh index 32918a096b..adaffc37d5 100755 --- a/crates/harness/tests/fixtures/fake-pi-rpc.sh +++ b/crates/harness/tests/fixtures/fake-pi-rpc.sh @@ -3,11 +3,23 @@ # covering the pi first-party RPC protocol surface — id-correlated command # responses, every event type the native driver maps, stdout noise/CRLF # tolerance, turn errors, auto-retry, compaction, extension UI dialogs, and -# the steering queue. Replaces fake-pi-acp.sh for the native driver. +# the steering queue — plus the prompt-delivery hardening surface +# (orchestration v2, ticket 02): prompt dispositions (started / queued / +# handled / none for old pi), a rejection while pi streams, a slow +# preflight, the settle probe (busy re-arm, failure, the settle race) and +# clear_queue strand recovery — and the mid-turn steering surface +# (orchestration v2, ticket 03): set_steering_mode all at session start, +# steer bursts delivered together in order, images riding a steer, and +# clear_queue-before-abort when a Stop finds steers pi never consumed — +# and the native fork surface (orchestration v2, ticket 19): the session +# entry tree behind get_entries (the driver walks it after each settle to +# record provider turn refs) and the `--fork` fork process (the source +# copy plus the exclusive fork RPC when the boundary is not the latest +# turn). # NOTE: the python below is embedded in a sh single-quoted string - it must # never contain an apostrophe (a stray one splits the script into argv). exec python3 -u -c ' -import json, os, signal, subprocess, sys, time +import base64, json, os, signal, subprocess, sys, time def emit(value): # CRLF on purpose: the driver must tolerate it (371d79b1 lessons). @@ -28,6 +40,7 @@ def reject(req, message): def user_message(text): emit({"type": "message_start", "message": {"role": "user", "content": [{"type": "text", "text": text}]}}) emit({"type": "message_end", "message": {"role": "user", "content": [{"type": "text", "text": text}]}}) + append_entry("user") def assistant_chunks(chunks, thinking=False, stop="stop", error=None): emit({"type": "message_start", "message": {"role": "assistant", "content": []}}) @@ -40,6 +53,7 @@ def assistant_chunks(chunks, thinking=False, stop="stop", error=None): if error: message["errorMessage"] = error emit({"type": "message_end", "message": message}) + append_entry("assistant") def turn_and_settle(reply=None, thinking=False, stop="stop", error=None): emit({"type": "turn_start"}) @@ -99,19 +113,95 @@ else: assert "--extension" not in argv, argv session_file = None if "--no-session" in argv else os.path.join(os.getcwd(), ".pi-fixture-session.jsonl") +if resumed_session is not None: + # get_state reports the file the process was resumed with (pi answers + # with the session it actually holds), so a forked chat that resumes + # the copy keeps reporting THAT file as its session. + session_file = resumed_session +fork_source = None +if "--fork" in argv: + # The native-fork launch (ticket 19): a short-lived process that copies + # the source session and re-roots it at the boundary. It never runs a + # prompt, never resumes, and never loads the MCP bridge (--fork cannot + # combine with --session / --no-session anyway). + fork_source = argv[argv.index("--fork") + 1] + assert "--session" not in argv and "--no-session" not in argv, argv + assert os.path.isfile(fork_source), "fork source missing: " + fork_source + session_file = os.path.join(os.getcwd(), ".pi-fixture-forked.jsonl") if session_file is not None: with open(session_file, "a") as handle: # exists so the driver resume check passes handle.write("fixture\n") session_id = "pi-fixture-session" thinking_level_seen = None set_model_seen = None +steering_mode_seen = None steering_queue = [] +# pi isStreaming flag, scripted by the scenarios that need a run in +# flight (an extension-initiated run). +streaming = False + +# --- session entry tree (ticket 19) --------------------------------------- +# Every user and assistant message appends a {type: message, id, parentId} +# entry so get_entries can serve the tree the driver walks after each +# settle for provider turn refs: the leaf chain u1 -> a1 -> u2 ... with the +# walk bounded by the `since` cursor the driver passes. +user_seq = 0 +assistant_seq = 0 +leaf_id = None +entries = [] + +def append_entry(role): + global user_seq, assistant_seq, leaf_id + # Ids number per ROLE, the FORK_TREE spelling (u1, a1, u2, a2): the + # turn-ref tests read them as the Nth user entry of the session, so a + # shared counter minting a2/u3 for the second exchange would misname + # every ref after the first turn. + if role == "user": + user_seq += 1 + entry_id = "u%d" % user_seq + else: + assistant_seq += 1 + entry_id = "a%d" % assistant_seq + entries.append({ + "type": "message", + "id": entry_id, + "parentId": leaf_id, + "message": {"role": role}, + }) + leaf_id = entry_id + +def entries_data(command): + # The get_entries payload: append order, the entries strictly after + # `since`, and the current leaf (None for an empty session). An unknown + # `since` cursor fails (pi rejects it; the driver falls back to a full + # listing). + since = command.get("since") + if since is None: + return {"entries": list(entries), "leafId": leaf_id} + ids = [entry["id"] for entry in entries] + if since not in ids: + return None + return {"entries": entries[ids.index(since) + 1:], "leafId": leaf_id} + +# The fork source copied history (ticket 19): three turns. The boundary +# tests exercise both "the latest turn" (last_turn u3: no RPC fork — the +# copy already ends there) and "an earlier turn" (last_turn u2 / u1: the +# exclusive fork RPC at the NEXT turn user entry, u3 / u2). +FORK_TREE = [ + {"type": "message", "id": "u1", "parentId": None, "message": {"role": "user"}}, + {"type": "message", "id": "a1", "parentId": "u1", "message": {"role": "assistant"}}, + {"type": "message", "id": "u2", "parentId": "a1", "message": {"role": "user"}}, + {"type": "message", "id": "a2", "parentId": "u2", "message": {"role": "assistant"}}, + {"type": "message", "id": "u3", "parentId": "a2", "message": {"role": "user"}}, + {"type": "message", "id": "a3", "parentId": "u3", "message": {"role": "assistant"}}, +] + def state_data(): return { "model": {"provider": "mock", "id": "mock/model", "name": "Mock", "contextWindow": 200000}, "thinkingLevel": thinking_level_seen or "medium", - "isStreaming": False, "isCompacting": False, + "isStreaming": streaming, "isCompacting": False, "steeringMode": "one-at-a-time", "followUpMode": "one-at-a-time", "sessionFile": session_file, "sessionId": session_id, "sessionName": "fixture", "autoCompactionEnabled": True, @@ -166,18 +256,70 @@ def dialog_flow(): turn_and_settle("dialogs done") def run_prompt(req): + global streaming text = req["message"] if text == "require-resume": assert resumed_session == session_file, f"engine did not resume: {resumed_session}" if text.startswith("You generate session titles"): assert title_mode, "title instructions must only run in the title sandbox" + response(req, {"disposition": "started"}) user_message(text) turn_and_settle("Fix Login Flow") return if text == "reject": reject(req, "A prompt is already running (busy)") return - response(req) # preflight acceptance + if text == "handled": + # pi runs the extension command itself: accepted, handled — no run + # and no agent_settled ever follow. + response(req, {"disposition": "handled"}) + return + if text == "old-pi-handled": + # pi < 0.99: no disposition field, and the command needed no run. + response(req) + return + if text == "old-pi": + # pi < 0.99: no disposition field — the driver infers whether a run + # started from the settle probe. + response(req) + user_message(text) + turn_and_settle("reply:" + text) + return + if text == "old-pi-stop": + # B3: pi < 0.99 (no disposition) arms the settle probe while the run + # is still live. A Stop then must NOT take the already-settled + # shortcut: it clears the queue and aborts like any live turn. The + # abort is recorded in the session file so the test can see it. + streaming = True + response(req) + user_message(text) + emit({"type": "agent_start"}) + emit({"type": "turn_start"}) + assistant_chunks(["working"]) + while True: + command = read_command() + kind = command["type"] + if kind == "get_state": + response(command, state_data()) + elif kind == "get_session_stats": + response(command, stats_data()) + elif kind == "clear_queue": + response(command, {"steering": [], "followUp": []}) + elif kind == "abort": + with open(session_file, "a") as handle: + handle.write("abort\n") + streaming = False + emit({"type": "agent_settled"}) + response(command) + return + else: + reject(command, "unexpected old-pi-stop command: " + kind) + if text == "slow-preflight": + # pi preflight (input handlers, before_agent_start hooks, the + # Roboco MCP bridge, compaction) runs longer than any fixed + # acceptance budget: the driver must wait, not time the run out. + time.sleep(31) + response(req, {"disposition": "started"}) # preflight acceptance if text == "crash": assistant_chunks(["partial text"]) @@ -194,11 +336,13 @@ def run_prompt(req): if text == "idle-crash": user_message(text) turn_and_settle("reply:" + text) - # Answer the driver post-settle stats poll, then exit while the + # Answer the driver post-settle probe + stats, then exit while the # session is parked: the next dispatch must resume by --session. while True: followup = read_command() - if followup["type"] == "get_session_stats": + if followup["type"] == "get_state": + response(followup, state_data()) + elif followup["type"] == "get_session_stats": response(followup, stats_data()) break time.sleep(0.1) @@ -283,30 +427,127 @@ def run_prompt(req): emit({"type": "turn_start"}) assistant_chunks(["first part "]) steer = read_command() - assert steer["type"] == "steer" and steer["message"] == "steered text", steer + # A mid-run steer is the same prompt with steer behavior: pi queues + # it while streaming. + assert steer["type"] == "prompt" and steer.get("streamingBehavior") == "steer", steer + assert steer["message"] == "steered text", steer steering_queue.append(steer["message"]) + # pi order: the queue_update precedes the acknowledgement. emit({"type": "queue_update", "steering": list(steering_queue), "followUp": []}) - response(steer) + response(steer, {"disposition": "queued"}) + # The current assistant turn keeps running a while before the + # boundary delivers the steer (realistic pacing; also widens the + # window engine tests use to observe the steer landing mid-run). + time.sleep(0.4) emit({"type": "turn_end", "message": {"role": "assistant", "content": [], "stopReason": "stop"}, "toolResults": []}) # pi delivers the steer at the boundary: queue drains, then the user - # message arrives, then the reply to it. + # message arrives (the consumed signal), then the reply to it. steering_queue.remove(steer["message"]) emit({"type": "queue_update", "steering": [], "followUp": []}) user_message(steer["message"]) turn_and_settle("reply-to-steer") return + if text == "steer-burst": + # Several quick steers (steering mode all): pi queues each while + # streaming and delivers them TOGETHER at the next step boundary, + # in send order — none dropped, none reordered. + user_message(text) + emit({"type": "turn_start"}) + assistant_chunks(["first part "]) + steers = [] + for _ in range(3): + steer = read_command() + assert steer["type"] == "prompt" and steer.get("streamingBehavior") == "steer", steer + steers.append(steer["message"]) + steering_queue.append(steer["message"]) + emit({"type": "queue_update", "steering": list(steering_queue), "followUp": []}) + response(steer, {"disposition": "queued"}) + assert steers == ["burst one", "burst two", "burst three"], steers + emit({"type": "turn_end", "message": {"role": "assistant", "content": [], "stopReason": "stop"}, "toolResults": []}) + del steering_queue[:] + emit({"type": "queue_update", "steering": [], "followUp": []}) + for message in steers: + user_message(message) + turn_and_settle("reply-to-burst") + return + if text == "steer-image": + # A steer carries its attached images: the prompt command must ride + # the same inline ImageContent blocks a turn-start prompt does. + user_message(text) + emit({"type": "turn_start"}) + assistant_chunks(["first part "]) + steer = read_command() + assert steer["type"] == "prompt" and steer.get("streamingBehavior") == "steer", steer + images = steer.get("images") + assert isinstance(images, list) and len(images) == 1, steer + assert images[0]["type"] == "image", images + assert images[0]["mimeType"] == "image/png", images + expected = b"\x89PNG\r\n\x1a\n" + b"roboco steer image" + assert base64.b64decode(images[0]["data"]) == expected, images + steering_queue.append(steer["message"]) + emit({"type": "queue_update", "steering": list(steering_queue), "followUp": []}) + response(steer, {"disposition": "queued"}) + emit({"type": "turn_end", "message": {"role": "assistant", "content": [], "stopReason": "stop"}, "toolResults": []}) + del steering_queue[:] + emit({"type": "queue_update", "steering": [], "followUp": []}) + user_message(steer["message"]) + turn_and_settle("reply-to-image-steer") + return + if text == "stop-clears-queue": + # A steer accepted into pi queue but never consumed when Stop lands: + # the driver must clear pi queue BEFORE the abort, so the steer is + # cancelled with the turn instead of delivering into the aborted run. + user_message(text) + emit({"type": "turn_start"}) + assistant_chunks(["working"]) + steer = read_command() + assert steer["type"] == "prompt" and steer.get("streamingBehavior") == "steer", steer + steering_queue.append(steer["message"]) + emit({"type": "queue_update", "steering": list(steering_queue), "followUp": []}) + response(steer, {"disposition": "queued"}) + # The marker: the steer is accepted (pi queue holds it) and the run + # is still streaming — this is the Stop window. + assistant_chunks(["steer queued"]) + cleared = read_command() + assert cleared["type"] == "clear_queue", cleared + assert steering_queue == ["stranded steer"], steering_queue + del steering_queue[:] + response(cleared, {"steering": [], "followUp": []}) + abort = read_command() + assert abort["type"] == "abort", abort + emit({"type": "agent_settled"}) + response(abort) + return + if text == "compaction-recover": + # An overflow error triggers compaction with a retry; the compaction + # succeeds and the turn is retried: the run must end COMPLETED — + # the recovered error no longer rejects the prompt. + user_message(text) + emit({"type": "turn_start"}) + assistant_chunks(["doomed"], stop="error", error="context window overflow") + emit({"type": "turn_end", "message": {"role": "assistant", "content": [], "stopReason": "error"}, "toolResults": []}) + emit({"type": "compaction_start", "reason": "overflow"}) + emit({"type": "compaction_end", "reason": "overflow", + "result": {"summary": "s", "firstKeptEntryId": "e1", "tokensBefore": 10, "estimatedTokensAfter": 5}, + "aborted": False, "willRetry": True}) + emit({"type": "turn_start"}) + assistant_chunks(["recovered"]) + emit({"type": "turn_end", "message": {"role": "assistant", "content": [], "stopReason": "stop"}, "toolResults": []}) + emit({"type": "agent_end", "messages": [], "willRetry": False}) + emit({"type": "agent_settled"}) + return if text == "steer-strand": - # The steer is accepted mid-run but never delivered: the pi queue + # The steer is accepted mid-run but never delivered: pi queue # check happened before it landed, so the run settles with the - # steer stranded. The driver must recover it via clear_queue and - # deliver it as a fresh prompt after the Done. + # steer stranded. The settle probe must clear the queue and hand + # the text back for the next turn. user_message(text) emit({"type": "turn_start"}) assistant_chunks(["reply part"]) steer = read_command() - assert steer["type"] == "steer", steer + assert steer["type"] == "prompt" and steer.get("streamingBehavior") == "steer", steer steering_queue.append(steer["message"]) - response(steer) + response(steer, {"disposition": "queued"}) emit({"type": "turn_end", "message": {"role": "assistant", "content": [], "stopReason": "stop"}, "toolResults": []}) emit({"type": "agent_end", "messages": [], "willRetry": False}) emit({"type": "agent_settled"}) @@ -347,6 +588,15 @@ def run_prompt(req): # must emit a second Done for it. user_message(text) turn_and_settle("reply:" + text) + # Close the prompted turn fully (settle probe + stats) before the + # wake run starts — the fake is single-threaded and cannot answer + # the probe while a scenario runs. + probe = read_command() + assert probe["type"] == "get_state", probe + response(probe, state_data()) + stats = read_command() + assert stats["type"] == "get_session_stats", stats + response(stats, stats_data()) time.sleep(0.3) emit({"type": "agent_start"}) emit({"type": "turn_start"}) @@ -354,10 +604,352 @@ def run_prompt(req): emit({"type": "turn_end", "message": {"role": "assistant", "content": [], "stopReason": "stop"}, "toolResults": []}) emit({"type": "agent_settled"}) return + if text == "reject-while-streaming": + # The prompted turn completes and parks; an extension-initiated run + # starts streaming; the user message sent then is delivered as a + # prompt (steer behavior) and pi REJECTS it. The driver must keep + # the session alive, surface the rejection, return the message to + # its queue, and re-deliver it once the extension run winds down. + user_message(text) + turn_and_settle("reply:" + text) + # Close the prompted turn fully (settle probe + stats) before the + # wake run starts — the fake is single-threaded and cannot answer + # the probe while a scenario sleeps. A probe left unanswered would + # both kill this scenario at the prompt assert (the buffered + # get_state is what read_command returns) and make the driver queue + # the steer behind the in-flight probe instead of delivering it as + # the prompt pi rejects. + probe = read_command() + assert probe["type"] == "get_state", probe + response(probe, state_data()) + stats = read_command() + assert stats["type"] == "get_session_stats", stats + response(stats, stats_data()) + # The turn-ref capture (ticket 19) rides between the settle stats + # and the Done: answer it before the streamed window opens. + refs = read_command() + assert refs["type"] == "get_entries", refs + response(refs, entries_data(refs)) + time.sleep(0.3) # the driver parks: the turn Done is out + streaming = True + emit({"type": "agent_start"}) + emit({"type": "turn_start"}) + assistant_chunks(["working"]) + prompt = read_command() + assert prompt["type"] == "prompt" and prompt.get("streamingBehavior") == "steer", prompt + reject(prompt, "A prompt is already running (busy)") + # The extension run winds down. + emit({"type": "turn_end", "message": {"role": "assistant", "content": [], "stopReason": "stop"}, "toolResults": []}) + emit({"type": "agent_end", "messages": [], "willRetry": False}) + streaming = False + emit({"type": "agent_settled"}) + # Whatever the driver asks next (settle probe, stats) until the + # message re-delivers as the next turn prompt — the loop tolerates + # the driver interleaving its probe and the re-delivery either way + # around the settle. + while True: + command = read_command() + if command["type"] == "get_state": + response(command, state_data()) + elif command["type"] == "get_session_stats": + response(command, stats_data()) + elif command["type"] == "get_entries": + data = entries_data(command) + assert data is not None, command + response(command, data) + elif command["type"] == "prompt": + assert command.get("streamingBehavior") == "steer", command + response(command, {"disposition": "started"}) + user_message(command["message"]) + turn_and_settle("reply:" + command["message"]) + break + else: + reject(command, "unexpected command: " + command["type"]) + return + if text == "late-ack": + # B4: a steer races the settle. The run settles while the steer is + # still in preflight (its acknowledgement has not landed), and the + # settle probe reads pi at REST. The driver must not end the turn + # on that quiescent reading while an input awaits its + # acknowledgement: the turn ends only after the ack lands, the + # acked steer runs, and ITS settle probes clean. + user_message(text) + emit({"type": "turn_start"}) + assistant_chunks(["working"]) + steer = read_command() + assert steer["type"] == "prompt" and steer.get("streamingBehavior") == "steer", steer + # Still in preflight: not in the queue yet (get_state reports no + # pending message), and the run settles with it unacknowledged. + emit({"type": "turn_end", "message": {"role": "assistant", "content": [], "stopReason": "stop"}, "toolResults": []}) + emit({"type": "agent_end", "messages": [], "willRetry": False}) + emit({"type": "agent_settled"}) + responded = False + while True: + command = read_command() + kind = command["type"] + if kind == "get_state": + response(command, state_data()) + elif kind == "get_session_stats": + response(command, stats_data()) + if not responded: + # The acknowledgement lands late, after the settle + # probe already read pi at rest. + responded = True + response(steer, {"disposition": "started"}) + user_message(steer["message"]) + turn_and_settle("reply-to-late-steer") + elif kind == "get_entries": + data = entries_data(command) + assert data is not None, command + response(command, data) + elif kind == "clear_queue": + drained = {"steering": list(steering_queue), "followUp": []} + del steering_queue[:] + response(command, drained) + else: + reject(command, "unexpected late-ack command: " + kind) + return + if text == "steer-burst-mixed": + # B5: a burst of steers where the MIDDLE one carries images: the + # driver must deliver the burst in mailbox order — the image steer + # cannot be overtaken on the wire by the text steer queued after + # it, so pi receives exactly [text, image, text]. + user_message(text) + emit({"type": "turn_start"}) + assistant_chunks(["first part "]) + steers = [] + for wants_image in (False, True, False): + steer = read_command() + assert steer["type"] == "prompt" and steer.get("streamingBehavior") == "steer", steer + if wants_image: + images = steer.get("images") + assert isinstance(images, list) and len(images) == 1, steer + expected = b"\x89PNG\r\n\x1a\n" + b"roboco burst image" + assert base64.b64decode(images[0]["data"]) == expected, images + else: + assert "images" not in steer, steer + steers.append(steer["message"]) + steering_queue.append(steer["message"]) + emit({"type": "queue_update", "steering": list(steering_queue), "followUp": []}) + response(steer, {"disposition": "queued"}) + assert steers == ["mixed one", "mixed two", "mixed three"], steers + emit({"type": "turn_end", "message": {"role": "assistant", "content": [], "stopReason": "stop"}, "toolResults": []}) + del steering_queue[:] + emit({"type": "queue_update", "steering": [], "followUp": []}) + for message in steers: + user_message(message) + turn_and_settle("reply-to-mixed-burst") + return + if text == "reject-cap": + # B1: pi rejects the parked message on every delivery. The driver + # caps the re-delivery after the retries: the message is handed + # back to the engine as a failed send and the turn ends ERRORED — + # never an endless loop of Error + Completed for a message that + # never ran. + user_message(text) + turn_and_settle("reply:" + text) + # Close the prompted turn fully (probe + stats + refs) before the + # parked message arrives, the way reject-while-streaming does. + probe = read_command() + assert probe["type"] == "get_state", probe + response(probe, state_data()) + stats = read_command() + assert stats["type"] == "get_session_stats", stats + response(stats, stats_data()) + refs = read_command() + assert refs["type"] == "get_entries", refs + response(refs, entries_data(refs)) + while True: + command = read_command() + kind = command["type"] + if kind == "prompt": + assert command.get("streamingBehavior") == "steer", command + reject(command, "A prompt is already running (busy)") + elif kind == "get_state": + response(command, state_data()) + elif kind == "get_session_stats": + response(command, stats_data()) + elif kind == "get_entries": + data = entries_data(command) + assert data is not None, command + response(command, data) + else: + reject(command, "unexpected reject-cap command: " + kind) + if text == "probe-fail-mailbox": + # B2: a message lands while the settle probe is in flight (the + # first answer is held open), so the driver queues it behind the + # turn; the probe then fails out and the turn ends errored. The + # queued message must still run as the next turn: the mailbox + # never wedges behind the dead turn. + user_message(text) + turn_and_settle("reply:" + text) + for attempt in range(3): + probe = read_command() + assert probe["type"] == "get_state", probe + if attempt == 0: + time.sleep(1.0) + reject(probe, "wedged") + while True: + command = read_command() + kind = command["type"] + if kind == "prompt": + assert command.get("streamingBehavior") == "steer", command + response(command, {"disposition": "started"}) + user_message(command["message"]) + turn_and_settle("reply:" + command["message"]) + elif kind == "get_state": + response(command, state_data()) + elif kind == "get_session_stats": + response(command, stats_data()) + elif kind == "get_entries": + data = entries_data(command) + assert data is not None, command + response(command, data) + else: + reject(command, "unexpected probe-fail-mailbox command: " + kind) + if text == "steer-strand-image": + # B6: a steer carrying images strands in pi queue at settle. The + # clear_queue recovery must re-queue the ORIGINAL message — the + # re-delivered prompt carries its images again, never bare text. + user_message(text) + emit({"type": "turn_start"}) + assistant_chunks(["reply part"]) + steer = read_command() + assert steer["type"] == "prompt" and steer.get("streamingBehavior") == "steer", steer + images = steer.get("images") + assert isinstance(images, list) and len(images) == 1, steer + steering_queue.append(steer["message"]) + response(steer, {"disposition": "queued"}) + emit({"type": "turn_end", "message": {"role": "assistant", "content": [], "stopReason": "stop"}, "toolResults": []}) + emit({"type": "agent_end", "messages": [], "willRetry": False}) + emit({"type": "agent_settled"}) + expected = b"\x89PNG\r\n\x1a\n" + b"roboco strand image" + while True: + command = read_command() + kind = command["type"] + if kind == "get_state": + response(command, state_data()) + elif kind == "clear_queue": + drained = {"steering": list(steering_queue), "followUp": []} + del steering_queue[:] + response(command, drained) + elif kind == "get_session_stats": + response(command, stats_data()) + elif kind == "get_entries": + data = entries_data(command) + assert data is not None, command + response(command, data) + elif kind == "prompt": + # The recovered message re-delivers with its images intact. + recovered = command.get("images") + assert isinstance(recovered, list) and len(recovered) == 1, command + assert base64.b64decode(recovered[0]["data"]) == expected, recovered + response(command, {"disposition": "started"}) + user_message(command["message"]) + turn_and_settle("reply-to-recovered-image-steer") + return + else: + reject(command, "unexpected steer-strand-image command: " + kind) + if text == "settle-race": + # The settle probe is in flight when the user message arrives: the + # driver must queue it behind the turn (or join the run) — never + # strand the turn. + user_message(text) + turn_and_settle("reply:" + text) + probe = read_command() + assert probe["type"] == "get_state", probe + # Hold the probe answer open so the raced message lands in the + # window. + time.sleep(1.0) + response(probe, state_data()) + return + if text == "busy-probe": + # The settle probe answers BUSY twice (pi still streaming), then at + # rest: the driver must re-arm after each busy report and end the + # turn only once pi rests. + user_message(text) + turn_and_settle("reply:" + text) + for attempt in range(3): + probe = read_command() + assert probe["type"] == "get_state", probe + if attempt < 2: + response(probe, dict(state_data(), isStreaming=True)) + else: + response(probe, state_data()) + stats = read_command() + assert stats["type"] == "get_session_stats", stats + response(stats, stats_data()) + return + if text == "probe-fail": + # The settle probe fails every time: the turn must end ERRORED + # (visibly, with the reason) — not hang, and not a silent kill. + user_message(text) + turn_and_settle("reply:" + text) + for attempt in range(3): + probe = read_command() + assert probe["type"] == "get_state", probe + reject(probe, "wedged") + return + if text == "fork-turn": + # The forked chat first turn (ticket 19): the main process must + # RESUME the session file the fork process created (never a fresh + # session, never the source), and the prompt must arrive BARE — a + # pasted history in front of it would not match this scenario at + # all, so the default happy path answering a pasted prompt is the + # oracle for the paste. + assert resumed_session is not None, "the forked session was not resumed" + assert resumed_session.endswith(".pi-fixture-forked.jsonl"), resumed_session + user_message(text) + turn_and_settle("reply:" + text) + return + if text.endswith("fork-fallback"): + # The refused/stale fork fallback (ticket 19): a FRESH session (no + # --session) with the budgeted history pasted in front of the + # prompt, so the scenario matches the prompt suffix and the + # PINEAPPLE assert is the oracle that the history actually rode + # along (a bare prompt fails it; codex parity: the fake-codex + # fork-refused arm asserts the same markers). The canned marker + # reply proves the scenario routed here — the default happy path + # would echo the whole pasted message instead. + assert resumed_session is None, "the fallback must start a fresh session" + assert "PINEAPPLE" in text, text + user_message(text) + turn_and_settle("reply:fork-fallback") + return # default happy path user_message(text) turn_and_settle("reply:" + text) +if fork_source is not None: + # fork-mode process: get_state (the copy session file, never the + # source), get_entries (the copied tree), the exclusive fork RPC when + # the boundary is not the latest turn, get_state again, then EOF (the + # driver shuts the process down). Markers land in the copy session + # file so tests can prove which steps ran. A source file named with + # "cancel" or "refuse" scripts the fork RPC failing those two ways. + with open(session_file, "a") as handle: + handle.write("fork-launch:" + fork_source + "\n") + while True: + command = read_command() + kind = command["type"] + if kind == "get_state": + response(command, dict(state_data(), sessionFile=session_file, sessionId="pi-fixture-forked")) + elif kind == "get_entries": + response(command, {"entries": [dict(entry) for entry in FORK_TREE], "leafId": FORK_TREE[-1]["id"]}) + elif kind == "fork": + assert command["entryId"] in ("u2", "u3"), command + with open(session_file, "a") as handle: + handle.write("fork-rpc:" + command["entryId"] + "\n") + base = os.path.basename(fork_source) + if "cancel" in base: + response(command, {"text": "the boundary prompt", "cancelled": True}) + elif "refuse" in base: + reject(command, "fork refused") + else: + response(command, {"text": "the boundary prompt", "cancelled": False}) + else: + reject(command, "unexpected fork command: " + kind) + while True: command = read_command() kind = command["type"] @@ -407,6 +999,27 @@ while True: with open(session_file, "a") as handle: handle.write("thinking:" + command["level"] + "\n") response(command) + elif kind == "set_steering_mode": + # The driver selects mode all at session start unless the user + # configured one. Title runs never steer, so the sandbox must never + # touch the setting. Recorded in the session file so tests can + # prove the command was sent (or not, when configured). + assert not title_mode, "title runs never set a steering mode" + assert command["mode"] == "all", command + steering_mode_seen = command["mode"] + if session_file is not None: + with open(session_file, "a") as handle: + handle.write("steering-mode:" + command["mode"] + "\n") + response(command) + elif kind == "get_entries": + # The settle capture (ticket 19): the driver walks the session + # tree after each settle to name the turn provider ref. An unknown + # `since` cursor is rejected (the driver re-reads the full tree). + data = entries_data(command) + if data is None: + reject(command, "unknown since cursor: " + str(command.get("since"))) + else: + response(command, data) elif kind == "get_session_stats": response(command, stats_data()) elif kind == "clear_queue": diff --git a/crates/harness/tests/managed_install.rs b/crates/harness/tests/managed_install.rs index 9378d875cf..cc7a4db0ae 100644 --- a/crates/harness/tests/managed_install.rs +++ b/crates/harness/tests/managed_install.rs @@ -48,6 +48,8 @@ async fn managed_install_reaches_session_started() { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }; let mut stream = tokio::time::timeout( diff --git a/crates/harness/tests/managed_install_failure.rs b/crates/harness/tests/managed_install_failure.rs index 441a9d9072..713aa9e994 100644 --- a/crates/harness/tests/managed_install_failure.rs +++ b/crates/harness/tests/managed_install_failure.rs @@ -60,6 +60,8 @@ async fn silent_npm_enoent_death_surfaces_decoded_error() { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }; let err = match harness.run(request, controls).await { diff --git a/crates/harness/tests/opencode.rs b/crates/harness/tests/opencode.rs index 2c229736a2..9d73250c60 100644 --- a/crates/harness/tests/opencode.rs +++ b/crates/harness/tests/opencode.rs @@ -16,7 +16,8 @@ use tokio::sync::{broadcast, mpsc, oneshot}; use roboco_harness::{ CancellationToken, Harness, HarnessError, OpencodeHarness, RunControls, SteerMessage, TurnControl}; use roboco_proto::{ - AgentEvent, DoneStatus, ReasoningLevel, RunRequest, SandboxLevel, ToolCall, UserInputAnswer, + AgentEvent, DoneStatus, HarnessId, ProviderFork, ProviderTurnRef, ReasoningLevel, RunRequest, + SandboxLevel, ToolCall, UserInputAnswer, }; // --------------------------------------------------------------------------- @@ -44,10 +45,39 @@ struct FakeOpencode { fail_session_creates: Arc>, /// Answer `POST …/abort` with a 500 (a wedged server). fail_aborts: Arc>, + /// Serve the 2.x `/api` surface instead: version-bearing `/api/info`, + /// `{data}`-wrapped responses, raw (unenveloped) SSE frames. + v2: bool, + /// Per-session message lists the fork's boundary resolution reads + /// (`GET /session/{id}/message`, `GET /api/session/{id}/message`). + /// Unlisted sessions 404 — the fork's "a call fails" fallback. + messages: Arc>>>, + /// Cursor-paginated 2.x message fixtures by session: whole pages, + /// served one `cursor=` step at a time (the pagination path of the + /// fork's boundary resolution). + paged_messages: Arc>>>>, + /// Fork outcomes by SOURCE session id: (status, response). An + /// unconfigured source 404s the fork (the refused-fork fallback). + forks: Arc< + Mutex< + std::collections::HashMap< + String, + (&'static str, Value), + >, + >, + >, } impl FakeOpencode { async fn start() -> Self { + Self::start_proto(false).await + } + + async fn start_v2() -> Self { + Self::start_proto(true).await + } + + async fn start_proto(v2: bool) -> Self { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let base = format!("http://{}", listener.local_addr().unwrap()); let (events, _) = broadcast::channel::<(u64, String)>(256); @@ -64,6 +94,10 @@ impl FakeOpencode { first_prompt_had_subscriber: Arc::new(Mutex::new(None)), fail_session_creates: Arc::new(Mutex::new(0)), fail_aborts: Arc::new(Mutex::new(false)), + v2, + messages: Arc::default(), + paged_messages: Arc::default(), + forks: Arc::default(), }; let accept = fake.clone(); tokio::spawn(async move { @@ -79,7 +113,8 @@ impl FakeOpencode { } /// Push one bus event (the driver accepts both the bare and the - /// `/global/event` envelope; the fake uses the enveloped form). + /// `/global/event` envelope; the fake uses the enveloped form on 1.x + /// and the raw 2.x frame shape on 2.x). fn emit(&self, payload: Value) { if payload["type"] == "session.status" && let Some(id) = payload["properties"]["sessionID"].as_str() @@ -89,10 +124,14 @@ impl FakeOpencode { .unwrap() .insert(id.to_owned(), payload["properties"]["status"].clone()); } - let framed = format!( - "data: {}\n\n", - json!({ "directory": "/", "payload": payload }) - ); + let framed = if self.v2 { + format!("data: {}\n\n", payload) + } else { + format!( + "data: {}\n\n", + json!({ "directory": "/", "payload": payload }) + ) + }; let mut backlog = self.backlog.lock().unwrap(); let seq = backlog.len() as u64; backlog.push((seq, framed.clone())); @@ -103,6 +142,33 @@ impl FakeOpencode { *self.providers.lock().unwrap() = providers; } + /// Install a session's message list (creation order) for the fork's + /// boundary resolution. + fn set_messages(&self, session: &str, messages: Vec) { + self.messages + .lock() + .unwrap() + .insert(session.to_owned(), messages); + } + + /// Install a session's message list as 2.x cursor pages (creation + /// order across the pages). + fn set_messages_paged(&self, session: &str, pages: Vec>) { + self.paged_messages + .lock() + .unwrap() + .insert(session.to_owned(), pages); + } + + /// Install the fork outcome for a source session (status + response + /// body; 1.x returns the session info raw, 2.x wrapped in `{data}`). + fn set_fork(&self, source: &str, status: &'static str, response: Value) { + self.forks + .lock() + .unwrap() + .insert(source.to_owned(), (status, response)); + } + fn posts_to(&self, path: &str) -> Vec { self.posts .lock() @@ -153,9 +219,12 @@ impl FakeOpencode { let mut parts = start.split_whitespace(); let method = parts.next().unwrap_or_default().to_owned(); let target = parts.next().unwrap_or_default().to_owned(); - let path = target.split('?').next().unwrap_or_default().to_owned(); + let (path, query) = match target.split_once('?') { + Some((path, query)) => (path.to_owned(), query.to_owned()), + None => (target, String::new()), + }; - if method == "GET" && path == "/global/event" { + if method == "GET" && (path == "/global/event" || path == "/api/event") { // Subscribe FIRST, then snapshot the backlog: frames landing // in between arrive on both channels and dedupe by sequence. let mut rx = self.events.subscribe(); @@ -190,7 +259,7 @@ impl FakeOpencode { } if method == "POST" { - if path.ends_with("/prompt_async") { + if path.ends_with("/prompt_async") || path.ends_with("/prompt") { let mut first = self.first_prompt_had_subscriber.lock().unwrap(); if first.is_none() { *first = Some(self.events.receiver_count() > 0); @@ -198,7 +267,7 @@ impl FakeOpencode { } self.posts.lock().unwrap().push((path.clone(), body)); } - let (status, payload) = self.route(&method, &path); + let (status, payload) = self.route(&method, &path, &query); let body = payload.to_string(); let resp = format!( "HTTP/1.1 {status}\r\ncontent-type: application/json\r\n\ @@ -211,7 +280,41 @@ impl FakeOpencode { } } - fn route(&self, method: &str, path: &str) -> (&'static str, Value) { + fn route(&self, method: &str, path: &str, query: &str) -> (&'static str, Value) { + if self.v2 { + return self.route_v2(method, path, query); + } + self.route_v1(method, path) + } + + /// The session id segment of a `/…/session/{id}/…` path. + fn session_id_of(path: &str, prefix: &str) -> Option { + let rest = path.strip_prefix(prefix)?; + let id = rest.split('/').next()?; + (!id.is_empty()).then(|| id.to_owned()) + } + + /// The message list for a session's boundary resolution, when the fake + /// has one (unlisted sessions 404 — the fork's resolution fallback). + fn message_list(&self, path: &str, prefix: &str) -> Option> { + let id = Self::session_id_of(path, prefix)?; + self.messages.lock().unwrap().get(&id).cloned() + } + + /// The configured fork outcome for a source session (unconfigured + /// sources 404 — the refused-fork fallback). + fn fork_outcome(&self, path: &str, prefix: &str) -> (&'static str, Value) { + let id = Self::session_id_of(path, prefix); + match id + .as_deref() + .and_then(|id| self.forks.lock().unwrap().get(id).cloned()) + { + Some((status, body)) => (status, body), + None => ("404 Not Found", json!({})), + } + } + + fn route_v1(&self, method: &str, path: &str) -> (&'static str, Value) { match (method, path) { ("GET", "/global/health") => ("200 OK", json!({ "healthy": true })), ("GET", "/provider") => ("200 OK", self.providers.lock().unwrap().clone()), @@ -239,6 +342,19 @@ impl FakeOpencode { Value::Object(self.statuses.lock().unwrap().clone()), ), ("GET", "/session/ses_resume") => ("200 OK", json!({ "id": "ses_resume" })), + // The fork's boundary resolution: a raw array of + // `{info: {id, role}, parts}` entries (the 1.18.31 shape). + ("GET", p) if p.starts_with("/session/") && p.ends_with("/message") => { + match self.message_list(p, "/session/") { + Some(list) => ("200 OK", Value::Array(list)), + None => ("404 Not Found", json!({})), + } + } + // A native fork: `POST /session/{id}/fork {messageID}` answering + // the new session's info (raw, 1.x). + ("POST", p) if p.starts_with("/session/") && p.ends_with("/fork") => { + self.fork_outcome(p, "/session/") + } ("GET", p) if p.starts_with("/session/") => ("404 Not Found", json!({})), ("POST", p) if p.ends_with("/prompt_async") => ("204 No Content", json!({})), ("POST", p) if p.ends_with("/abort") && *self.fail_aborts.lock().unwrap() => ( @@ -252,6 +368,85 @@ impl FakeOpencode { _ => ("404 Not Found", json!({ "missing": path })), } } + + /// The 2.x `/api` surface: version-bearing info endpoints (protocol + /// detection), `{data}`-wrapped responses, and the fork/message routes + /// in their 2.x shapes. + fn route_v2(&self, method: &str, path: &str, query: &str) -> (&'static str, Value) { + match (method, path) { + ("GET", "/api/info") | ("GET", "/api/health") | ("GET", "/api/status") => { + ("200 OK", json!({ "version": "2.0.11", "healthy": true })) + } + ("GET", "/api/model") => ( + "200 OK", + // Non-empty: the catalog-sync retry loop must not stall tests. + json!({ "data": [{ + "providerID": "test-provider", + "id": "muse", + "name": "Muse", + "limit": { "context": 100000 }, + "variants": [], + "enabled": true, + }] }), + ), + ("GET", "/api/command") => { + ("200 OK", json!({ "data": self.commands.lock().unwrap().clone() })) + } + ("POST", "/api/session") => { + let mut fails = self.fail_session_creates.lock().unwrap(); + if *fails > 0 { + *fails -= 1; + ("500 Internal Server Error", json!({ "name": "UnknownError" })) + } else { + ("200 OK", json!({ "data": { "id": "ses_test2" } })) + } + } + ("GET", "/api/session/active") => ("200 OK", json!({ "data": {} })), + // The fork's boundary resolution: `{data, cursor}` pages of + // flat `{id, type}` entries (the 2.0.11 shape). A paged fixture + // answers one page per `cursor=` step; the driver's first page + // rides `order=asc` (no cursor). + ("GET", p) if p.starts_with("/api/session/") && p.ends_with("/message") => { + let session = Self::session_id_of(p, "/api/session/"); + let pages = session + .as_deref() + .and_then(|id| self.paged_messages.lock().unwrap().get(id).cloned()); + if let Some(pages) = pages { + let index = query + .split('&') + .find_map(|kv| kv.strip_prefix("cursor=")) + .and_then(|cursor| cursor.parse::().ok()) + .unwrap_or(0); + let page = pages.get(index).cloned().unwrap_or_default(); + let next = (index + 1 < pages.len()).then(|| (index + 1).to_string()); + let cursor = match next { + Some(next) => json!({ "next": next }), + None => json!({}), + }; + return ("200 OK", json!({ "data": page, "cursor": cursor })); + } + match self.message_list(p, "/api/session/") { + Some(list) => ("200 OK", json!({ "data": list, "cursor": {} })), + None => ("404 Not Found", json!({})), + } + } + // A native fork: `POST /api/session/{id}/fork {before}` + // answering `{data: {id}}`. + ("POST", p) if p.starts_with("/api/session/") && p.ends_with("/fork") => { + self.fork_outcome(p, "/api/session/") + } + ("GET", p) if p.starts_with("/api/session/") => { + let id = Self::session_id_of(p, "/api/session/").unwrap_or_default(); + ("200 OK", json!({ "data": { "id": id } })) + } + ("POST", p) if p.ends_with("/prompt") => ("200 OK", json!({})), + ("POST", p) if p.ends_with("/interrupt") => ("200 OK", json!(true)), + ("POST", p) if p.contains("/permission/") || p.contains("/question/") => { + ("200 OK", json!(true)) + } + _ => ("404 Not Found", json!({ "missing": path })), + } + } } // --------------------------------------------------------------------------- @@ -270,6 +465,8 @@ fn request(prompt: &str) -> RunRequest { auto_approve: true, attachments: Vec::new(), resume: None, + provider_fork: None, + resume_fallback_history: None, worktree: None, mcp: None, } @@ -304,6 +501,23 @@ fn harness(fake: &FakeOpencode) -> OpencodeHarness { OpencodeHarness::new().with_base_url(fake.base.clone()) } +/// A request carrying a native-fork directive (what the engine's first +/// dispatch of a forked chat injects from the recorded intent). +fn fork_request( + prompt: &str, + session_id: &str, + last_turn: &str, + fallback_history: Option<&str>, +) -> RunRequest { + let mut request = request(prompt); + request.provider_fork = Some(ProviderFork { + session_id: session_id.into(), + last_turn: last_turn.into(), + fallback_history: fallback_history.map(str::to_owned), + }); + request +} + /// Emit the standard opening frames of an assistant turn. fn assistant_message(fake: &FakeOpencode, session: &str, message: &str) { fake.emit(json!({ @@ -323,8 +537,66 @@ fn idle(fake: &FakeOpencode, session: &str) { })); } +/// 2.x bus frames: an execution starts, the assistant message is fixed by +/// `session.step.started`, and text streams as text items on it. +fn v2_assistant_message(fake: &FakeOpencode, session: &str, message: &str) { + fake.emit(json!({ + "type": "session.execution.started", + "data": { "sessionID": session }, + })); + fake.emit(json!({ + "type": "session.step.started", + "data": { "sessionID": session, "assistantMessageID": message }, + })); + fake.emit(json!({ + "type": "session.text.started", + "data": { "sessionID": session, "assistantMessageID": message, "ordinal": 0, "text": "" }, + })); +} + +fn v2_text(fake: &FakeOpencode, session: &str, message: &str, text: &str) { + fake.emit(json!({ + "type": "session.text.delta", + "data": { + "sessionID": session, "assistantMessageID": message, "ordinal": 0, + "delta": text, + }, + })); + fake.emit(json!({ + "type": "session.text.ended", + "data": { + "sessionID": session, "assistantMessageID": message, "ordinal": 0, + "text": text, + }, + })); +} + +fn v2_idle(fake: &FakeOpencode, session: &str) { + fake.emit(json!({ + "type": "session.execution.succeeded", + "data": { "sessionID": session }, + })); +} + +/// The next event, passing over the driver's `TurnRef` bookkeeping +/// (t3-orchestration-v2 ticket 20): one rides every turn's stream ahead of +/// its content and says nothing about it, so tests that assert on content +/// order read through this. The turn-ref tests themselves use +/// [`next_event_with_refs`] and assert the refs where they sit. async fn next_event( stream: &mut (impl futures::Stream> + Unpin), +) -> AgentEvent { + loop { + let event = next_event_with_refs(stream).await; + if !matches!(event, AgentEvent::TurnRef { .. }) { + return event; + } + } +} + +/// The raw next event, `TurnRef` included. +async fn next_event_with_refs( + stream: &mut (impl futures::Stream> + Unpin), ) -> AgentEvent { tokio::time::timeout(Duration::from_secs(10), stream.next()) .await @@ -348,13 +620,16 @@ async fn wait_posts(fake: &FakeOpencode, path: &str, n: usize) -> Vec { .unwrap_or_else(|_| panic!("{path} never saw {n} posts")) } -/// Drain until a Done arrives; returns everything seen (Done last). +/// Drain until a Done arrives; returns everything seen (Done last). The +/// refs ride the drained window too — assertions that count them stay +/// honest, and the content-order assertions in this suite key on Done, +/// `any`, or the last element, not on the refs' position. async fn drain_to_done( stream: &mut (impl futures::Stream> + Unpin), ) -> Vec { let mut events = Vec::new(); loop { - let ev = next_event(stream).await; + let ev = next_event_with_refs(stream).await; let done = matches!(&ev, AgentEvent::Done { .. }); events.push(ev); if done { @@ -429,7 +704,9 @@ async fn thinking_streams_and_the_turn_settles_only_on_idle() { })); let text = next_event(&mut stream).await; assert!(matches!(&text, AgentEvent::TextDelta { text } if text == "Hello")); - let quiet = tokio::time::timeout(Duration::from_millis(600), stream.next()).await; + // Nothing but TurnRef bookkeeping may arrive in the quiet gap — the + // skipping reader keeps the future pending while only refs land. + let quiet = tokio::time::timeout(Duration::from_millis(600), next_event(&mut stream)).await; assert!(quiet.is_err(), "nothing may settle a quiet-but-live turn"); idle(&fake, "ses_test"); @@ -506,7 +783,10 @@ async fn foreign_session_idle_never_settles_our_turn() { assistant_message(&fake, "ses_test", "msg_1"); idle(&fake, "ses_OTHER"); - let quiet = tokio::time::timeout(Duration::from_millis(600), stream.next()).await; + // Only the turn's own TurnRef may be queued here (ticket 20) — the + // skipping reader holds the gap open for real traffic, which a + // foreign idle never produces. + let quiet = tokio::time::timeout(Duration::from_millis(600), next_event(&mut stream)).await; assert!(quiet.is_err(), "a foreign session's idle settled our turn"); idle(&fake, "ses_test"); @@ -921,7 +1201,7 @@ async fn provider_retries_surface_and_cap_out() { fake.emit(retry(1)); fake.emit(retry(3)); let ev = next_event(&mut stream).await; - let AgentEvent::Error { message } = &ev else { + let AgentEvent::Error { message, .. } = &ev else { panic!("expected a retry error chip, got {ev:?}"); }; assert!( @@ -932,7 +1212,7 @@ async fn provider_retries_surface_and_cap_out() { fake.emit(retry(8)); let ev = next_event(&mut stream).await; - let AgentEvent::Error { message } = &ev else { + let AgentEvent::Error { message, .. } = &ev else { panic!("expected the give-up chip, got {ev:?}"); }; assert!(message.contains("Giving up"), "{message}"); @@ -971,7 +1251,7 @@ async fn session_error_with_no_content_settles_errored() { let ev = next_event(&mut stream).await; assert!(matches!( &ev, - AgentEvent::Error { message } if message.contains("no credentials") + AgentEvent::Error { message, .. } if message.contains("no credentials") )); // opencode re-emits the same failure with an exception-name prefix and a // stack — that must NOT mint a second chip (field report: every failure @@ -983,7 +1263,7 @@ async fn session_error_with_no_content_settles_errored() { "data": { "message": "ProviderAuthError: no credentials for anthropic\n at stack" }, }}, })); - let quiet = tokio::time::timeout(Duration::from_millis(400), stream.next()).await; + let quiet = tokio::time::timeout(Duration::from_millis(400), next_event(&mut stream)).await; assert!( quiet.is_err(), "duplicate error must not mint a second chip" @@ -1352,3 +1632,518 @@ async fn dollar_selected_skill_uses_opencode_native_command_with_arguments() { idle(&fake, "ses_test"); drain_to_done(&mut stream).await; } + +// --------------------------------------------------------------------------- +// Provider turn refs and native forks (t3-orchestration-v2 ticket 20) +// --------------------------------------------------------------------------- + +/// The budgeted transcript the engine attaches as a fork's fallback — +/// what a refused fork must paste in front of the prompt. +const BUDGETED_HISTORY: &str = "Continue this side chat using the following prior transcript as context.\n\ +\nuser: Remember PINEAPPLE\n\n\n"; + +#[tokio::test] +async fn prompts_carry_a_client_message_id_and_each_turn_gets_its_ref() { + let fake = FakeOpencode::start().await; + let (controls, steer, _token) = controls(); + let mut stream = harness(&fake) + .run(request("hi"), controls) + .await + .expect("run starts"); + + let started = next_event(&mut stream).await; + assert!(matches!( + &started, + AgentEvent::SessionStarted { session_id, .. } if session_id == "ses_test" + )); + let _commands = next_event(&mut stream).await; + + // The prompt travels under a client-chosen message id (the turn's + // provider ref — the anchor a fork cuts the session at). + let prompts = wait_posts(&fake, "/session/ses_test/prompt_async", 1).await; + let first_id = prompts[0]["messageID"].as_str().unwrap().to_owned(); + assert!(first_id.starts_with("msg_"), "{first_id}"); + + // The ref rides the stream BEFORE the turn's content: the engine + // stamps it on the entry the turn is about to stream. + let turn_ref = next_event_with_refs(&mut stream).await; + assert_eq!( + turn_ref, + AgentEvent::TurnRef { + reference: ProviderTurnRef { + harness: HarnessId::Opencode, + session_id: "ses_test".into(), + turn_id: first_id.clone(), + } + } + ); + + assistant_message(&fake, "ses_test", "msg_1"); + fake.emit(json!({ + "type": "message.part.updated", + "properties": { "part": { + "id": "prt_t", "messageID": "msg_1", "sessionID": "ses_test", + "type": "text", "text": "Hello", + }}, + })); + let text = next_event(&mut stream).await; + assert!(matches!(&text, AgentEvent::TextDelta { text } if text == "Hello")); + idle(&fake, "ses_test"); + let first = drain_to_done(&mut stream).await; + assert_eq!( + first + .iter() + .filter(|ev| matches!(ev, AgentEvent::TurnRef { .. })) + .count(), + 0, + "the turn's one ref was delivered before its content: {first:?}" + ); + + // A steer at the turn boundary starts the next turn: a NEW id, a new + // ref — the turns stay separately addressable for a fork boundary. + steer + .send(SteerMessage { + prompt: "redirect".into(), + message_id: None, + attachments: Vec::new(), + config: None, + }) + .await + .expect("steer queued"); + let steered = next_event(&mut stream).await; + assert!(matches!(&steered, AgentEvent::Steered { .. })); + let prompts = wait_posts(&fake, "/session/ses_test/prompt_async", 2).await; + let second_id = prompts[1]["messageID"].as_str().unwrap().to_owned(); + assert!(second_id.starts_with("msg_"), "{second_id}"); + assert_ne!(second_id, first_id); + let turn_ref = next_event_with_refs(&mut stream).await; + assert_eq!( + turn_ref, + AgentEvent::TurnRef { + reference: ProviderTurnRef { + harness: HarnessId::Opencode, + session_id: "ses_test".into(), + turn_id: second_id, + } + } + ); + + assistant_message(&fake, "ses_test", "msg_2"); + fake.emit(json!({ + "type": "message.part.updated", + "properties": { "part": { + "id": "prt_t2", "messageID": "msg_2", "sessionID": "ses_test", + "type": "text", "text": "Redirected", + }}, + })); + let text = next_event(&mut stream).await; + assert!(matches!(&text, AgentEvent::TextDelta { text } if text == "Redirected")); + idle(&fake, "ses_test"); + let second = drain_to_done(&mut stream).await; + assert_eq!( + second + .iter() + .filter(|ev| matches!(ev, AgentEvent::TurnRef { .. })) + .count(), + 0, + "the steer turn's one ref was delivered before its content: {second:?}" + ); +} + +#[tokio::test] +async fn v2_prompts_carry_a_client_message_id_and_emit_the_turn_ref() { + let fake = FakeOpencode::start_v2().await; + let (controls, _steer, _token) = controls(); + let mut stream = harness(&fake) + .run(request("hi"), controls) + .await + .expect("run starts"); + + let started = next_event(&mut stream).await; + assert!(matches!( + &started, + AgentEvent::SessionStarted { session_id, .. } if session_id == "ses_test2" + )); + let _commands = next_event(&mut stream).await; + + // 2.x spells the client-chosen id `id` (must be `msg_`-prefixed). + let prompts = wait_posts(&fake, "/api/session/ses_test2/prompt", 1).await; + let message_id = prompts[0]["id"].as_str().unwrap().to_owned(); + assert!(message_id.starts_with("msg_"), "{message_id}"); + + let turn_ref = next_event_with_refs(&mut stream).await; + assert_eq!( + turn_ref, + AgentEvent::TurnRef { + reference: ProviderTurnRef { + harness: HarnessId::Opencode, + session_id: "ses_test2".into(), + turn_id: message_id, + } + } + ); + + v2_assistant_message(&fake, "ses_test2", "amsg_1"); + v2_text(&fake, "ses_test2", "amsg_1", "Hello"); + let text = next_event(&mut stream).await; + assert!(matches!(&text, AgentEvent::TextDelta { text } if text == "Hello")); + v2_idle(&fake, "ses_test2"); + let events = drain_to_done(&mut stream).await; + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some("ses_test2".into()), + }) + ); +} + +#[tokio::test] +async fn a_native_fork_cuts_the_source_session_at_the_next_turn() { + // OpenCode's fork boundary is exclusive: keeping the boundary turn + // means cutting before the NEXT turn's user message. + let fake = FakeOpencode::start().await; + fake.set_messages( + "ses_src", + vec![ + json!({ "info": { "id": "msg_a", "role": "user", "sessionID": "ses_src" } }), + json!({ "info": { "id": "msg_a_reply", "role": "assistant", "sessionID": "ses_src" } }), + json!({ "info": { "id": "msg_b", "role": "user", "sessionID": "ses_src" } }), + json!({ "info": { "id": "msg_b_reply", "role": "assistant", "sessionID": "ses_src" } }), + ], + ); + fake.set_fork("ses_src", "200 OK", json!({ "id": "ses_fork" })); + let (controls, _steer, _token) = controls(); + let mut stream = harness(&fake) + .run(fork_request("hi", "ses_src", "msg_a", None), controls) + .await + .expect("run starts"); + + // The run continues the FORKED session (the fork response's id)… + let started = next_event(&mut stream).await; + assert!(matches!( + &started, + AgentEvent::SessionStarted { session_id, .. } if session_id == "ses_fork" + )); + let _commands = next_event(&mut stream).await; + // …cut at the next turn's user message, not at the boundary itself… + let forks = wait_posts(&fake, "/session/ses_src/fork", 1).await; + assert_eq!(forks[0], json!({ "messageID": "msg_b" })); + // …and nothing is pasted: the prompt carries only the user text, and + // no fresh session was created behind the fork's back. + let prompts = wait_posts(&fake, "/session/ses_fork/prompt_async", 1).await; + assert_eq!(prompts[0]["parts"][0]["text"], "hi"); + assert_eq!(prompts[0]["parts"].as_array().map(Vec::len), Some(1)); + assert!(fake.posts_to("/session").is_empty(), "no fresh create"); + // The forked turn's ref names the forked session. + let message_id = prompts[0]["messageID"].as_str().unwrap().to_owned(); + let turn_ref = next_event_with_refs(&mut stream).await; + assert_eq!( + turn_ref, + AgentEvent::TurnRef { + reference: ProviderTurnRef { + harness: HarnessId::Opencode, + session_id: "ses_fork".into(), + turn_id: message_id, + } + } + ); + + assistant_message(&fake, "ses_fork", "msg_1"); + fake.emit(json!({ + "type": "message.part.updated", + "properties": { "part": { + "id": "prt_t", "messageID": "msg_1", "sessionID": "ses_fork", + "type": "text", "text": "forked answer", + }}, + })); + let text = next_event(&mut stream).await; + assert!(matches!(&text, AgentEvent::TextDelta { text } if text == "forked answer")); + idle(&fake, "ses_fork"); + let events = drain_to_done(&mut stream).await; + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some("ses_fork".into()), + }) + ); +} + +#[tokio::test] +async fn a_fork_at_the_latest_turn_copies_the_whole_session() { + // No next turn: no boundary at all — the whole session is copied. + let fake = FakeOpencode::start().await; + fake.set_messages( + "ses_src", + vec![ + json!({ "info": { "id": "msg_a", "role": "user", "sessionID": "ses_src" } }), + json!({ "info": { "id": "msg_a_reply", "role": "assistant", "sessionID": "ses_src" } }), + ], + ); + fake.set_fork("ses_src", "200 OK", json!({ "id": "ses_fork" })); + let (controls, _steer, _token) = controls(); + let mut stream = harness(&fake) + .run(fork_request("hi", "ses_src", "msg_a", None), controls) + .await + .expect("run starts"); + + let started = next_event(&mut stream).await; + assert!(matches!( + &started, + AgentEvent::SessionStarted { session_id, .. } if session_id == "ses_fork" + )); + let forks = wait_posts(&fake, "/session/ses_src/fork", 1).await; + assert_eq!(forks[0], json!({})); + let prompts = wait_posts(&fake, "/session/ses_fork/prompt_async", 1).await; + assert_eq!(prompts[0]["parts"][0]["text"], "hi"); + + assistant_message(&fake, "ses_fork", "msg_1"); + idle(&fake, "ses_fork"); + drain_to_done(&mut stream).await; +} + +#[tokio::test] +async fn a_v2_native_fork_cuts_before_the_next_user_message() { + let fake = FakeOpencode::start_v2().await; + fake.set_messages( + "ses_src", + vec![ + json!({ "id": "msg_a", "type": "user", "time": { "created": 1 } }), + json!({ "id": "msg_a_reply", "type": "assistant", "time": { "created": 2 } }), + json!({ "id": "msg_b", "type": "user", "time": { "created": 3 } }), + json!({ "id": "msg_b_reply", "type": "assistant", "time": { "created": 4 } }), + ], + ); + fake.set_fork("ses_src", "200 OK", json!({ "data": { "id": "ses_fork2" } })); + let (controls, _steer, _token) = controls(); + let mut stream = harness(&fake) + .run(fork_request("hi", "ses_src", "msg_a", None), controls) + .await + .expect("run starts"); + + let started = next_event(&mut stream).await; + assert!(matches!( + &started, + AgentEvent::SessionStarted { session_id, .. } if session_id == "ses_fork2" + )); + let _commands = next_event(&mut stream).await; + // 2.x spells the exclusive boundary `before` and wraps the answer in + // `{data}` — the forked session id comes from there. + let forks = wait_posts(&fake, "/api/session/ses_src/fork", 1).await; + assert_eq!(forks[0], json!({ "before": "msg_b" })); + // Nothing is pasted: the prompt carries only the user text. + let prompts = wait_posts(&fake, "/api/session/ses_fork2/prompt", 1).await; + assert_eq!(prompts[0]["text"], "hi"); + assert!(fake.posts_to("/api/session").is_empty(), "no fresh create"); + let message_id = prompts[0]["id"].as_str().unwrap().to_owned(); + let turn_ref = next_event_with_refs(&mut stream).await; + assert_eq!( + turn_ref, + AgentEvent::TurnRef { + reference: ProviderTurnRef { + harness: HarnessId::Opencode, + session_id: "ses_fork2".into(), + turn_id: message_id, + } + } + ); + + v2_assistant_message(&fake, "ses_fork2", "amsg_1"); + v2_text(&fake, "ses_fork2", "amsg_1", "forked answer"); + let text = next_event(&mut stream).await; + assert!(matches!(&text, AgentEvent::TextDelta { text } if text == "forked answer")); + v2_idle(&fake, "ses_fork2"); + let events = drain_to_done(&mut stream).await; + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some("ses_fork2".into()), + }) + ); +} + +#[tokio::test] +async fn a_v2_fork_at_the_latest_turn_sends_no_boundary() { + let fake = FakeOpencode::start_v2().await; + fake.set_messages( + "ses_src", + vec![ + json!({ "id": "msg_a", "type": "user", "time": { "created": 1 } }), + json!({ "id": "msg_a_reply", "type": "assistant", "time": { "created": 2 } }), + ], + ); + fake.set_fork("ses_src", "200 OK", json!({ "data": { "id": "ses_fork2" } })); + let (controls, _steer, _token) = controls(); + let mut stream = harness(&fake) + .run(fork_request("hi", "ses_src", "msg_a", None), controls) + .await + .expect("run starts"); + + let started = next_event(&mut stream).await; + assert!(matches!( + &started, + AgentEvent::SessionStarted { session_id, .. } if session_id == "ses_fork2" + )); + let forks = wait_posts(&fake, "/api/session/ses_src/fork", 1).await; + assert_eq!(forks[0], json!({})); + let prompts = wait_posts(&fake, "/api/session/ses_fork2/prompt", 1).await; + assert_eq!(prompts[0]["text"], "hi"); + + v2_assistant_message(&fake, "ses_fork2", "amsg_1"); + v2_idle(&fake, "ses_fork2"); + drain_to_done(&mut stream).await; +} + +#[tokio::test] +async fn a_refused_fork_falls_back_to_a_fresh_session_with_the_budgeted_history() { + let fake = FakeOpencode::start().await; + fake.set_messages( + "ses_src", + vec![ + json!({ "info": { "id": "msg_a", "role": "user", "sessionID": "ses_src" } }), + json!({ "info": { "id": "msg_a_reply", "role": "assistant", "sessionID": "ses_src" } }), + json!({ "info": { "id": "msg_b", "role": "user", "sessionID": "ses_src" } }), + ], + ); + // No fork outcome configured: the POST 404s — the server refused. + let (controls, _steer, _token) = controls(); + let mut stream = harness(&fake) + .run( + fork_request("hi", "ses_src", "msg_a", Some(BUDGETED_HISTORY)), + controls, + ) + .await + .expect("run starts"); + + // The boundary resolved (the refused call still addressed the next + // turn)… + let forks = wait_posts(&fake, "/session/ses_src/fork", 1).await; + assert_eq!(forks[0], json!({ "messageID": "msg_b" })); + // …but the turn then starts a FRESH session… + let started = next_event(&mut stream).await; + assert!(matches!( + &started, + AgentEvent::SessionStarted { session_id, .. } if session_id == "ses_test" + )); + let _commands = next_event(&mut stream).await; + // The fresh session's turn still emits its ref (the entry the engine + // stamps is addressable for a LATER fork of this chat). + let _turn_ref = next_event_with_refs(&mut stream).await; + assert!(matches!(&_turn_ref, AgentEvent::TurnRef { .. })); + assert_eq!(fake.posts_to("/session").len(), 1, "one fresh create"); + // …whose prompt carries the budgeted history in front of the user + // message, byte-identical to the non-native paste. + let prompts = wait_posts(&fake, "/session/ses_test/prompt_async", 1).await; + assert_eq!( + prompts[0]["parts"][0]["text"], + format!("{BUDGETED_HISTORY}hi") + ); + + assistant_message(&fake, "ses_test", "msg_1"); + fake.emit(json!({ + "type": "message.part.updated", + "properties": { "part": { + "id": "prt_t", "messageID": "msg_1", "sessionID": "ses_test", + "type": "text", "text": "fresh answer", + }}, + })); + let text = next_event(&mut stream).await; + assert!(matches!(&text, AgentEvent::TextDelta { text } if text == "fresh answer")); + idle(&fake, "ses_test"); + let events = drain_to_done(&mut stream).await; + assert!(events.iter().all(|ev| !matches!(ev, AgentEvent::Error { .. }))); + assert_eq!( + events.last(), + Some(&AgentEvent::Done { + status: DoneStatus::Completed, + result: None, + error: None, + session_id: Some("ses_test".into()), + }) + ); +} + +#[tokio::test] +async fn a_fork_boundary_the_server_cannot_resolve_falls_back_to_the_paste() { + // The message list itself fails (the session is gone or unreadable): + // "a call fails" — same budgeted-paste fallback, and no fork call is + // ever made. + let fake = FakeOpencode::start_v2().await; + let (controls, _steer, _token) = controls(); + let mut stream = harness(&fake) + .run( + fork_request("hi", "ses_src", "msg_a", Some(BUDGETED_HISTORY)), + controls, + ) + .await + .expect("run starts"); + + let started = next_event(&mut stream).await; + assert!(matches!( + &started, + AgentEvent::SessionStarted { session_id, .. } if session_id == "ses_test2" + )); + assert_eq!(fake.posts_to("/api/session").len(), 1, "one fresh create"); + assert!( + fake.posts_to("/api/session/ses_src/fork").is_empty(), + "the fork is never called with an unresolvable boundary" + ); + let prompts = wait_posts(&fake, "/api/session/ses_test2/prompt", 1).await; + assert_eq!(prompts[0]["text"], format!("{BUDGETED_HISTORY}hi")); + + v2_assistant_message(&fake, "ses_test2", "amsg_1"); + v2_idle(&fake, "ses_test2"); + let events = drain_to_done(&mut stream).await; + assert!(events.iter().all(|ev| !matches!(ev, AgentEvent::Error { .. }))); +} + +#[tokio::test] +async fn a_v2_fork_resolution_follows_the_message_list_cursor() { + // The boundary turn sits on the first page, the next user message on + // the second: the driver must follow the cursor before it cuts. + let fake = FakeOpencode::start_v2().await; + fake.set_messages_paged( + "ses_src", + vec![ + vec![ + json!({ "id": "msg_a", "type": "user", "time": { "created": 1 } }), + json!({ "id": "msg_a_reply", "type": "assistant", "time": { "created": 2 } }), + ], + vec![ + json!({ "id": "msg_b", "type": "user", "time": { "created": 3 } }), + json!({ "id": "msg_b_reply", "type": "assistant", "time": { "created": 4 } }), + ], + ], + ); + fake.set_fork("ses_src", "200 OK", json!({ "data": { "id": "ses_fork2" } })); + let (controls, _steer, _token) = controls(); + let mut stream = harness(&fake) + .run(fork_request("hi", "ses_src", "msg_a", None), controls) + .await + .expect("run starts"); + + let started = next_event(&mut stream).await; + assert!(matches!( + &started, + AgentEvent::SessionStarted { session_id, .. } if session_id == "ses_fork2" + )); + let _commands = next_event(&mut stream).await; + // The cut addresses the next turn's user message — which only exists + // on the second page, so the driver followed the cursor to find it. + let forks = wait_posts(&fake, "/api/session/ses_src/fork", 1).await; + assert_eq!(forks[0], json!({ "before": "msg_b" })); + let prompts = wait_posts(&fake, "/api/session/ses_fork2/prompt", 1).await; + assert_eq!(prompts[0]["text"], "hi"); + + v2_assistant_message(&fake, "ses_fork2", "amsg_1"); + v2_idle(&fake, "ses_fork2"); + drain_to_done(&mut stream).await; +} diff --git a/crates/harness/tests/pi.rs b/crates/harness/tests/pi.rs index 4cb5332944..2da1dae5cb 100644 --- a/crates/harness/tests/pi.rs +++ b/crates/harness/tests/pi.rs @@ -10,6 +10,7 @@ use std::path::PathBuf; use std::time::Duration; use futures::StreamExt; +use futures::stream::BoxStream; use tokio::sync::{mpsc, oneshot}; use roboco_harness::{CancellationToken, Harness, PiHarness, RunControls, SteerMessage, TurnControl}; @@ -54,6 +55,8 @@ fn request_cwd(prompt: &str, cwd: &str) -> RunRequest { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } @@ -130,6 +133,27 @@ fn text_of(events: &[AgentEvent]) -> String { .collect() } +/// Collect events through the next `Done` (bounded), keeping the stream +/// open for further turns. +async fn until_done( + stream: &mut BoxStream<'static, Result>, + events: &mut Vec, + budget: Duration, +) { + loop { + let event = tokio::time::timeout(budget, stream.next()) + .await + .expect("progress") + .expect("stream open") + .expect("stream event"); + let done = matches!(event, AgentEvent::Done { .. }); + events.push(event); + if done { + return; + } + } +} + // --------------------------------------------------------------------------- // Descriptor + catalog surfaces // --------------------------------------------------------------------------- @@ -140,7 +164,10 @@ fn pi_surfaces_match_registry_expectations() { assert_eq!(pi.id(), HarnessId::Pi); assert_eq!(pi.display_name(), "Pi"); assert!(pi.supports_steering()); - assert_eq!(pi.steering_mode(), SteeringMode::TurnBoundary); + // Step-boundary (orchestration v2 ticket 03): pi's steer queue delivers + // after the current assistant turn's tool calls — the engine registry + // descriptor must agree (its own tests check the pair). + assert_eq!(pi.steering_mode(), SteeringMode::StepBoundary); assert_eq!( pi.reasoning_levels(), &[ @@ -431,6 +458,11 @@ async fn title_runs_spawn_an_isolated_one_shot_and_answer_with_a_title() { // A hostile caller smuggles interaction surfaces; run_title must clear // every one of them before spawning. req.resume = Some("some-old-session".into()); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: "some-old-session.jsonl".into(), + last_turn: "u1".into(), + fallback_history: None, + }); req.attachments = vec!["/etc/passwd".into()]; req.auto_approve = true; let mut stream = harness().run_title(req, ctl).await.unwrap(); @@ -450,6 +482,8 @@ async fn title_runs_spawn_an_isolated_one_shot_and_answer_with_a_title() { // No session file in the scratch dir: the title sandbox is ephemeral // and never resumes. (The fixture asserts the spawn flags itself.) assert!(!scratch.path().join(".pi-fixture-session.jsonl").exists()); + // The smuggled fork directive never spawned a fork process either. + assert!(!scratch.path().join(".pi-fixture-forked.jsonl").exists()); // pi participates in device-local title generation. assert!(roboco_harness::supports_titles(HarnessId::Pi)); } @@ -496,7 +530,7 @@ async fn errored_assistant_message_rejects_the_prompt() { // The failed message also surfaces as a visible error chip. assert!(events.iter().any(|e| matches!( e, - AgentEvent::Error { message } if message == "provider returned 529" + AgentEvent::Error { message, .. } if message == "provider returned 529" ))); } @@ -515,7 +549,7 @@ async fn exhausted_auto_retry_rejects_the_prompt_and_recovery_completes() { // Retry status surfaces as a visible notice. assert!(events.iter().any(|e| matches!( e, - AgentEvent::Error { message } if message.contains("attempt 1/3") + AgentEvent::Error { message, .. } if message.contains("attempt 1/3") ))); let (controls, _steer, _token) = controls(); @@ -592,7 +626,7 @@ async fn lost_resume_pointer_announces_fresh_context() { assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)]); assert!(events.iter().any(|e| matches!( e, - AgentEvent::Error { message } if message.contains("without the previous context") + AgentEvent::Error { message, .. } if message.contains("without the previous context") ))); assert_eq!(text_of(&events), "reply:fresh"); } @@ -947,3 +981,1008 @@ async fn wake_turn_after_settle_still_gets_its_done() { ); assert!(text_of(&events).contains("wake output")); } + +// --------------------------------------------------------------------------- +// Prompt delivery hardening: dispositions, rejections, the settle probe +// (orchestration v2, ticket 02) +// --------------------------------------------------------------------------- + +/// A prompt pi handles without a run (an extension command) ends the turn — +/// the settle probe confirms pi at rest — instead of sticking the chat on +/// Working forever, and the session keeps serving. +#[tokio::test] +async fn a_handled_prompt_ends_the_turn_and_parks_the_session() { + let (controls, steer, _token) = controls(); + let mut stream = harness().run(request("handled"), controls).await.unwrap(); + let mut events = Vec::new(); + until_done(&mut stream, &mut events, Duration::from_secs(15)).await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + // The session parks: the next message runs as a normal turn. + steer + .send(SteerMessage::text("after handled")) + .await + .unwrap(); + let mut second = Vec::new(); + until_done(&mut stream, &mut second, Duration::from_secs(15)).await; + assert_eq!(dones(&second), vec![(DoneStatus::Completed, None)], "{second:?}"); + assert!(second.iter().any(|e| matches!(e, AgentEvent::Steered { .. }))); + assert_eq!(text_of(&second), "reply:after handled"); +} + +/// A prompt rejected while pi is streaming surfaces the rejection, returns +/// the message to the queue, and never ends the session: the message +/// re-delivers once pi comes to rest, and the session keeps serving. +#[tokio::test] +async fn a_prompt_rejected_while_pi_streams_keeps_the_session_and_the_message() { + let (controls, steer, _token) = controls(); + let mut stream = harness() + .run(request("reject-while-streaming"), controls) + .await + .unwrap(); + let mut events = Vec::new(); + let mut sent = false; + loop { + let event = tokio::time::timeout(Duration::from_secs(15), stream.next()) + .await + .expect("progress") + .expect("stream open") + .expect("stream event"); + let is_working = matches!(&event, AgentEvent::TextDelta { text } if text == "working"); + let saw_reply = text_of(&events).contains("reply:steered while streaming"); + let is_done = matches!(&event, AgentEvent::Done { .. }); + events.push(event); + if is_working && !sent { + // The extension run is streaming: this is the window where the + // message is delivered and rejected. + sent = true; + steer + .send(SteerMessage::text("steered while streaming")) + .await + .unwrap(); + } + if is_done && saw_reply { + break; + } + } + // The prompted turn, the extension run, and the re-delivered message: + // three completed turns — the session never died, the turn never + // stranded. + assert_eq!( + dones(&events), + vec![(DoneStatus::Completed, None); 3], + "{events:?}" + ); + // The rejection surfaced... + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::Error { message, .. } if message.contains("busy") + )), + "{events:?}" + ); + // ...and the message survived the rejection and ran. + assert!( + text_of(&events).contains("reply:steered while streaming"), + "{events:?}" + ); + // The session is alive: a follow-up still runs. + steer.send(SteerMessage::text("still alive")).await.unwrap(); + let mut tail = Vec::new(); + until_done(&mut stream, &mut tail, Duration::from_secs(15)).await; + assert_eq!(dones(&tail), vec![(DoneStatus::Completed, None)], "{tail:?}"); + assert_eq!(text_of(&tail), "reply:still alive"); +} + +/// A slow pi preflight (extensions, the Roboco MCP bridge, compaction) is +/// allowed: no fixed acceptance budget kills the run — only Stop interrupts +/// a slow acceptance. +#[tokio::test] +async fn a_slow_pi_preflight_completes_the_run() { + let (controls, steer, _token) = controls(); + drop(steer); + let started = std::time::Instant::now(); + let mut stream = harness().run(request("slow-preflight"), controls).await.unwrap(); + let mut events = Vec::new(); + until_done(&mut stream, &mut events, Duration::from_secs(60)).await; + assert!(started.elapsed() >= Duration::from_secs(31)); + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + assert_eq!(text_of(&events), "reply:slow-preflight"); +} + +/// Stop interrupts a slow acceptance: no fixed timeout kills the run, but +/// Stop still ends it — exactly one `Interrupted` Done, never an +/// error-then-interrupted pair from the unanswered acceptance. +#[tokio::test] +async fn stop_interrupts_a_slow_pi_preflight() { + let (controls, steer, token) = controls(); + drop(steer); + let mut stream = harness().run(request("slow-preflight"), controls).await.unwrap(); + // The acceptance is in flight (the fake holds its answer for 31 s). + tokio::time::sleep(Duration::from_millis(500)).await; + token.cancel(); + let mut events = Vec::new(); + while let Some(event) = + tokio::time::timeout(Duration::from_secs(10), stream.next()).await.expect("progress") + { + let event = event.expect("stream event"); + let done = matches!(event, AgentEvent::Done { .. }); + events.push(event); + if done { + break; + } + } + assert_eq!(dones(&events), vec![(DoneStatus::Interrupted, None)], "{events:?}"); + // The run never started: nothing streamed before the Stop. + assert!(text_of(&events).is_empty(), "{events:?}"); +} + +/// A prompt racing `agent_settled` either joins the run or starts the next +/// turn: it is delivered exactly once and the turn never strands. +#[tokio::test] +async fn a_prompt_racing_agent_settled_never_strands_the_turn() { + let (controls, steer, _token) = controls(); + let mut stream = harness().run(request("settle-race"), controls).await.unwrap(); + let mut events = Vec::new(); + let mut sent = false; + while dones(&events).len() < 2 { + let event = tokio::time::timeout(Duration::from_secs(15), stream.next()) + .await + .expect("progress") + .expect("stream open") + .expect("stream event"); + let race_now = matches!(&event, AgentEvent::TextDelta { text } if text == "reply:settle-race") + && !sent; + events.push(event); + if race_now { + // The settle probe is in flight (the fixture holds its answer + // open): the message lands behind the turn. + sent = true; + tokio::time::sleep(Duration::from_millis(150)).await; + steer.send(SteerMessage::text("race steer")).await.unwrap(); + } + } + assert_eq!( + dones(&events), + vec![(DoneStatus::Completed, None), (DoneStatus::Completed, None)], + "{events:?}" + ); + // The raced message ran — joined the run or started the next turn. + assert!(text_of(&events).contains("reply:settle-race")); + assert!(text_of(&events).contains("reply:race steer"), "{events:?}"); + assert!(events.iter().any(|e| matches!(e, AgentEvent::Steered { .. }))); +} + +/// A settle probe that reports busy (pi still streaming) re-arms: the turn +/// ends only once pi is at rest — never stranded, never errored. +#[tokio::test] +async fn a_busy_settle_probe_re_arms_until_pi_rests() { + let (controls, steer, _token) = controls(); + drop(steer); + let events = run_to_end(&harness(), request("busy-probe"), controls).await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + assert_eq!(text_of(&events), "reply:busy-probe"); +} + +/// A settle probe that keeps failing ends the turn ERRORED — visibly, with +/// the reason — instead of hanging or silently killing the process. +#[tokio::test] +async fn a_failing_settle_probe_ends_the_turn_errored() { + let (controls, steer, _token) = controls(); + drop(steer); + let events = run_to_end(&harness(), request("probe-fail"), controls).await; + let dones = dones(&events); + assert_eq!(dones.len(), 1, "{events:?}"); + assert_eq!(dones[0].0, DoneStatus::Errored); + let error = dones[0].1.as_deref().unwrap(); + assert!(error.contains("probe") && error.contains("wedged"), "{error}"); +} + +/// pi older than 0.99 reports no disposition: the driver infers from whether +/// a run starts — a running prompt settles normally, and a handled one ends +/// the turn instead of sticking on Working forever. +#[tokio::test] +async fn old_pi_without_disposition_is_inferred_not_assumed() { + let (first, steer, _token) = controls(); + drop(steer); + let events = run_to_end(&harness(), request("old-pi"), first).await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + assert_eq!(text_of(&events), "reply:old-pi"); + + let (controls, steer, _token) = controls(); + let mut stream = harness().run(request("old-pi-handled"), controls).await.unwrap(); + let mut events = Vec::new(); + until_done(&mut stream, &mut events, Duration::from_secs(15)).await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + // The session lives on: the next message runs. + steer.send(SteerMessage::text("revived")).await.unwrap(); + let mut tail = Vec::new(); + until_done(&mut stream, &mut tail, Duration::from_secs(15)).await; + assert_eq!(dones(&tail), vec![(DoneStatus::Completed, None)], "{tail:?}"); + assert_eq!(text_of(&tail), "reply:revived"); +} + +// --------------------------------------------------------------------------- +// Mid-turn steering (orchestration v2, ticket 03): pi is a step-boundary +// harness — steering mode "all" at session start, steers carrying images, +// clear_queue before abort, compaction recovery, bursts in order. +// --------------------------------------------------------------------------- + +/// The steering mode is set to "all" at session start unless the user +/// configured one: pi's one-at-a-time default would deliver a burst of +/// steers one assistant turn at a time; roboco's steers belong together. +#[tokio::test] +async fn steering_mode_defaults_to_all_at_session_start() { + let home = tempfile::tempdir().unwrap(); + let cwd = tempfile::tempdir().unwrap(); + let (controls, steer, _token) = controls(); + drop(steer); + let events = run_to_end( + &harness().with_agent_dir(home.path()), + request_cwd("hello", &cwd.path().display().to_string()), + controls, + ) + .await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + // The fixture records the wire selection in its session file. + let recorded = + std::fs::read_to_string(cwd.path().join(".pi-fixture-session.jsonl")).unwrap(); + assert!(recorded.contains("steering-mode:all"), "{recorded}"); +} + +/// A steering mode the user already configured (the agent directory's +/// settings — their own edit, or `/steering` picked globally) is never +/// overwritten: the driver sends no `set_steering_mode` at all. +#[tokio::test] +async fn a_configured_steering_mode_is_never_overwritten() { + let home = tempfile::tempdir().unwrap(); + std::fs::write( + home.path().join("settings.json"), + r#"{"steeringMode": "one-at-a-time"}"#, + ) + .unwrap(); + let cwd = tempfile::tempdir().unwrap(); + let (controls, steer, _token) = controls(); + drop(steer); + let events = run_to_end( + &harness().with_agent_dir(home.path()), + request_cwd("hello", &cwd.path().display().to_string()), + controls, + ) + .await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + let recorded = + std::fs::read_to_string(cwd.path().join(".pi-fixture-session.jsonl")).unwrap(); + assert!(!recorded.contains("steering-mode:"), "{recorded}"); +} + +/// A mode configured in the project `.pi/settings.json` counts the same: +/// an explicit choice, never overwritten. +#[tokio::test] +async fn project_settings_count_as_a_configured_steering_mode() { + let home = tempfile::tempdir().unwrap(); + let cwd = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(cwd.path().join(".pi")).unwrap(); + std::fs::write( + cwd.path().join(".pi").join("settings.json"), + r#"{"steeringMode": "all"}"#, + ) + .unwrap(); + let (controls, steer, _token) = controls(); + drop(steer); + let events = run_to_end( + &harness().with_agent_dir(home.path()), + request_cwd("hello", &cwd.path().display().to_string()), + controls, + ) + .await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + let recorded = + std::fs::read_to_string(cwd.path().join(".pi-fixture-session.jsonl")).unwrap(); + assert!(!recorded.contains("steering-mode:"), "{recorded}"); +} + +/// A mid-run steer carries its staged images inline — the same +/// `ImageContent` blocks a turn-start prompt rides — so steering with a +/// screenshot reaches pi. The fixture asserts the wire block itself. +#[tokio::test] +async fn steers_carry_their_attached_images() { + let image = tempfile::tempdir().unwrap(); + let png = image.path().join("shot.png"); + std::fs::write(&png, b"\x89PNG\r\n\x1a\nroboco steer image").unwrap(); + let (controls, steer, _token) = controls(); + let mut stream = harness().run(request("steer-image"), controls).await.unwrap(); + let mut events = Vec::new(); + let mut sent = false; + while let Some(event) = stream.next().await { + let event = event.unwrap(); + let steer_now = matches!(&event, AgentEvent::TextDelta { text } if text == "first part ") + && !sent; + events.push(event); + if steer_now { + sent = true; + steer + .send(SteerMessage { + prompt: "look at this".into(), + message_id: None, + attachments: vec![png.display().to_string()], + config: None, + }) + .await + .unwrap(); + } + if matches!(events.last(), Some(AgentEvent::Done { .. })) { + break; + } + } + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + assert!(events.iter().any(|e| matches!(e, AgentEvent::Steered { .. })), "{events:?}"); + assert!(text_of(&events).contains("reply-to-image-steer"), "{events:?}"); +} + +/// Stop sends `clear_queue` before `abort`: steers pi queued but never +/// consumed are cancelled with the turn — never delivered into the aborted +/// run. The fixture asserts the wire order and that the cleared queue held +/// the stranded steer; here the run ends interrupted, exactly once. +#[tokio::test] +async fn stop_clears_pi_queue_before_abort() { + let (controls, steer, token) = controls(); + let mut stream = harness().run(request("stop-clears-queue"), controls).await.unwrap(); + let events = tokio::time::timeout(Duration::from_secs(10), async { + let mut events = Vec::new(); + let mut sent = false; + while let Some(event) = stream.next().await { + let event = event.unwrap(); + let steer_now = + matches!(&event, AgentEvent::TextDelta { text } if text == "working") && !sent; + let stop_now = matches!(&event, AgentEvent::TextDelta { text } if text == "steer queued"); + events.push(event); + if steer_now { + sent = true; + steer.send(SteerMessage::text("stranded steer")).await.unwrap(); + } + if stop_now { + token.cancel(); + } + if matches!(events.last(), Some(AgentEvent::Done { .. })) { + break; + } + } + events + }) + .await + .expect("stop must settle"); + assert_eq!(dones(&events), vec![(DoneStatus::Interrupted, None)], "{events:?}"); +} + +/// A run that recovered from an error through a successful compaction retry +/// ends completed, not errored: the recovered error no longer rejects the +/// prompt. +#[tokio::test] +async fn a_compaction_recovered_run_completes() { + let (controls, steer, _token) = controls(); + drop(steer); + let events = run_to_end(&harness(), request("compaction-recover"), controls).await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + // The overflow error surfaced (as a transient notice) and the retried + // turn still completed the run. + assert!(text_of(&events).contains("doomed"), "{events:?}"); + assert!(text_of(&events).contains("recovered"), "{events:?}"); + assert!( + events.iter().any(|e| matches!(e, AgentEvent::Error { message, .. } if message.contains("overflow"))), + "{events:?}" + ); +} + +/// A burst mixing an image steer stays in mailbox order (B5): the driver +/// loads a steer's images before queueing its wire call, so the +/// FuturesUnordered's completion order cannot overtake the image steer +/// with the text steer queued after it — pi receives exactly +/// [text, image, text]. The fixture asserts the wire order and the image +/// block itself; here the burst folds in with its boundaries. +#[tokio::test] +async fn a_burst_with_an_image_steer_stays_in_order() { + let image = tempfile::tempdir().unwrap(); + let png = image.path().join("burst.png"); + std::fs::write(&png, b"\x89PNG\r\n\x1a\nroboco burst image").unwrap(); + let (controls, steer, _token) = controls(); + let mut stream = harness().run(request("steer-burst-mixed"), controls).await.unwrap(); + let mut events = Vec::new(); + let mut sent = false; + while let Some(event) = stream.next().await { + let event = event.unwrap(); + let steer_now = matches!(&event, AgentEvent::TextDelta { text } if text == "first part ") + && !sent; + events.push(event); + if steer_now { + sent = true; + for (text, with_image) in [ + ("mixed one", None), + ("mixed two", Some(png.display().to_string())), + ("mixed three", None), + ] { + steer + .send(SteerMessage { + prompt: text.into(), + message_id: None, + attachments: with_image.map(|path| vec![path]).unwrap_or_default(), + config: None, + }) + .await + .unwrap(); + } + } + if matches!(events.last(), Some(AgentEvent::Done { .. })) { + break; + } + } + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + let steered = events + .iter() + .filter(|e| matches!(e, AgentEvent::Steered { .. })) + .count(); + assert_eq!(steered, 3, "{events:?}"); + assert!(text_of(&events).contains("reply-to-mixed-burst"), "{events:?}"); +} + +/// A steer whose acknowledgement lands AFTER the settle does not end the +/// turn early (B4): the quiescent probe reads pi at rest while the input +/// is still in preflight, and the driver must wait it out — the turn ends +/// once, after the acked steer runs and settles in its own turn. +#[tokio::test] +async fn a_late_acknowledgement_does_not_end_the_turn_early() { + let (controls, steer, _token) = controls(); + let mut stream = harness().run(request("late-ack"), controls).await.unwrap(); + let mut events = Vec::new(); + let mut sent = false; + while let Some(event) = stream.next().await { + let event = event.unwrap(); + let steer_now = matches!(&event, AgentEvent::TextDelta { text } if text == "working") + && !sent; + events.push(event); + if steer_now { + sent = true; + steer.send(SteerMessage::text("late steer")).await.unwrap(); + } + if matches!(events.last(), Some(AgentEvent::Done { .. })) { + break; + } + } + // Exactly one Done, and only after the late steer ran: the reply to + // the steer is in the SAME turn (no Done between the settle and the + // ack), and the turn completes. + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + assert!(text_of(&events).contains("reply-to-late-steer"), "{events:?}"); + assert!(events.iter().any(|e| matches!(e, AgentEvent::Steered { .. })), "{events:?}"); +} + +/// A message pi keeps rejecting is capped, not retried forever (B1): past +/// the re-delivery attempts the message is handed back to the engine as a +/// failed send and the turn ends ERRORED — one Error naming the attempts, +/// and NEVER a Done{Completed} for a message that never ran. +#[tokio::test] +async fn an_endlessly_rejected_message_is_capped_and_ends_the_turn_errored() { + let (controls, steer, _token) = controls(); + let mut stream = harness().run(request("reject-cap"), controls).await.unwrap(); + let mut events = Vec::new(); + let mut sent = false; + while dones(&events).len() < 2 { + let event = tokio::time::timeout(Duration::from_secs(20), stream.next()) + .await + .expect("progress") + .expect("stream open") + .expect("stream event"); + // The prompted turn completes and parks; the parked message goes + // out after the first Done. + let send_now = + matches!(&event, AgentEvent::Done { status: DoneStatus::Completed, .. }) && !sent; + events.push(event); + if send_now { + sent = true; + steer.send(SteerMessage::text("always rejected")).await.unwrap(); + } + } + drop(steer); + let dones = dones(&events); + assert_eq!(dones.len(), 2, "{events:?}"); + assert_eq!(dones[0], (DoneStatus::Completed, None), "{events:?}"); + match &dones[1] { + (DoneStatus::Errored, Some(error)) => assert!( + error.contains("pi rejected the message 2 times"), + "the cap ends the turn errored with the attempt count: {error}" + ), + other => panic!("the capped rejection ends the turn errored: {other:?}"), + } + // One rejection Error per attempt — and no third attempt (the cap). + let rejections = events + .iter() + .filter(|e| matches!(e, AgentEvent::Error { message, .. } if message.contains("rejected"))) + .count(); + assert_eq!(rejections, 2, "{events:?}"); +} + +/// The mailbox never wedges behind a dead turn (B2): a message queued +/// behind the settle probe still runs as the next turn after the probe +/// fails out and the turn ends errored. +#[tokio::test] +async fn a_message_queued_behind_a_failing_probe_still_runs() { + let (controls, steer, _token) = controls(); + let mut stream = harness().run(request("probe-fail-mailbox"), controls).await.unwrap(); + let mut events = Vec::new(); + let mut sent = false; + while dones(&events).len() < 2 { + let event = tokio::time::timeout(Duration::from_secs(20), stream.next()) + .await + .expect("progress") + .expect("stream open") + .expect("stream event"); + let send_now = matches!( + &event, + AgentEvent::TextDelta { text } if text == "reply:probe-fail-mailbox" + ) && !sent; + events.push(event); + if send_now { + // The fixture holds the first probe answer open: the message + // lands behind the settling turn. + sent = true; + tokio::time::sleep(Duration::from_millis(300)).await; + steer.send(SteerMessage::text("queued during the probe")).await.unwrap(); + } + } + drop(steer); + let dones = dones(&events); + assert_eq!( + dones.len(), + 2, + "the failed turn, then the queued message's own turn: {dones:?}" + ); + assert_eq!(dones[0].0, DoneStatus::Errored); + assert_eq!(dones[1].0, DoneStatus::Completed, "{events:?}"); + assert!( + text_of(&events).contains("reply:queued during the probe"), + "the queued message ran its own turn: {events:?}" + ); +} + +/// A stranded steer is recovered INTACT (B6): the images it was sent with +/// ride the re-delivery too — the recovery re-queues the original +/// message, never a bare-text copy. The fixture asserts the re-delivered +/// prompt's image block on the wire. +#[tokio::test] +async fn a_stranded_steer_recovers_with_its_images() { + let image = tempfile::tempdir().unwrap(); + let png = image.path().join("strand.png"); + std::fs::write(&png, b"\x89PNG\r\n\x1a\nroboco strand image").unwrap(); + let (controls, steer, _token) = controls(); + let mut stream = harness().run(request("steer-strand-image"), controls).await.unwrap(); + let mut events = Vec::new(); + let mut sent = false; + while dones(&events).len() < 2 { + let event = tokio::time::timeout(Duration::from_secs(15), stream.next()) + .await + .expect("progress") + .expect("stream open") + .expect("stream event"); + let steer_now = matches!(&event, AgentEvent::TextDelta { text } if text == "reply part") + && !sent; + events.push(event); + if steer_now { + sent = true; + steer + .send(SteerMessage { + prompt: "stranded with an image".into(), + message_id: None, + attachments: vec![png.display().to_string()], + config: None, + }) + .await + .unwrap(); + } + } + // Recovered via clear_queue after the first Done, re-delivered as the + // next turn's prompt with its image (the fixture checks the wire). + assert_eq!( + dones(&events), + vec![(DoneStatus::Completed, None), (DoneStatus::Completed, None)], + "{events:?}" + ); + assert!( + text_of(&events).contains("reply-to-recovered-image-steer"), + "the recovered steer ran its own turn: {events:?}" + ); + assert!(events.iter().any(|e| matches!(e, AgentEvent::Steered { .. })), "{events:?}"); +} + +/// A Stop while the settle probe is armed during a LIVE run (B3: an old pi +/// with no disposition arms it at acceptance) still clears the queue and +/// aborts — the "already settled" shortcut is only for a truly settled +/// turn. The fixture records the abort in its session file. +#[tokio::test] +async fn stop_with_a_probe_armed_mid_run_still_aborts() { + let cwd = tempfile::tempdir().unwrap(); + let (controls, _steer, token) = controls(); + let req = request_cwd("old-pi-stop", &cwd.path().display().to_string()); + let mut stream = harness().run(req, controls).await.unwrap(); + let mut events = Vec::new(); + while dones(&events).is_empty() { + let event = tokio::time::timeout(Duration::from_secs(15), stream.next()) + .await + .expect("progress") + .expect("stream open") + .expect("stream event"); + let stop_now = matches!(&event, AgentEvent::TextDelta { text } if text == "working"); + events.push(event); + if stop_now { + // Past the probe's re-arm delay: it is armed (pi reports busy). + tokio::time::sleep(Duration::from_millis(400)).await; + token.cancel(); + } + } + assert_eq!(dones(&events), vec![(DoneStatus::Interrupted, None)], "{events:?}"); + let recorded = + std::fs::read_to_string(cwd.path().join(".pi-fixture-session.jsonl")).unwrap(); + assert!(recorded.contains("abort\n"), "Stop must abort the live run: {recorded}"); +} + +/// Several quick steers are all delivered, in order: pi (steering mode +/// "all") queues each while streaming and delivers them together at the +/// next step boundary. The fixture asserts the send order on the wire; +/// here every steer folds in with its own boundary and one turn ends. +#[tokio::test] +async fn a_burst_of_steers_is_delivered_in_order() { + let (controls, steer, _token) = controls(); + let mut stream = harness().run(request("steer-burst"), controls).await.unwrap(); + let mut events = Vec::new(); + let mut sent = false; + while let Some(event) = stream.next().await { + let event = event.unwrap(); + let steer_now = matches!(&event, AgentEvent::TextDelta { text } if text == "first part ") + && !sent; + events.push(event); + if steer_now { + sent = true; + for text in ["burst one", "burst two", "burst three"] { + steer.send(SteerMessage::text(text)).await.unwrap(); + } + } + if matches!(events.last(), Some(AgentEvent::Done { .. })) { + break; + } + } + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + let steered = events + .iter() + .filter(|e| matches!(e, AgentEvent::Steered { .. })) + .count(); + assert_eq!(steered, 3, "{events:?}"); + assert!(text_of(&events).contains("reply-to-burst"), "{events:?}"); +} + +// --------------------------------------------------------------------------- +// Provider turn refs + native forks (t3-orchestration-v2 ticket 19) +// --------------------------------------------------------------------------- + +/// The session-tree helpers: the driver names the turn's ref after each +/// settle, the engine stamps it on the assistant entry, and a fork reads +/// the boundary off it. The unit level (walks, boundaries) lives in the +/// driver module; this proves the wire shape end to end. +fn session_file_of(events: &[AgentEvent]) -> String { + events + .iter() + .find_map(|e| match e { + AgentEvent::SessionStarted { session_id, .. } => Some(session_id.clone()), + _ => None, + }) + .expect("SessionStarted") +} + +fn turn_refs_of(events: &[AgentEvent]) -> Vec { + events + .iter() + .filter_map(|e| match e { + AgentEvent::TurnRef { reference } => Some(reference.clone()), + _ => None, + }) + .collect() +} + +/// Each settled turn emits its provider turn ref: the session file plus +/// the turn's user entry id — the first turn's prompt, and the second +/// turn's prompt once the parked session runs another message (the `since` +/// cursor bounds the second capture to the second turn's entries). +#[tokio::test] +async fn turn_refs_name_the_session_file_and_each_turns_user_entry() { + let cwd = tempfile::tempdir().unwrap(); + let cwd = cwd.path().display().to_string(); + let (controls, steer, _token) = controls(); + let mut stream = harness().run(request_cwd("first", &cwd), controls).await.unwrap(); + let mut events = Vec::new(); + until_done(&mut stream, &mut events, Duration::from_secs(15)).await; + // The parked session runs a second turn on the same process. + steer.send(SteerMessage::text("second")).await.unwrap(); + let mut tail = Vec::new(); + until_done(&mut stream, &mut tail, Duration::from_secs(15)).await; + events.extend(tail); + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None); 2], "{events:?}"); + let session = session_file_of(&events); + assert_eq!( + turn_refs_of(&events), + vec![ + roboco_proto::ProviderTurnRef { + harness: HarnessId::Pi, + session_id: session.clone(), + turn_id: "u1".into(), + }, + roboco_proto::ProviderTurnRef { + harness: HarnessId::Pi, + session_id: session.clone(), + turn_id: "u2".into(), + }, + ], + "{events:?}" + ); + // The refs name the file the runs actually continue. + assert!(session.ends_with(".pi-fixture-session.jsonl"), "{session}"); + assert!(std::path::Path::new(&session).is_file()); +} + +/// A turn a steer landed in names its LAST user entry — the steer — as the +/// ref: pi's exclusive fork boundary is "the user entry after the ref", so +/// naming the last consumed entry keeps the whole turn (prompt, steer and +/// reply) when a fork cuts at the next turn's start. +#[tokio::test] +async fn a_steered_turn_names_its_last_user_entry() { + let (controls, steer, _token) = controls(); + let mut stream = harness().run(request("steer-flow"), controls).await.unwrap(); + let mut events = Vec::new(); + let mut sent = false; + while let Some(event) = stream.next().await { + let event = event.unwrap(); + let steer_now = matches!(&event, AgentEvent::TextDelta { text } if text == "first part ") + && !sent; + events.push(event); + if steer_now { + sent = true; + steer.send(SteerMessage::text("steered text")).await.unwrap(); + } + if matches!(events.last(), Some(AgentEvent::Done { .. })) { + break; + } + } + let session = session_file_of(&events); + assert_eq!( + turn_refs_of(&events), + vec![roboco_proto::ProviderTurnRef { + harness: HarnessId::Pi, + session_id: session, + // u1 = the prompt, u2 = the steer: the turn consumed the steer + // last, so that is the ref. + turn_id: "u2".into(), + }], + "{events:?}" + ); +} + +/// A native fork at the source's LATEST turn: the fork process launches +/// with `--fork` and sends no RPC fork (the copy already ends at the +/// boundary), the run resumes the FORKED session file, and nothing is +/// pasted — the fake refuses the turn if the budgeted history reaches the +/// prompt or the run resumes anything but the forked file. +#[tokio::test] +async fn a_native_fork_at_the_latest_turn_copies_the_whole_session() { + let cwd = tempfile::tempdir().unwrap(); + let source = cwd.path().join("source-session.jsonl"); + std::fs::write(&source, b"fixture session\n").unwrap(); + let (controls, steer, _token) = controls(); + drop(steer); + let mut req = request_cwd("fork-turn", &cwd.path().display().to_string()); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: source.display().to_string(), + // The fake fork tree's NEWEST user entry: forking there keeps the + // whole copied session — no RPC fork. + last_turn: "u3".into(), + // What the engine would have pasted for a fresh session — the + // driver must NOT use it when the fork succeeds. + fallback_history: Some( + "Continue this side chat using the following prior transcript as context.\n\ + \nuser: Remember PINEAPPLE\n\n\n" + .into(), + ), + }); + let events = run_to_end(&harness(), req, controls).await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + // The run continues the FORKED session file, never the source. + let session = session_file_of(&events); + assert!(session.ends_with(".pi-fixture-forked.jsonl"), "{session}"); + assert!(std::path::Path::new(&session).is_file()); + assert_ne!(session, source.display().to_string()); + // The fork process ran with --fork and sent NO fork RPC: the markers + // in the forked session file prove both. + let forked = std::fs::read_to_string(cwd.path().join(".pi-fixture-forked.jsonl")).unwrap(); + assert!(forked.contains("fork-launch:"), "{forked}"); + assert!(!forked.contains("fork-rpc:"), "{forked}"); + assert_eq!(text_of(&events), "reply:fork-turn"); +} + +/// A native fork at an EARLIER turn also sends the RPC `fork` with the +/// user entry id of the turn AFTER the boundary: pi's fork is exclusive +/// (it re-roots before the named user message), so the forked session ends +/// exactly at the boundary. Nothing is pasted (the fake refuses it). +#[tokio::test] +async fn a_native_fork_at_an_earlier_turn_forks_at_the_next_turns_user_entry() { + let cwd = tempfile::tempdir().unwrap(); + let source = cwd.path().join("source-session.jsonl"); + std::fs::write(&source, b"fixture session\n").unwrap(); + let (controls, steer, _token) = controls(); + drop(steer); + let mut req = request_cwd("fork-turn", &cwd.path().display().to_string()); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: source.display().to_string(), + // The fake fork tree's SECOND user entry: the boundary is turn 2, + // so the RPC fork must name turn 3's user entry (u3). + last_turn: "u2".into(), + fallback_history: Some( + "Continue this side chat using the following prior transcript as context.\n\ + \nuser: Remember PINEAPPLE\n\n\n" + .into(), + ), + }); + let events = run_to_end(&harness(), req, controls).await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + let session = session_file_of(&events); + assert!(session.ends_with(".pi-fixture-forked.jsonl"), "{session}"); + // The RPC fork named the turn AFTER the boundary. + let forked = std::fs::read_to_string(cwd.path().join(".pi-fixture-forked.jsonl")).unwrap(); + assert!(forked.contains("fork-rpc:u3"), "{forked}"); + assert_eq!(text_of(&events), "reply:fork-turn"); +} + +/// A stale boundary ref (an entry the session branch no longer holds) +/// refuses the fork: the run starts a FRESH session with the budgeted +/// transcript in front of the prompt — never a failed send. The fake +/// asserts the pasted history and refuses a --session resume. +#[tokio::test] +async fn a_stale_fork_boundary_falls_back_to_the_budgeted_paste() { + let cwd = tempfile::tempdir().unwrap(); + let source = cwd.path().join("source-session.jsonl"); + std::fs::write(&source, b"fixture session\n").unwrap(); + let (controls, steer, _token) = controls(); + drop(steer); + let mut req = request_cwd("fork-fallback", &cwd.path().display().to_string()); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: source.display().to_string(), + // Not on the fake tree branch: the driver must refuse the fork. + last_turn: "u-gone".into(), + fallback_history: Some( + "Continue this side chat using the following prior transcript as context.\n\ + \nuser: Remember PINEAPPLE\n\n\n" + .into(), + ), + }); + let events = run_to_end(&harness(), req, controls).await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + // A fresh session served the turn (the fake refuses the fork-turn + // scenario on a resume). + let session = session_file_of(&events); + assert!(session.ends_with(".pi-fixture-session.jsonl"), "{session}"); + assert_eq!(text_of(&events), "reply:fork-fallback"); + // The refusal surfaced. + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::Error { message, .. } if message.contains("pi could not fork") + )), + "{events:?}" + ); + // No fork RPC ever ran (the marker proves the refusal came first). + let forked = cwd.path().join(".pi-fixture-forked.jsonl"); + if forked.exists() { + let forked = std::fs::read_to_string(&forked).unwrap(); + assert!(!forked.contains("fork-rpc:"), "{forked}"); + } +} + +/// A fork a pi extension cancels falls back the same way: fresh session, +/// budgeted transcript, visible notice. +#[tokio::test] +async fn a_cancelled_fork_falls_back_to_the_budgeted_paste() { + let cwd = tempfile::tempdir().unwrap(); + let source = cwd.path().join("fork-cancel-source.jsonl"); + std::fs::write(&source, b"fixture session\n").unwrap(); + let (controls, steer, _token) = controls(); + drop(steer); + let mut req = request_cwd("fork-fallback", &cwd.path().display().to_string()); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: source.display().to_string(), + // An earlier-turn boundary, so the cancelled RPC actually fires + // (the source name scripts the cancellation). + last_turn: "u2".into(), + fallback_history: Some( + "Continue this side chat using the following prior transcript as context.\n\ + \nuser: Remember PINEAPPLE\n\n\n" + .into(), + ), + }); + let events = run_to_end(&harness(), req, controls).await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + let session = session_file_of(&events); + assert!(session.ends_with(".pi-fixture-session.jsonl"), "{session}"); + assert_eq!(text_of(&events), "reply:fork-fallback"); + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::Error { message, .. } if message.contains("pi could not fork") + )), + "{events:?}" + ); +} + +/// A fork whose source session file is gone never spawns the fork process: +/// the run starts fresh with the budgeted paste (the same fallback as a +/// lost resume pointer). +#[tokio::test] +async fn a_fork_with_a_missing_source_file_falls_back_to_the_budgeted_paste() { + let cwd = tempfile::tempdir().unwrap(); + let (controls, steer, _token) = controls(); + drop(steer); + let mut req = request_cwd("fork-fallback", &cwd.path().display().to_string()); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: cwd.path().join("no-such-session.jsonl").display().to_string(), + last_turn: "u1".into(), + fallback_history: Some( + "Continue this side chat using the following prior transcript as context.\n\ + \nuser: Remember PINEAPPLE\n\n\n" + .into(), + ), + }); + let events = run_to_end(&harness(), req, controls).await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + assert_eq!(text_of(&events), "reply:fork-fallback"); + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::Error { message, .. } if message.contains("pi could not fork") + )), + "{events:?}" + ); + // No fork process ever ran. + assert!(!cwd.path().join(".pi-fixture-forked.jsonl").exists()); +} + +/// A fork RPC pi REFUSES (success: false) falls back the same way: fresh +/// session, budgeted transcript, visible notice. +#[tokio::test] +async fn a_refused_fork_falls_back_to_the_budgeted_paste() { + let cwd = tempfile::tempdir().unwrap(); + let source = cwd.path().join("fork-refuse-source.jsonl"); + std::fs::write(&source, b"fixture session\n").unwrap(); + let (controls, steer, _token) = controls(); + drop(steer); + let mut req = request_cwd("fork-fallback", &cwd.path().display().to_string()); + req.provider_fork = Some(roboco_proto::ProviderFork { + session_id: source.display().to_string(), + // An earlier-turn boundary, so the refused RPC actually fires + // (the source name scripts the refusal). + last_turn: "u2".into(), + fallback_history: Some( + "Continue this side chat using the following prior transcript as context.\n\ + \nuser: Remember PINEAPPLE\n\n\n" + .into(), + ), + }); + let events = run_to_end(&harness(), req, controls).await; + assert_eq!(dones(&events), vec![(DoneStatus::Completed, None)], "{events:?}"); + let session = session_file_of(&events); + assert!(session.ends_with(".pi-fixture-session.jsonl"), "{session}"); + assert_eq!(text_of(&events), "reply:fork-fallback"); + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::Error { message, .. } if message.contains("pi could not fork") + )), + "{events:?}" + ); + // The refused RPC still ran (its marker proves the refusal, not a + // silent skip). + let forked = std::fs::read_to_string(cwd.path().join(".pi-fixture-forked.jsonl")).unwrap(); + assert!(forked.contains("fork-rpc:u3"), "{forked}"); +} diff --git a/crates/harness/tests/real_acp_lifecycle.rs b/crates/harness/tests/real_acp_lifecycle.rs index 724a76c245..935ef87328 100644 --- a/crates/harness/tests/real_acp_lifecycle.rs +++ b/crates/harness/tests/real_acp_lifecycle.rs @@ -33,6 +33,8 @@ async fn live_run(cancel: bool) { harness: None, model: None, reasoning: None, model_options: serde_json::Map::new(), cwd: cwd.path().display().to_string(), sandbox: SandboxLevel::WorkspaceWrite, auto_approve: true, + provider_fork: None, + resume_fallback_history: None, attachments: Vec::new(), worktree: None, resume: None, mcp: None, }; let mut stream = PiHarness::new() diff --git a/crates/harness/tests/real_quiet_survey.rs b/crates/harness/tests/real_quiet_survey.rs index 451f239731..20e8fa6b2d 100644 --- a/crates/harness/tests/real_quiet_survey.rs +++ b/crates/harness/tests/real_quiet_survey.rs @@ -68,6 +68,8 @@ async fn probe_once(harness: AcpHarness) -> ProbeOutcome { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, }; let mut stream = match harness.run(req, controls).await { Ok(s) => s, diff --git a/crates/harness/tests/windows_native.rs b/crates/harness/tests/windows_native.rs index 8f57b14078..ea8a6c36bf 100644 --- a/crates/harness/tests/windows_native.rs +++ b/crates/harness/tests/windows_native.rs @@ -215,6 +215,8 @@ fn request(cwd: &Path, prompt: &str, resume: Option<&str>) -> RunRequest { worktree: None, mcp: None, resume: resume.map(str::to_owned), + provider_fork: None, + resume_fallback_history: None, } } diff --git a/crates/mobile-client/src/cache.rs b/crates/mobile-client/src/cache.rs index b0a95cb786..827a29132d 100644 --- a/crates/mobile-client/src/cache.rs +++ b/crates/mobile-client/src/cache.rs @@ -353,6 +353,7 @@ mod tests { "engine", &format!("chat-{i}"), &[SessionMessageEntry { + origin_chat_id: None, id: format!("m{i}"), role: roboco_doc::MessageRole::User, parts: vec![], @@ -360,6 +361,7 @@ mod tests { device_id: String::new(), status: None, continuation_of: None, + provider_turn: None, duration_ms: None, }], ) diff --git a/crates/mobile-client/src/clock.rs b/crates/mobile-client/src/clock.rs index b003df8c6d..4cac9b87e7 100644 --- a/crates/mobile-client/src/clock.rs +++ b/crates/mobile-client/src/clock.rs @@ -83,6 +83,8 @@ mod tests { status, started_at: Some(updated - TimeDelta::minutes(5)), updated_at: updated, + limit: None, + scheduled_resume: None, } } diff --git a/crates/mobile-client/src/new_session.rs b/crates/mobile-client/src/new_session.rs index 616cedcae7..9514adedcd 100644 --- a/crates/mobile-client/src/new_session.rs +++ b/crates/mobile-client/src/new_session.rs @@ -104,6 +104,7 @@ impl PendingChat { /// merged engine row so `build_row` needs no special case. pub(crate) fn synthetic_chat(&self, chat_id: &str) -> roboco_proto::Chat { roboco_proto::Chat { + agent_origin: None, id: chat_id.to_owned(), device_id: self.device_id.clone(), title: None, diff --git a/crates/mobile-client/src/notify.rs b/crates/mobile-client/src/notify.rs index d57b14faa5..ebc0c49f69 100644 --- a/crates/mobile-client/src/notify.rs +++ b/crates/mobile-client/src/notify.rs @@ -692,11 +692,14 @@ mod tests { status, started_at: Some(updated - chrono::TimeDelta::minutes(5)), updated_at: updated, + limit: None, + scheduled_resume: None, } } fn chat(raw: &str, title: Option<&str>, parent: Option<&str>) -> Chat { Chat { + agent_origin: None, id: raw.into(), device_id: "engine".into(), created_at: Utc::now(), diff --git a/crates/mobile-client/src/session/commands.rs b/crates/mobile-client/src/session/commands.rs index 027098031b..c4fb1c4036 100644 --- a/crates/mobile-client/src/session/commands.rs +++ b/crates/mobile-client/src/session/commands.rs @@ -220,6 +220,8 @@ impl SessionCore { // background subagents. auto_approve: false, resume: None, + provider_fork: None, + resume_fallback_history: None, attachments: paths.to_vec(), worktree: request.worktree.clone(), mcp: None, diff --git a/crates/mobile-client/src/session/composer.rs b/crates/mobile-client/src/session/composer.rs index d36000f5f5..ccda754a68 100644 --- a/crates/mobile-client/src/session/composer.rs +++ b/crates/mobile-client/src/session/composer.rs @@ -338,6 +338,8 @@ mod tests { status, started_at: Some(updated - TimeDelta::minutes(5)), updated_at: updated, + limit: None, + scheduled_resume: None, } } @@ -405,6 +407,7 @@ mod tests { #[test] fn open_input_finds_the_newest_unresolved_question() { let entry = |id: &str, parts: Vec| Arc::new(SessionMessageEntry { + origin_chat_id: None, id: id.into(), role: MessageRole::Assistant, parts, @@ -412,6 +415,7 @@ mod tests { device_id: String::new(), status: None, continuation_of: None, + provider_turn: None, duration_ms: None, }); let question = |id: &str| roboco_proto::UserInputQuestion { diff --git a/crates/mobile-client/src/session/mod.rs b/crates/mobile-client/src/session/mod.rs index e78d75a743..b99eb169b0 100644 --- a/crates/mobile-client/src/session/mod.rs +++ b/crates/mobile-client/src/session/mod.rs @@ -1685,6 +1685,7 @@ mod tests { /// One assistant entry carrying the given tool parts. fn entry(parts: Vec) -> Arc { Arc::new(SessionMessageEntry { + origin_chat_id: None, id: "e1".into(), role: roboco_doc::MessageRole::Assistant, parts, @@ -1692,6 +1693,7 @@ mod tests { device_id: "dev".into(), status: Some(roboco_doc::MessageStatus::Complete), continuation_of: None, + provider_turn: None, duration_ms: None, }) } diff --git a/crates/mobile-client/src/session/transcript.rs b/crates/mobile-client/src/session/transcript.rs index ef9c4a328b..bc09a96225 100644 --- a/crates/mobile-client/src/session/transcript.rs +++ b/crates/mobile-client/src/session/transcript.rs @@ -535,6 +535,7 @@ mod tests { fn entry(id: &str, text: &str) -> SessionMessageEntry { SessionMessageEntry { + origin_chat_id: None, id: id.into(), role: roboco_doc::MessageRole::User, parts: vec![MessagePart::Text { @@ -545,6 +546,7 @@ mod tests { device_id: String::new(), status: None, continuation_of: None, + provider_turn: None, duration_ms: None, } } diff --git a/crates/mobile-client/src/workspace/change_requests.rs b/crates/mobile-client/src/workspace/change_requests.rs index 6048050aec..4f88e066dd 100644 --- a/crates/mobile-client/src/workspace/change_requests.rs +++ b/crates/mobile-client/src/workspace/change_requests.rs @@ -304,6 +304,7 @@ mod tests { fn chat(id: &str, branch: Option<&str>, archived: bool) -> Chat { Chat { + agent_origin: None, id: id.into(), device_id: "dev-1".into(), title: None, diff --git a/crates/mobile-client/src/workspace/view.rs b/crates/mobile-client/src/workspace/view.rs index 91e6351b52..8df6e24969 100644 --- a/crates/mobile-client/src/workspace/view.rs +++ b/crates/mobile-client/src/workspace/view.rs @@ -1250,6 +1250,7 @@ mod tests { fn chat(id: &str) -> Chat { Chat { + agent_origin: None, id: id.into(), device_id: "dev-1".into(), title: None, @@ -1293,6 +1294,8 @@ mod tests { status, started_at: Some(at(updated)), updated_at: at(updated), + limit: None, + scheduled_resume: None, } } diff --git a/crates/mobile-client/tests/notify_engine.rs b/crates/mobile-client/tests/notify_engine.rs index 1c1add72fe..153e048d11 100644 --- a/crates/mobile-client/tests/notify_engine.rs +++ b/crates/mobile-client/tests/notify_engine.rs @@ -328,6 +328,8 @@ fn run_request(prompt: &str) -> roboco_proto::RunRequest { worktree: None, mcp: None, resume: None, + provider_fork: None, + resume_fallback_history: None, } } @@ -432,6 +434,7 @@ async fn side_chats_never_emit() { None, None, Some("same-chat".to_owned()), + None, ) .unwrap(); let side = rig.client.scope(&rig.engine_id, "side-chat"); diff --git a/crates/mobile-client/tests/real_engine.rs b/crates/mobile-client/tests/real_engine.rs index a5ed8de5ea..c5de41c80e 100644 --- a/crates/mobile-client/tests/real_engine.rs +++ b/crates/mobile-client/tests/real_engine.rs @@ -768,7 +768,7 @@ async fn scoped_ids_feed_watches_and_transcript_connections_work() { core.doc_host .open("same-chat") .unwrap() - .write_user_message("m1", "hello", 123) + .write_user_message("m1", "hello", 123, None) .unwrap(); let listener = serve(&core, engine_dir.path()).await; let url = pairing_url(&listener, engine_dir.path(), None); @@ -929,7 +929,7 @@ async fn large_reset_streams_intact_while_calls_stay_fast() { core.doc_host .open("same-chat") .unwrap() - .write_user_message("big-message", &big, 123) + .write_user_message("big-message", &big, 123, None) .unwrap(); let listener = serve(&core, engine_dir.path()).await; let url = pairing_url(&listener, engine_dir.path(), None); diff --git a/crates/mobile-client/tests/side_chats_engine.rs b/crates/mobile-client/tests/side_chats_engine.rs index a3bc8aba8a..7e22b58030 100644 --- a/crates/mobile-client/tests/side_chats_engine.rs +++ b/crates/mobile-client/tests/side_chats_engine.rs @@ -270,6 +270,7 @@ async fn rig_with_background( }), Some("/checkout/parent".into()), None, + None, ) .unwrap(); let listener = serve(&core, engine_dir.path()).await; @@ -691,6 +692,7 @@ async fn the_subagent_doc_streams_and_counts_while_it_runs() { .unwrap() .doc() .push_message(&roboco_doc::schema::SessionMessageEntry { + origin_chat_id: None, duration_ms: None, id: "resumed".into(), role: roboco_doc::MessageRole::Assistant, @@ -701,6 +703,7 @@ async fn the_subagent_doc_streams_and_counts_while_it_runs() { created_at: 2_000, device_id: rig.core.device_id.clone(), status: Some(roboco_doc::MessageStatus::Complete), + provider_turn: None, continuation_of: None, }) .unwrap(); diff --git a/crates/mobile/src/layout/mod.rs b/crates/mobile/src/layout/mod.rs index 5f41b03c78..c97c3c3c35 100644 --- a/crates/mobile/src/layout/mod.rs +++ b/crates/mobile/src/layout/mod.rs @@ -541,6 +541,7 @@ pub(crate) fn debug_input(entries: Vec, working: bool) -> Transcript .into_iter() .map(|e| { Arc::new(SessionMessageEntry { + origin_chat_id: None, parts: vec![MessagePart::Text { id: "t0".into(), text: e.text, @@ -551,6 +552,7 @@ pub(crate) fn debug_input(entries: Vec, working: bool) -> Transcript device_id: String::new(), status: Some(if e.streaming { MessageStatus::Streaming } else { MessageStatus::Complete }), continuation_of: None, + provider_turn: None, duration_ms: None, }) }) diff --git a/crates/mobile/src/layout/tests.rs b/crates/mobile/src/layout/tests.rs index f7964d1be2..b40f3a9010 100644 --- a/crates/mobile/src/layout/tests.rs +++ b/crates/mobile/src/layout/tests.rs @@ -244,6 +244,7 @@ fn thinking_renders_styled_markdown_not_markers() { let mut w = worker(390.0); w.input = TranscriptInput { entries: vec![Arc::new(SessionMessageEntry { + origin_chat_id: None, id: "a".into(), role: MessageRole::Assistant, parts: vec![MessagePart::Reasoning { id: "r0".into(), text: reasoning.into() }], @@ -251,6 +252,7 @@ fn thinking_renders_styled_markdown_not_markers() { device_id: String::new(), status: Some(MessageStatus::Complete), continuation_of: None, + provider_turn: None, duration_ms: None, })], ..Default::default() @@ -284,6 +286,7 @@ fn thinking_renders_styled_markdown_not_markers() { let mut w = worker(width); w.input = TranscriptInput { entries: vec![Arc::new(SessionMessageEntry { + origin_chat_id: None, id: "a".into(), role: MessageRole::Assistant, parts: vec![MessagePart::Reasoning { id: "r0".into(), text: reasoning.into() }], @@ -291,6 +294,7 @@ fn thinking_renders_styled_markdown_not_markers() { device_id: String::new(), status: Some(MessageStatus::Complete), continuation_of: None, + provider_turn: None, duration_ms: None, })], ..Default::default() @@ -317,6 +321,7 @@ fn streaming_thought_markdown_settles_to_the_fresh_parse() { let text = "**Checking** the `parser`\n\n1. first step\n2. second with [docs](https://example.com)\n\n> note\n\n```rust\nfn main() {}\n```"; let entry = |status: MessageStatus, text: &str| { Arc::new(SessionMessageEntry { + origin_chat_id: None, id: "a".into(), role: MessageRole::Assistant, parts: vec![MessagePart::Reasoning { id: "r0".into(), text: text.to_owned() }], @@ -324,6 +329,7 @@ fn streaming_thought_markdown_settles_to_the_fresh_parse() { device_id: String::new(), status: Some(status), continuation_of: None, + provider_turn: None, duration_ms: None, }) }; @@ -367,6 +373,7 @@ fn thought_input(text: &str, streaming: bool) -> TranscriptInput { use roboco_doc::schema::{MessageRole, SessionMessageEntry}; TranscriptInput { entries: vec![Arc::new(SessionMessageEntry { + origin_chat_id: None, id: "a".into(), role: MessageRole::Assistant, parts: vec![MessagePart::Reasoning { id: "r0".into(), text: text.to_owned() }], @@ -374,6 +381,7 @@ fn thought_input(text: &str, streaming: bool) -> TranscriptInput { device_id: String::new(), status: Some(if streaming { MessageStatus::Streaming } else { MessageStatus::Complete }), continuation_of: None, + provider_turn: None, duration_ms: None, })], ..Default::default() @@ -509,6 +517,7 @@ fn bench_layout_passes() { text.push_str(["word ", "more ", "text\n\n", "`code` ", "**bold** "][i % 5]); let mut e = entries.clone(); e.push(Arc::new(SessionMessageEntry { + origin_chat_id: None, id: "live".into(), role: MessageRole::Assistant, parts: vec![MessagePart::Text { id: "t0".into(), text: text.clone() }], @@ -516,6 +525,7 @@ fn bench_layout_passes() { device_id: String::new(), status: Some(MessageStatus::Streaming), continuation_of: None, + provider_turn: None, duration_ms: None, })); w.input = TranscriptInput { entries: e, pending: vec![], working: true, working_since_ms: None, streaming: true }; @@ -642,6 +652,7 @@ fn running_subagent_shows_a_spinner_after_its_spawn_resolves() { let mut w = worker(390.0); w.input = TranscriptInput { entries: vec![Arc::new(SessionMessageEntry { + origin_chat_id: None, id: "a".into(), role: MessageRole::Assistant, parts: vec![spawn(status)], @@ -649,6 +660,7 @@ fn running_subagent_shows_a_spinner_after_its_spawn_resolves() { device_id: String::new(), status: Some(MessageStatus::Complete), continuation_of: None, + provider_turn: None, duration_ms: None, })], ..Default::default() @@ -693,6 +705,7 @@ fn spawn_cards_with_a_doc_ref_become_subagent_links() { let mut w = worker(390.0); w.input = TranscriptInput { entries: vec![Arc::new(SessionMessageEntry { + origin_chat_id: None, id: "a".into(), role: MessageRole::Assistant, parts: vec![part], @@ -700,6 +713,7 @@ fn spawn_cards_with_a_doc_ref_become_subagent_links() { device_id: String::new(), status: Some(MessageStatus::Complete), continuation_of: None, + provider_turn: None, duration_ms: None, })], ..Default::default() diff --git a/crates/proto/src/agent.rs b/crates/proto/src/agent.rs index b743f02728..b7ba2e960c 100644 --- a/crates/proto/src/agent.rs +++ b/crates/proto/src/agent.rs @@ -29,6 +29,18 @@ pub enum HarnessId { Mock, } +/// The continue prompt a Limited chat's **Resume** sends as a normal run +/// (orchestration v2): one shared spelling so a resumed transcript reads +/// the same from every surface that sends it (the desktop banner today; +/// the engine's scheduled resumes join it in later tickets). +pub const RESUME_CONTINUE_PROMPT: &str = "Continue where you left off."; + +/// The tail of the header the Roboco MCP server puts on a parent's message +/// to its notifying side chat (orchestration v2): `[Message from Roboco +/// chat