Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
"id": "compact_wx0ujsgwtaqd",
"version": 1,
"type": "compacted",
"compactedAt": "2026-09-22T18:45:48.437Z",
"sourceTrajectories": [
"traj_4yi8dr1zi7ud"
],
"dateRange": {
"start": "2026-09-22T18:37:03.545Z",
"end": "2026-09-22T18:45:47.737Z"
},
"summary": {
"totalDecisions": 1,
"totalEvents": 1,
"uniqueAgents": [
"default"
]
},
"decisionGroups": [
{
"category": "other",
"decisions": [
{
"question": "Keep action-result reconnect durability out of this fix",
"chosen": "Keep action-result reconnect durability out of this fix",
"reasoning": "Reviewed plan requires readiness contract repair plus diagnostics; engine replay idempotency is unproven.",
"fromTrajectory": "traj_4yi8dr1zi7ud"
}
]
}
],
"keyLearnings": [],
"keyFindings": [],
"filesAffected": [],
"commits": []
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Trajectory Compaction: Sep 22, 2026 - Sep 22, 2026

## Summary
- Sessions: 1
- Decisions: 1
- Events: 1
- Agents: default
- Files: 0
- Commits: 0

## Other
- Keep action-result reconnect durability out of this fix -> Keep action-result reconnect durability out of this fix (traj_4yi8dr1zi7ud)

## Key Learnings
- None

## Key Findings
- None
8 changes: 7 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,16 @@ All notable changes to Agent Relay will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased - Patch]
## [Unreleased - Minor]

### Changed

- Targeted `fleet spawn` now waits for harness readiness; the broker releases workers that miss its 90-second readiness window. Confirmed targeted spawns require `--confirm-timeout` of at least 95000ms.

### Fixed

- Targeted `fleet spawn` requests explicit readiness proof, preventing healthy launches from being rejected for missing proof; unconfirmed launches report `ready:false` while obsolete handlers remain rejected.
- `@agent-relay/sdk` `placement.spawn` only asks a node to verify readiness when it will wait for the answer, and confirms against the contract it requested, so `confirm` omitted no longer arms a 90-second readiness kill switch and `verifyReady: false` no longer fails a healthy launch.
- Broker `manual_flush` recovery now replays a missing cumulative-ACK predecessor without duplicating an already-completed PTY injection, restores it ahead of parked successors, and reports the head/ACK/received sequence gap plus the reconciliation action in `message flush` and `message auto` results.

## [12.4.1] - 2026-09-22
Expand Down
12 changes: 11 additions & 1 deletion crates/broker/src/node_control.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1727,7 +1727,13 @@ fn handle_disconnected_command(
Some(FleetControlCommand::RegisterAgent { reply, .. }) => {
let _ = reply.send(Err(register_agent_error.to_string()));
}
Some(FleetControlCommand::Send(_)) | Some(FleetControlCommand::HeartbeatNow) => {}
Some(FleetControlCommand::Send(message)) => {
if let BrokerToRelaycast::ActionResult(result) = &message {
tracing::warn!(invocation_id = %result.invocation_id, frame_kind = "action.result",
"dropping fleet frame while disconnected");
}
}
Some(FleetControlCommand::HeartbeatNow) => {}
Some(FleetControlCommand::Shutdown) | None => return DisconnectedCommandOutcome::Shutdown,
}
DisconnectedCommandOutcome::Handled
Expand Down Expand Up @@ -2258,6 +2264,10 @@ where
}
}
if sent.is_err() {
if let BrokerToRelaycast::ActionResult(result) = &message {
tracing::warn!(invocation_id = %result.invocation_id, frame_kind = "action.result",
"dropping fleet frame after wire send failure");
}
return Break(ControlRunResult::Disconnected {
application_ready: application_liveness.ready,
});
Expand Down
2 changes: 1 addition & 1 deletion crates/broker/src/pty_worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ const STARTUP_READY_WARNING: Duration = Duration::from_secs(25);
/// that never receives its task is a total loss, while a brief typed a little
/// early is recoverable. Bounded below `WORKER_READY_DEADLINE` (90s) so the work
/// is released before an unready harness is reaped.
const STARTUP_READY_TIMEOUT: Duration = Duration::from_secs(60);
pub(crate) const STARTUP_READY_TIMEOUT: Duration = Duration::from_secs(60);
const STARTUP_BUFFER_MAX: usize = 12_000;
const STARTUP_BUFFER_KEEP: usize = 8_000;
const CODEX_STARTUP_SETTLE: Duration = Duration::from_secs(1);
Expand Down
34 changes: 32 additions & 2 deletions crates/broker/src/runtime/fleet.rs
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ pub(super) fn close_terminal_sessions_for_worker(
pub(super) struct PendingVerifiedSpawn {
pub(super) invocation_id: String,
pub(super) deadline: Instant,
pub(super) started: Instant,
pub(super) generation: Uuid,
}

Expand Down Expand Up @@ -1432,6 +1433,9 @@ impl BrokerRuntime {
/// to this node). Replies with `action.result { output }` on success or
/// `{ error }` on failure.
async fn handle_fleet_action_spawn(&mut self, invoke: ActionInvoke) {
let started = Instant::now();
let verify_ready = super::relaycast_events::relaycast_spawn_verifies_ready(&invoke.input);
tracing::info!(invocation_id = %invoke.invocation_id, verify_ready, "fleet spawn received");
let Some(name) = action_invoke_agent_name(&invoke) else {
self.reply_action_error(&invoke.invocation_id, "spawn_missing_agent_name")
.await;
Expand Down Expand Up @@ -1538,7 +1542,8 @@ impl BrokerRuntime {

self.publish_fleet_load(true).await;

let verify_ready = super::relaycast_events::relaycast_spawn_verifies_ready(&ws_value);
tracing::info!(invocation_id = %invoke.invocation_id, worker = %name, verify_ready,
elapsed_ms = started.elapsed().as_millis() as u64, "fleet spawn launch returned");

let spawn_outcome =
fleet_spawn_outcome(spawn_result, &name, self.workers.is_worker_live(&name));
Expand All @@ -1560,6 +1565,9 @@ impl BrokerRuntime {
(worker.ready_at.is_some(), worker.generation)
};
if already_ready {
tracing::info!(invocation_id = %invoke.invocation_id, worker = %name,
verify_ready, elapsed_ms = started.elapsed().as_millis() as u64,
"sending verified fleet spawn result");
self.send_fleet_action_result(verified_spawn_ready_result(
invoke.invocation_id,
&name,
Expand All @@ -1571,6 +1579,7 @@ impl BrokerRuntime {
PendingVerifiedSpawn {
invocation_id: invoke.invocation_id,
deadline: Instant::now() + VERIFIED_SPAWN_READY_TIMEOUT,
started,
generation,
},
);
Expand Down Expand Up @@ -1749,6 +1758,7 @@ impl BrokerRuntime {
}

async fn send_fleet_action_result(&self, result: ActionResult) {
tracing::info!(invocation_id = %result.invocation_id, "sending fleet action result");
let _ = self
.fleet_control_tx
.send(FleetControlCommand::Send(BrokerToRelaycast::ActionResult(
Expand Down Expand Up @@ -1929,14 +1939,15 @@ pub(super) fn confirm_pending_delivery_and_resolve_fleet_ack(
((!already_held).then_some(pending), resolved)
}

/// Every spawn success includes `ready`; true is reserved for proven harness readiness.
fn fleet_spawn_action_result(
invocation_id: &str,
name: &WorkerName,
spawn_result: Result<()>,
) -> ActionResult {
let result = match spawn_result {
Ok(()) => ActionResultPayload::Output(ActionResultOutput {
output: json!({ "spawned": true, "name": name.as_str() }),
output: json!({ "spawned": true, "ready": false, "name": name.as_str() }),
}),
Err(error) => ActionResultPayload::Error(ActionResultError {
error: format!("spawn_failed: {error:#}"),
Expand Down Expand Up @@ -3137,6 +3148,25 @@ mod tests {
);
}

#[test]
fn spawn_success_always_declares_readiness() {
let name = WorkerName::from("Probe");
let unverified = fleet_spawn_action_result("inv-launch", &name, Ok(()));
let verified = verified_spawn_ready_result("inv-ready".into(), &name);
for (result, ready) in [(unverified, false), (verified, true)] {
let ActionResultPayload::Output(output) = result.result else {
panic!("live spawn must succeed");
};
assert_eq!(
output.output,
json!({"spawned": true, "ready": ready, "name": "Probe"})
);
}
// CLI/SDK default confirmation budget is 120s (fleet.ts / relaycast.ts).
assert!(crate::pty_worker::STARTUP_READY_TIMEOUT < VERIFIED_SPAWN_READY_TIMEOUT);
assert!(VERIFIED_SPAWN_READY_TIMEOUT < Duration::from_secs(120));
}

fn test_agent_spec(session_id: Option<&str>, harness_session_id: Option<&str>) -> AgentSpec {
AgentSpec {
name: WorkerName::from("agent-a"),
Expand Down
9 changes: 8 additions & 1 deletion crates/broker/src/runtime/maintenance.rs
Original file line number Diff line number Diff line change
Expand Up @@ -234,8 +234,15 @@ impl BrokerRuntime {
let owned = workers.owned_spawn_generations.get(name).cloned();
let completion = super::fleet::verified_spawn_failed_result(
invocation_id.clone(),
"spawn_readiness_timeout",
"spawn_readiness_timeout: worker released after failing to reach harness readiness",
);
// This result goes out through identity cleanup or the fleet channel
// directly rather than `send_fleet_action_result`, so it would
// otherwise be the one spawn outcome missing the correlation log —
// and it is the outcome most likely to be investigated.
tracing::info!(invocation_id = %invocation_id, worker = %name, verify_ready = true,
deferred_to_identity_cleanup = owned.is_some(),
"sending fleet action result");
if let Some((_, http)) = owned {
super::identity_cleanup::schedule_identity_cleanup(
workers,
Expand Down
13 changes: 13 additions & 0 deletions crates/broker/src/runtime/relaycast_events.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1478,6 +1478,19 @@ mod tests {
assert!(error.contains("harnessId is not supported"));
}

#[test]
fn placement_spawn_requests_harness_readiness() {
// Literal SDK placementActionInput payload; persona stays engine-owned.
let mut payload = json!({"capability":"spawn:claude", "cli":"claude",
"node":"node-a", "target_node":"node-a", "name":"Probe", "verify_ready":true});
assert!(relaycast_spawn_verifies_ready(&payload));
payload.as_object_mut().unwrap().remove("verify_ready");
assert!(!relaycast_spawn_verifies_ready(&payload));
payload["capability"] = json!("spawn:persona");
payload["cli"] = json!("persona");
assert!(!relaycast_spawn_verifies_ready(&payload));
}

#[test]
fn verified_spawn_contract_is_read_from_request_or_harness_metadata() {
let verified = json!({
Expand Down
3 changes: 3 additions & 0 deletions crates/broker/src/runtime/worker_events.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1391,6 +1391,9 @@ impl BrokerRuntime {
.then(|| pending_verified_spawns.remove(&name))
.flatten();
if let Some(pending) = pending {
tracing::info!(invocation_id = %pending.invocation_id, worker = %name,
verify_ready = true, elapsed_ms = pending.started.elapsed().as_millis() as u64,
"sending verified fleet spawn result");
let _ = fleet_control_tx
.send(FleetControlCommand::Send(
crate::fleet_wire::BrokerToRelaycast::ActionResult(
Expand Down
Loading
Loading