diff --git a/.agentworkforce/trajectories/compacted/compact_1akx23pm5dm3_2026-09-19.json b/.agentworkforce/trajectories/compacted/compact_1akx23pm5dm3_2026-09-19.json new file mode 100644 index 00000000..1134b6a8 --- /dev/null +++ b/.agentworkforce/trajectories/compacted/compact_1akx23pm5dm3_2026-09-19.json @@ -0,0 +1,23 @@ +{ + "id": "compact_1akx23pm5dm3", + "version": 1, + "type": "compacted", + "compactedAt": "2026-09-19T05:15:58.999Z", + "sourceTrajectories": [ + "traj_c5xu27ivwl60" + ], + "dateRange": { + "start": "2026-09-19T05:13:58.329Z", + "end": "2026-09-19T05:15:55.144Z" + }, + "summary": { + "totalDecisions": 0, + "totalEvents": 0, + "uniqueAgents": [] + }, + "decisionGroups": [], + "keyLearnings": [], + "keyFindings": [], + "filesAffected": [], + "commits": [] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/compacted/compact_1akx23pm5dm3_2026-09-19.md b/.agentworkforce/trajectories/compacted/compact_1akx23pm5dm3_2026-09-19.md new file mode 100644 index 00000000..ff36e1a4 --- /dev/null +++ b/.agentworkforce/trajectories/compacted/compact_1akx23pm5dm3_2026-09-19.md @@ -0,0 +1,18 @@ +# Trajectory Compaction: Sep 18, 2026 - Sep 18, 2026 + +## Summary +- Sessions: 1 +- Decisions: 0 +- Events: 0 +- Agents: None +- Files: 0 +- Commits: 0 + +## Decision Groups +- None + +## Key Learnings +- None + +## Key Findings +- None \ No newline at end of file diff --git a/.agentworkforce/trajectories/compacted/compact_yy5pk8tn1xrh_2026-09-19.json b/.agentworkforce/trajectories/compacted/compact_yy5pk8tn1xrh_2026-09-19.json new file mode 100644 index 00000000..9f719200 --- /dev/null +++ b/.agentworkforce/trajectories/compacted/compact_yy5pk8tn1xrh_2026-09-19.json @@ -0,0 +1,23 @@ +{ + "id": "compact_yy5pk8tn1xrh", + "version": 1, + "type": "compacted", + "compactedAt": "2026-09-19T05:05:50.727Z", + "sourceTrajectories": [ + "traj_trmwuy4eqhbh" + ], + "dateRange": { + "start": "2026-09-19T05:03:48.917Z", + "end": "2026-09-19T05:05:46.848Z" + }, + "summary": { + "totalDecisions": 0, + "totalEvents": 0, + "uniqueAgents": [] + }, + "decisionGroups": [], + "keyLearnings": [], + "keyFindings": [], + "filesAffected": [], + "commits": [] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/compacted/compact_yy5pk8tn1xrh_2026-09-19.md b/.agentworkforce/trajectories/compacted/compact_yy5pk8tn1xrh_2026-09-19.md new file mode 100644 index 00000000..ff36e1a4 --- /dev/null +++ b/.agentworkforce/trajectories/compacted/compact_yy5pk8tn1xrh_2026-09-19.md @@ -0,0 +1,18 @@ +# Trajectory Compaction: Sep 18, 2026 - Sep 18, 2026 + +## Summary +- Sessions: 1 +- Decisions: 0 +- Events: 0 +- Agents: None +- Files: 0 +- Commits: 0 + +## Decision Groups +- None + +## Key Learnings +- None + +## Key Findings +- None \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 553ea125..c1910840 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,11 @@ This project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). Packages without a separate changelog are covered by the cross-package notes below. -## [Unreleased] +## [Unreleased - Patch] + +### Fixed + +- A node that reconnects after its delivery socket dropped now receives every message queued during the outage. Reconnect replay for cursor-aware brokers was scoped to the sessions whose delivery readiness or node routing changed while re-announcing their inventory, so a host that already reported those sessions as delivery-ready replayed nothing: the queued messages stayed in the mailbox until their TTL and the agents were never woken. A heartbeat that refreshes the broker's capability roster between reconnect and re-announcement no longer changes how that reconnect replays. ## [8.11.3] - 2026-09-19 diff --git a/FLOW_NOTES.md b/FLOW_NOTES.md new file mode 100644 index 00000000..2c4a2e6c --- /dev/null +++ b/FLOW_NOTES.md @@ -0,0 +1,233 @@ +# Node reconnect delivery replay — findings + +Branch: `fix/node-reconnect-delivery-replay` +Incident: workspace `rw_7ccfea89`, `cast.agentrelay.com`, engine 8.11.3 — a message +fanned out to a channel at 04:27 while a node's delivery socket was down was never +pushed after the node reconnected at 04:33. + +## Root cause + +`inventory.sync` (`packages/engine/src/engine/node.ts`, `handleNodeControlMessage`) +was the reconnect-replay trigger for cursor-negotiated brokers, but it derived its +replay scope from a *state transition* instead of from the certification itself: + +```ts +const newlyReadyAgentIds = result.reconciledAgentIds.filter((agentId) => + !isProviderAgentDeliveryReady(registry, workspaceId, nodeId, providerName, agentId)); +... +const replayAgentIds = [...new Set([...newlyReadyAgentIds, ...result.newlyRoutedAgentIds])]; +await deliverPendingToNode(db, registry, workspaceId, nodeId, { providerName, agentIds: replayAgentIds }); +``` + +Why that strands a reconnect: + +- Per the PR #443 contract, a cursor-negotiated `node.register` deliberately does + **not** replay (`if (!cursorHandshake)` guards the drain), because no identity is + cursor-ready yet. The certification frame is therefore the node's *only* + reconnect-replay trigger. +- `newlyReadyAgentIds` is non-empty only when the registry reports the listed + identities as *not yet* delivery-ready at sync time. That holds for the in-process + adapter (a reconnect creates a fresh `NodeConn` with an empty ready-set) — which is + why every in-tree reconnect test passed — but not for a socket owner whose + ready-set is keyed per node+provider rather than per connection. (A registry + that omits the optional readiness hooks is not affected: `node.register` + rejects its cursor negotiation, so it stays on immediate replay.) +- `newlyRoutedAgentIds` is non-empty only when the agent's binding *moved*. After a + transport-only reconnect the agent row still points at the same node/provider, so + it is empty too. + +Both terms empty ⇒ `agentIds: []` ⇒ `deliverPendingToNode` short-circuits +(`if (wantedIds?.size === 0) return 0`). Nothing is replayed, nothing is logged, and +the queued rows sit in the mailbox until their TTL. The engine had made replay of a +certified session contingent on registry bookkeeping it does not own. + +Reproduced in-process before the fix (scratch test, now folded into the conformance +suite): reconnect a cursor-negotiated node whose identities the socket owner already +reports delivery-ready, sync the inventory ⇒ 0 deliver frames, delivery row still +`queued`. With the fix ⇒ the frame is pushed and the row moves to `delivered`. + +This also matches the live shape exactly: the message stayed durably in the channel, +the node came back online, no `deliver` frame ever reached the broker, and a later +message would not have unblocked it either — the broker's monotonic-seq gate holds +any higher `seq` as a Gap while `seq 1` is missing. + +## Fix + +`packages/engine/src/engine/node.ts`, `inventory.sync` case: + +- **Cursor-negotiated connection** → replay the full certified set + (`result.reconciledAgentIds`), exactly as `agent.register` / `agent.recover` replay + the single identity they just made ready. This is what README already documents: + "an `inventory.sync` certifies that the listed provider-owned sessions retained + their in-memory cursors and may replay". +- **Legacy immediate-delivery connection** → unchanged behaviour: `node.register` + already flushed the whole node to it and nothing gates its later sends, so only + identities this sync newly routed to the node are replayed. Replaying more there + would duplicate the register-time flush (two existing conformance tests assert the + exactly-once frame counts and caught this during development). +- The handshake mode is recovered from the connection it was negotiated on — + see the review round below, which replaced the first cut of this (an inference + from the provider's persisted capabilities) after it proved sensitive to + heartbeat roster updates. +- The dropped `isProviderAgentDeliveryReady` filter was provably a no-op on the + legacy branch (immediate mode reports every identity ready), so removing it changes + nothing there. + +Invariants preserved: dedupe is still the cumulative delivery cursor +(`seq > agents.delivery_ack_seq`, status in `queued`/`delivered`), ordering is still +ascending `seq` with bounded 50-row pages under a per-identity high-water mark, and +`replayPendingToNode` still re-checks `isProviderAgentDeliveryReady` before *every* +frame, so a certified-but-not-ready identity is still never flushed (PR #443). + +### Callsites touched + +- `packages/engine/src/engine/node.ts` + - `handleNodeControlMessage` → `case 'inventory.sync'`: replay scope (the fix). + - new `providerAdvertisesDeliveryCursor()`; `registerAgentViaNode` and + `recoverAgentViaNode` now use it in place of their two duplicated inline + capability lookups (behaviour identical). +- `packages/engine/src/__tests__/conformance/delivery.test.ts`: new + `reconnect replay of an outage backlog` block (4 tests). Three of them fail on the + pre-fix code; the acked-not-re-sent one is a guard that must pass both ways. +- `CHANGELOG.md`, `packages/engine/CHANGELOG.md`: `[Unreleased - Patch]` entries. + +Untouched: `openapi.yaml`, `README.md` — no wire-visible change. The fix makes +behaviour match the reconnect/replay contract README already states. + +## Paths checked and cleared + +- `node-reconnect.ts` / `handleNodeReconnect` → thin delegate to + `deliverPendingToNode` with node-wide scope; correct, and unaffected. +- Channel fanout **does** create per-node pending rows: `buildChannelDeliveryWrite` + inserts one `deliveries` row per channel member regardless of socket state, with + `route_node_*` resolved from the active binding. Root-cause shape 3 ruled out. +- `deliverPendingToNode` itself (bounded drain, in-flight coalescing, readiness and + ACK re-checks per page and per row) behaves correctly; the bug was purely in the + scope its caller passed. + +## Remaining risks / follow-ups (not fixed here) + +1. **Duplicate-connection arbitration vs. a half-open socket.** Reproduced in-process: + if the engine never observes the old socket's close, the fan-out's live send lands + in the dead socket, `sendToProvider` returns `true`, and the row is marked + `delivered` — which makes it invisible to `sweepDueNodeDeliveries`, whose candidate + query filters `status = 'queued'`. Reconnect replay is then the only recovery path + (it does include `delivered`-but-unacked rows, so the fix above recovers these once + the surviving connection certifies its sessions). +2. **Reconnect classified as a duplicate instance.** For a broker that sends no + `provider` identity, `resolveProviderIdentity` synthesizes `instance_id` from the + *connection id*, so every reconnect looks like a different instance. If the dead + incumbent framed within `PROVIDER_ATTACH_LIVENESS_MS` (35 s), the reconnecting + socket's `node.register` is rejected `provider_instance_conflict` and cannot bind + at all; it self-heals only once the incumbent goes stale. Left as-is deliberately: + with no client-asserted instance id the engine genuinely cannot separate a + reconnect from a second live process, and the current policy (spec §3.1, shared + with the cloud NodeDO via `providerAttachDecision`) fails closed. The real fix is + for provider-less brokers to assert a stable `instance_id`. +3. **Repeated certifications re-send unacked frames.** On a cursor-negotiated + connection, a second `inventory.sync` before the node acks will re-push rows with + `seq > delivery_ack_seq` (including ones already marked `delivered`). That is the + documented at-least-once contract and the broker's monotonic-seq gate drops the + duplicate, but a broker that certifies on a short timer will see redundant frames. + If that shows up in production, gate the certified replay on a per-connection + "already resumed" marker rather than re-narrowing the scope. +4. The out-of-process socket owner (relaycast-cloud NodeDO) is not testable from this + repo. The fix removes the engine's dependence on that owner's readiness + bookkeeping, so it is correct for either mirroring choice — but confirming the + NodeDO's ready-set lifetime across reconnects is still worth doing. + +## Verification + +- `npx vitest run` in `packages/engine`: 99 files, 1148 tests passed. +- `npm run typecheck` (engine) and `npm run lint` (engine): clean. +- `npx turbo test` (repo): 18/18 tasks successful. +- New tests confirmed failing on the pre-fix `node.ts` (3 of 4, by design). +- Note for local runs: `better-sqlite3@11` has no bindings for Node 26 — the suite + was run on Node 22.23.2. + +## Review round + +Reviewer verdict (`REVIEW_VERDICT.json`) on `7d5eceb4`: not approved, one P1. + +### [P1] Replay scope inferred from mutable provider capabilities + +The first cut recovered the handshake mode with +`providerAdvertisesDeliveryCursor()`, reading `node_providers.capabilities` — +a row `heartbeatNode` rewrites wholesale whenever a `node.heartbeat` carries a +`capabilities` roster. The mode is a property of the *connection* +(`node.register` negotiates it and hands it to the registry per connection), so +deriving it from roster state let a heartbeat silently renegotiate it: + +- Cursor-negotiated reconnect → heartbeat advertising only `spawn:claude` → + `inventory.sync`: `cursorGated` false, `newlyRoutedAgentIds` empty, so the + certification replayed nothing — the exact strand the branch set out to fix, + reachable again through a different door. +- The inverse: a heartbeat adding `relay:delivery-cursor-v1` to an + immediate-delivery connection made a later sync replay the unacked frames + `node.register` had already flushed. + +Fixed in `dd6582d7`, on both axes the reviewer offered: + +1. **Connection-scoped mode (primary).** New optional + `NodeConnectionRegistry.providerDeliveryReadinessMode()` returns the mode + `setProviderDeliveryReadiness()` configured for the provider's *current* + connection (`undefined` when none is bound, or when the caller's + `connectionId` is no longer current). The in-process adapter reads it off the + same `NodeConn.deliveryReadyAgentIds` encoding readiness already uses — + `null` is immediate, a `Set` is agent-scoped, `undefined` is "never + registered on this connection". `inventory.sync` asks the registry through + the new `connectionNegotiatedDeliveryCursor()` helper. +2. **Registration-owned capability (fallback).** A registry on the older port + contract (the out-of-process `relaycast-cloud` NodeDO, which cannot be + updated from this repo) has no mode to report, so the helper still falls back + to the persisted advertisement — and `heartbeatNode` now keeps that + advertisement out of a heartbeat's reach: + `withRegisteredProtocolCapabilities()` carries the registered + `relay:delivery-cursor-v1` entry over a roster refresh and drops one a + heartbeat tries to introduce. Protocol capabilities are negotiated at + `node.register` and answered in its acceptance list; a heartbeat only + refreshes spawn/action capacity. This also stabilises the same inference in + `registerAgentViaNode` / `recoverAgentViaNode`, which decide whether their + reply carries `delivery_ack_seq`. + +Either fix alone holds the behaviour: with the adapter's new method stubbed out, +the whole delivery suite still passes on the fallback path (checked), which is +what certifies the out-of-process owner. + +### Coverage added + +Three tests in `delivery.test.ts` → `reconnect replay of an outage backlog`, all +three failing against the pre-fix `node.ts` (verified by swapping in `7d5eceb4`'s +file) and passing after: + +- `replays the certified backlog when a heartbeat roster omits the cursor + capability` — the reviewer's reproduction: cursor-negotiated reconnect, empty + connection readiness, roster-only heartbeat, then `inventory.sync` ⇒ 1 frame. +- `does not re-flush an immediate connection whose heartbeat roster adds the + cursor capability` — the inverse ⇒ still exactly the 1 register-time frame. +- `keeps the registered cursor advertisement out of reach of a heartbeat roster + refresh` — asserts the persisted `node_providers.capabilities` directly, so + the fallback axis is covered independently of the registry method. + +### Notes corrected (reviewer's non-blocking accuracy points) + +- The root-cause section and the engine changelog claimed a registry omitting + the optional readiness hooks was also stranded, "because the shared + `isProviderAgentDeliveryReady` helper defaults to `true`". Wrong: such a + registry is refused the cursor capability at `node.register` + (`delivery_readiness_unsupported`) and stays on immediate replay, which + flushes at register time. Both texts now say only what holds — an owner whose + ready-set is keyed per node+provider rather than per connection. +- Scope of this review, per the verdict: the engine's delivery path only. The + branch contains no CLI enrollment/claim changes, so CLI stale-claim and + crash-safety behaviour is not certified by it. Transport delivery remains + at-least-once by contract; the exactly-once test pins suppression after a + cumulative ACK, not broker-side dedupe. +- Follow-ups 1–4 above stand unchanged; none was in the blocking set. + +### Verify gate (re-run, Node 22.23.2) + +- `npx vitest run` in `packages/engine`: 99 files, 1151 tests passed (1148 + 3). +- `npm run typecheck` and `npm run lint` (engine): clean. +- `npx turbo test` (repo): 18/18 tasks successful. +- Not pushed. diff --git a/REVIEW_VERDICT.json b/REVIEW_VERDICT.json new file mode 100644 index 00000000..71c5b0df --- /dev/null +++ b/REVIEW_VERDICT.json @@ -0,0 +1,5 @@ +{ + "approved": true, + "blocking": [], + "notes": "Would ship the engine-only diff at dd6582d7 versus origin/main. Reviewed FLOW_NOTES.md, delivery replay/ACK/readiness paths, provider registration and heartbeat handling, inventory identity validation, and surrounding engine enrollment code. Certified cursor sessions now replay even when already marked ready; legacy sessions retain register-time replay without an extra inventory flush. Connection mode and heartbeat-preserved protocol capabilities protect both registry and compatibility-fallback paths. Pending queued and delivered-but-unacked rows remain replayable, with existing ownership, expiry, readiness and cumulative-ACK checks preserved. Verification on Node 22.23.2: 162 tests passed across delivery, node, nodeProviders and nodeRegistrationContract; engine typecheck passed. Isolated mutation checks: origin/main node.ts causes four of the seven new tests to fail at intended assertions; 7d5eceb4 causes all three heartbeat regressions to fail; current code with registry-mode lookup disabled passes all 66 delivery tests. Important scope: transport remains at-least-once. Repeated inventory.sync before ACK can resend unacknowledged frames; the exactly-once-named test ACKs before repeating sync and proves suppression after cumulative ACK, not broker-side exactly-once processing. No CLI enrollment/claim implementation changes are present, so CLI stale-claim and filesystem crash-safety behavior is not certified. The external cloud socket owner is not exercised directly; fallback behavior is exercised locally. Existing half-open socket and provider-instance arbitration limitations documented in FLOW_NOTES.md remain outside this fix." +} diff --git a/packages/engine/CHANGELOG.md b/packages/engine/CHANGELOG.md index 789c02a2..1fd44fa7 100644 --- a/packages/engine/CHANGELOG.md +++ b/packages/engine/CHANGELOG.md @@ -7,8 +7,9 @@ See the [root changelog](../../CHANGELOG.md) for cross-package release highlight The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Patch] +- Replay a reconnecting node's queued deliveries for every session its `inventory.sync` certifies, instead of only the sessions whose delivery readiness or provider routing changed during that sync. A cursor-negotiated connection gets no replay at `node.register` by contract, so the certification is its only reconnect-replay trigger; scoping it to a readiness transition stranded the whole outage backlog on any socket owner that already reported the listed identities as delivery-ready (a `NodeConnectionRegistry` whose ready-set is keyed per node and provider rather than per connection). Legacy immediate-delivery connections are unchanged — `node.register` still flushes the node to them, and a later sync replays only the identities it newly routed there. Which branch a connection takes is now the handshake `node.register` negotiated for that connection, read back from the registry through the new optional `NodeConnectionRegistry.providerDeliveryReadinessMode()`; a `node.heartbeat` roster refresh can no longer add or drop the `relay:delivery-cursor-v1` advertisement a registration settled, so a heartbeat between registration and certification can neither strand the backlog nor re-flush an immediate connection. Replay remains bounded, ordered by ascending `seq`, gated on per-identity delivery readiness, and deduped by the cumulative delivery cursor, so acked deliveries are never re-sent. - Push `context.update` events only to nodes that can hold a live socket, and at most four at a time. Channel, presence and agent-scoped fan-out previously targeted every node with an active binding, including offline ones; a workspace with thousands of offline nodes sent hundreds of node Durable Object fetches per channel join or presence change. Hosted, that fan-out runs in the background of the triggering request and starved the request's own write-admission lease release, which timed out and held the workspace's write or lifecycle lane for the full lease TTL (`workspace_busy`). Offline WebSocket nodes had no socket, so no event that was previously delivered is dropped. `draining` and `http_push` nodes stay eligible. ## [8.11.0] - 2026-09-17 diff --git a/packages/engine/src/__tests__/conformance/delivery.test.ts b/packages/engine/src/__tests__/conformance/delivery.test.ts index 91fca373..cafb230b 100644 --- a/packages/engine/src/__tests__/conformance/delivery.test.ts +++ b/packages/engine/src/__tests__/conformance/delivery.test.ts @@ -11,7 +11,7 @@ import { contextUpdatesOfType, type TestStack, } from './harness.js'; -import { agents, deliveries, messages } from '../../db/schema.js'; +import { agents, deliveries, messages, nodeProviders } from '../../db/schema.js'; import * as messageEngine from '../../engine/message.js'; import * as deliveryEngine from '../../engine/delivery.js'; import { ensureDirectNodeForAgent } from '../../engine/node.js'; @@ -1904,6 +1904,378 @@ describe('durable delivery api', () => { ]); }); + /** + * A reconnecting broker certifies its surviving sessions with `inventory.sync`, + * and that certified set is what gets replayed. These cases pin the reconnect + * drain to the certification itself rather than to a readiness/routing + * transition: a socket owner that already reports the listed identities as + * delivery-ready (an out-of-process owner whose ready-set is keyed per + * node+provider, not per connection) must still get the outage backlog. + */ + describe('reconnect replay of an outage backlog', () => { + /** + * Reconnect a cursor-negotiated node and certify `agentIds` through + * `inventory.sync`, with the socket owner already reporting them + * delivery-ready — the state a remote socket owner presents when its + * ready-set survives the transport reconnect. + */ + async function reconnectAndSync( + ws: { workspaceKey: string; workspaceId: string }, + node: { id: string; name: string }, + agents: Array<{ agentId: string; name: string }>, + opts: { alreadyReady?: boolean; heartbeatCapabilities?: string[] } = {}, + ) { + const reconnected = await enrollAndAttachNode(ws, { + id: node.id, + name: node.name, + cursorHandshake: true, + }); + expect(reconnected.sock.ofType('deliver')).toHaveLength(0); + if (opts.heartbeatCapabilities) { + await reconnected.handle.handleMessage(JSON.stringify({ + v: 1, + type: 'node.heartbeat', + load: 0, + active_agents: agents.length, + handlers_live: true, + capabilities: opts.heartbeatCapabilities.map((name) => ({ name, kind: 'capacity' })), + })); + } + if (opts.alreadyReady !== false) { + stack.runtime.realtime.markProviderAgentsDeliveryReady( + ws.workspaceId, + node.id, + DEFAULT_PROVIDER_NAME, + undefined, + agents.map((agent) => agent.agentId), + ); + } + await reconnected.handle.handleMessage(JSON.stringify({ + v: 1, + type: 'inventory.sync', + agents: agents.map((agent) => ({ + agent_id: agent.agentId, + name: agent.name, + session_ref: `sess-${agent.name}`, + })), + })); + await stack.settle(); + return reconnected; + } + + it('replays a message queued while the node socket was down', async () => { + const ws = await createWorkspace(stack.app, 'mailbox-outage-replay'); + const alice = await registerAgent(stack.app, ws.workspaceKey, 'alice'); + const node = await enrollAndAttachNode(ws, { cursorHandshake: true }); + const bob = await registerViaNode(node, 'bob'); + + await node.handle.handleClose(); + const post = await stack.app.request('/v1/channels/general/messages', { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${alice.token}` }, + body: JSON.stringify({ text: 'queued during the outage' }), + }); + expect(post.status).toBe(201); + const messageId = ((await post.json()) as { data: { id: string } }).data.id; + await stack.settle(); + + const reconnected = await reconnectAndSync(ws, node, [bob]); + + expect(deliverFramesOfType(reconnected.sock, 'message.created')).toEqual([ + expect.objectContaining({ agent: bob.name, msg_id: messageId, seq: 1 }), + ]); + }); + + it('does not re-send a delivery the node already acked before the outage', async () => { + const ws = await createWorkspace(stack.app, 'mailbox-outage-acked'); + const alice = await registerAgent(stack.app, ws.workspaceKey, 'alice'); + const node = await enrollAndAttachNode(ws, { cursorHandshake: true }); + const bob = await registerViaNode(node, 'bob'); + + const post = await stack.app.request('/v1/channels/general/messages', { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${alice.token}` }, + body: JSON.stringify({ text: 'acked before the outage' }), + }); + expect(post.status).toBe(201); + await waitForAssertion(() => { + expect(deliverFramesOfType(node.sock, 'message.created')).toHaveLength(1); + }); + await node.handle.handleMessage(JSON.stringify({ + v: 1, + type: 'delivery.ack', + agent: bob.name, + up_to_seq: 1, + })); + await node.handle.handleClose(); + + const reconnected = await reconnectAndSync(ws, node, [bob]); + + expect(deliverFramesOfType(reconnected.sock, 'message.created')).toEqual([]); + }); + + it('drains two outage deliveries oldest-first, exactly once across repeated syncs', async () => { + const ws = await createWorkspace(stack.app, 'mailbox-outage-ordering'); + const alice = await registerAgent(stack.app, ws.workspaceKey, 'alice'); + const node = await enrollAndAttachNode(ws, { cursorHandshake: true }); + const bob = await registerViaNode(node, 'bob'); + + await node.handle.handleClose(); + const texts = ['first while offline', 'second while offline']; + const messageIds: string[] = []; + for (const text of texts) { + const post = await stack.app.request('/v1/channels/general/messages', { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${alice.token}` }, + body: JSON.stringify({ text }), + }); + expect(post.status).toBe(201); + messageIds.push(((await post.json()) as { data: { id: string } }).data.id); + } + await stack.settle(); + + const reconnected = await reconnectAndSync(ws, node, [bob]); + expect(deliverFramesOfType(reconnected.sock, 'message.created')).toEqual([ + expect.objectContaining({ msg_id: messageIds[0], seq: 1 }), + expect.objectContaining({ msg_id: messageIds[1], seq: 2 }), + ]); + + // The cumulative cursor is the only dedupe: once the node acks the drained + // range, a further certification of the same session replays nothing. + await reconnected.handle.handleMessage(JSON.stringify({ + v: 1, + type: 'delivery.ack', + agent: bob.name, + up_to_seq: 2, + })); + await reconnected.handle.handleMessage(JSON.stringify({ + v: 1, + type: 'inventory.sync', + agents: [{ agent_id: bob.agentId, name: bob.name, session_ref: 'sess-bob' }], + })); + await stack.settle(); + expect(deliverFramesOfType(reconnected.sock, 'message.created')).toHaveLength(2); + }); + + it('does not push to an agent that the reconnect has not made delivery-ready', async () => { + const ws = await createWorkspace(stack.app, 'mailbox-outage-not-ready'); + const alice = await registerAgent(stack.app, ws.workspaceKey, 'alice'); + const node = await enrollAndAttachNode(ws, { cursorHandshake: true }); + const bob = await registerViaNode(node, 'bob'); + const carol = await registerViaNode(node, 'carol'); + + await node.handle.handleClose(); + const post = await stack.app.request('/v1/channels/general/messages', { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${alice.token}` }, + body: JSON.stringify({ text: 'queued for both sessions' }), + }); + expect(post.status).toBe(201); + await stack.settle(); + + // Only bob's session survived the reconnect; carol's restarted and is not + // certified by this inventory, so she stays gated until she re-announces. + const reconnected = await reconnectAndSync(ws, node, [bob]); + expect(deliverFramesOfType(reconnected.sock, 'message.created').map((frame) => frame.agent)) + .toEqual([bob.name]); + + await reconnected.handle.handleMessage(JSON.stringify({ + v: 1, + id: 'resume-carol', + type: 'agent.recover', + name: carol.name, + expected_agent_id: carol.agentId, + resumable: true, + session_ref: 'sess-carol', + })); + await stack.settle(); + expect(deliverFramesOfType(reconnected.sock, 'message.created').map((frame) => frame.agent)) + .toEqual([bob.name, carol.name]); + }); + + /** + * The handshake belongs to the CONNECTION, not to the provider's roster: a + * heartbeat between `node.register` and the certification refreshes spawn + * capacity and may omit (or add) the delivery-cursor advertisement. Neither + * direction may change how the live connection replays. + */ + it('replays the certified backlog when a heartbeat roster omits the cursor capability', async () => { + const ws = await createWorkspace(stack.app, 'mailbox-outage-heartbeat-roster'); + const alice = await registerAgent(stack.app, ws.workspaceKey, 'alice'); + const node = await enrollAndAttachNode(ws, { cursorHandshake: true }); + const bob = await registerViaNode(node, 'bob'); + + await node.handle.handleClose(); + const post = await stack.app.request('/v1/channels/general/messages', { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${alice.token}` }, + body: JSON.stringify({ text: 'queued before a roster-only heartbeat' }), + }); + expect(post.status).toBe(201); + const messageId = ((await post.json()) as { data: { id: string } }).data.id; + await stack.settle(); + + // The reconnect negotiated the cursor handshake and so got no + // register-time flush; the heartbeat that follows advertises spawn + // capacity only. The certification is still this connection's only + // replay trigger. + const reconnected = await reconnectAndSync(ws, node, [bob], { + alreadyReady: false, + heartbeatCapabilities: ['spawn:claude'], + }); + + expect(deliverFramesOfType(reconnected.sock, 'message.created')).toEqual([ + expect.objectContaining({ agent: bob.name, msg_id: messageId, seq: 1 }), + ]); + }); + + it('does not re-flush an immediate connection whose heartbeat roster adds the cursor capability', async () => { + const ws = await createWorkspace(stack.app, 'mailbox-outage-heartbeat-promote'); + const alice = await registerAgent(stack.app, ws.workspaceKey, 'alice'); + const node = await enrollAndAttachNode(ws); + const bob = await registerViaNode(node, 'bob'); + + await node.handle.handleClose(); + const post = await stack.app.request('/v1/channels/general/messages', { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${alice.token}` }, + body: JSON.stringify({ text: 'flushed by the legacy register' }), + }); + expect(post.status).toBe(201); + await stack.settle(); + + // Legacy handshake: `node.register` flushes the whole node, so the + // backlog is already on the socket (delivered, not yet acked). + const reconnected = await enrollAndAttachNode(ws, { id: node.id, name: node.name }); + await stack.settle(); + expect(deliverFramesOfType(reconnected.sock, 'message.created')).toHaveLength(1); + + await reconnected.handle.handleMessage(JSON.stringify({ + v: 1, + type: 'node.heartbeat', + load: 0, + active_agents: 1, + handlers_live: true, + capabilities: [ + { name: 'spawn:claude', kind: 'capacity' }, + { name: FLEET_DELIVERY_CURSOR_CAPABILITY, kind: 'capacity' }, + ], + })); + await reconnected.handle.handleMessage(JSON.stringify({ + v: 1, + type: 'inventory.sync', + agents: [{ agent_id: bob.agentId, name: bob.name, session_ref: 'sess-bob' }], + })); + await stack.settle(); + + // A roster heartbeat cannot promote the connection to cursor-gated, so + // the certification replays nothing the register-time flush already sent. + expect(deliverFramesOfType(reconnected.sock, 'message.created')).toHaveLength(1); + }); + + it('keeps the registered cursor advertisement out of reach of a heartbeat roster refresh', async () => { + const ws = await createWorkspace(stack.app, 'mailbox-heartbeat-capabilities'); + const cursorNode = await enrollAndAttachNode(ws, { cursorHandshake: true }); + const legacyNode = await enrollAndAttachNode(ws, { id: 'node_legacy', name: 'legacy-node' }); + + const rosterOnly = { + v: 1, + type: 'node.heartbeat', + load: 0, + active_agents: 0, + handlers_live: true, + capabilities: [{ name: 'spawn:claude', kind: 'capacity' }], + }; + await cursorNode.handle.handleMessage(JSON.stringify(rosterOnly)); + await legacyNode.handle.handleMessage(JSON.stringify({ + ...rosterOnly, + capabilities: [ + { name: 'spawn:claude', kind: 'capacity' }, + { name: FLEET_DELIVERY_CURSOR_CAPABILITY, kind: 'capacity' }, + ], + })); + await stack.settle(); + + const advertised = async (nodeId: string) => { + const [row] = await stack.runtime.deps.db + .select({ capabilities: nodeProviders.capabilities }) + .from(nodeProviders) + .where(and(eq(nodeProviders.nodeId, nodeId), eq(nodeProviders.name, DEFAULT_PROVIDER_NAME))); + return (row?.capabilities ?? []).map((capability) => capability.name); + }; + + // The negotiated capability survives a heartbeat that omits it... + expect(await advertised(cursorNode.id)).toContain(FLEET_DELIVERY_CURSOR_CAPABILITY); + // ...and a heartbeat cannot introduce one the registration never made. + expect(await advertised(legacyNode.id)).not.toContain(FLEET_DELIVERY_CURSOR_CAPABILITY); + }); + + /** + * A superseded connection's in-flight `inventory.sync` is not the live + * connection's certification. The registry answers `undefined` for a + * connection that no longer owns the provider — reading the persisted + * advertisement instead would classify the stale frame by whichever + * registration owns the provider NOW, replaying certified deliveries to + * sessions the replacement never certified. + */ + it('does not replay the outage backlog on a superseded connection certification', async () => { + const ws = await createWorkspace(stack.app, 'mailbox-outage-stale-sync'); + const alice = await registerAgent(stack.app, ws.workspaceKey, 'alice'); + const nodeA = await enrollAndAttachNode(ws, { cursorHandshake: true }); + const bob = await registerViaNode(nodeA, 'bob'); + + await nodeA.handle.handleClose(); + const post = await stack.app.request('/v1/channels/general/messages', { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${alice.token}` }, + body: JSON.stringify({ text: 'queued before the stale certification' }), + }); + expect(post.status).toBe(201); + const messageId = ((await post.json()) as { data: { id: string } }).data.id; + await stack.settle(); + + // The broker's fresh connection owns the provider now and persisted its + // own cursor advertisement at register. Nothing is certified yet — but + // an out-of-process socket owner can still report bob ready: its + // ready-set is keyed per node+provider, not per connection, and survives + // the transport reconnect (the state this whole replay path exists for). + const nodeB = await enrollAndAttachNode(ws, { + id: nodeA.id, + name: nodeA.name, + cursorHandshake: true, + }); + expect(nodeB.sock.ofType('deliver')).toHaveLength(0); + stack.runtime.realtime.markProviderAgentsDeliveryReady( + ws.workspaceId, + nodeB.id, + DEFAULT_PROVIDER_NAME, + undefined, + [bob.agentId], + ); + + // The old connection's queued sync finally runs. Its certification is + // not this connection's: it must not drain the backlog to bob. + await nodeA.handle.handleMessage(JSON.stringify({ + v: 1, + type: 'inventory.sync', + agents: [{ agent_id: bob.agentId, name: bob.name, session_ref: 'sess-bob' }], + })); + await stack.settle(); + expect(deliverFramesOfType(nodeB.sock, 'message.created')).toHaveLength(0); + + // The replacement's own certification still drains the backlog. + await nodeB.handle.handleMessage(JSON.stringify({ + v: 1, + type: 'inventory.sync', + agents: [{ agent_id: bob.agentId, name: bob.name, session_ref: 'sess-bob' }], + })); + await stack.settle(); + expect(deliverFramesOfType(nodeB.sock, 'message.created')).toEqual([ + expect.objectContaining({ agent: bob.name, msg_id: messageId, seq: 1 }), + ]); + }); + }); + it('preserves hyphenated mentions through node reconnect replay', async () => { const ws = await createWorkspace(stack.app, 'mention-replay'); const alice = await registerAgent(stack.app, ws.workspaceKey, 'alice'); diff --git a/packages/engine/src/adapters/node/realtime.ts b/packages/engine/src/adapters/node/realtime.ts index 744abc2b..7f12374b 100644 --- a/packages/engine/src/adapters/node/realtime.ts +++ b/packages/engine/src/adapters/node/realtime.ts @@ -636,6 +636,26 @@ export class InProcessRealtime implements RealtimeBus, ConnectionRegistry, NodeC } } + providerDeliveryReadinessMode( + workspaceId: string, + nodeId: string, + providerName: string, + connectionId?: string | undefined, + ): 'immediate' | 'agent_scoped' | undefined { + const nodeKey = this.nodeKey(workspaceId, nodeId); + const currentConnectionId = this.providerConnId(nodeKey, providerName); + if (!currentConnectionId || (connectionId && currentConnectionId !== connectionId)) return undefined; + const conn = this.nodeConnections.get(currentConnectionId); + if (!conn || conn.workspaceId !== workspaceId || conn.nodeId !== nodeId || conn.providerName !== providerName) { + return undefined; + } + // `undefined` is "never configured on this connection" (no node.register + // yet), which is not a negotiated mode; null is immediate, a Set is + // agent-scoped — the same encoding isProviderAgentDeliveryReady reads. + if (conn.deliveryReadyAgentIds === undefined) return undefined; + return conn.deliveryReadyAgentIds === null ? 'immediate' : 'agent_scoped'; + } + markProviderAgentsDeliveryReady( workspaceId: string, nodeId: string, diff --git a/packages/engine/src/engine/node.ts b/packages/engine/src/engine/node.ts index 003c90ad..17b6821d 100644 --- a/packages/engine/src/engine/node.ts +++ b/packages/engine/src/engine/node.ts @@ -25,7 +25,7 @@ import { codedError } from '../lib/httpError.js'; import { acceptTaskInvocation, completeTaskInvocation } from './taskInvocation.js'; import { runAtomic, runAtomicWrites } from '../ports/database.js'; import type { AtomicWrite, EngineDb } from '../ports/database.js'; -import { isProviderAgentDeliveryReady, type NodeConnectionRegistry } from '../ports/realtime.js'; +import type { NodeConnectionRegistry } from '../ports/realtime.js'; import { generateId } from './snowflake.js'; import { assertRegistrableAgentName } from './agent.js'; import { rotateAgentIdentity } from './agentIdentity.js'; @@ -201,6 +201,94 @@ function supportsProviderDeliveryReadiness(registry: NodeConnectionRegistry): bo && typeof registry.isProviderAgentDeliveryReady === 'function'; } +/** + * Whether a provider's current registration advertised the delivery-cursor + * capability. Registration persists the advertised set, so later frames on the + * same provider can recover the handshake `node.register` negotiated without + * the client re-asserting it. + */ +async function providerAdvertisesDeliveryCursor( + db: EngineDb, + workspaceId: string, + nodeId: string, + providerName: string, +): Promise { + const [provider] = await db + .select({ capabilities: nodeProviders.capabilities }) + .from(nodeProviders) + .where(and( + eq(nodeProviders.workspaceId, workspaceId), + eq(nodeProviders.nodeId, nodeId), + eq(nodeProviders.name, providerName), + )); + return provider?.capabilities?.some( + (capability) => capability.name === FLEET_DELIVERY_CURSOR_CAPABILITY, + ) ?? false; +} + +/** + * Capabilities that are NEGOTIATED at `node.register` (answered in its + * acceptance list and wired into the connection's delivery mode) rather than + * merely advertised. A heartbeat refreshes the provider's roster of spawn / + * action capacity; it is not a renegotiation, so it may neither drop nor + * introduce one of these. Dropping would demote a live cursor-gated connection + * that was deliberately denied the register-time flush — stranding its backlog; + * introducing would promote an immediate connection and replay frames that + * flush already sent. The registered value stands until the next + * `node.register`. + */ +const NEGOTIATED_PROTOCOL_CAPABILITIES: readonly string[] = [FLEET_DELIVERY_CURSOR_CAPABILITY]; + +function isNegotiatedProtocolCapability(capability: FleetCapability): boolean { + return NEGOTIATED_PROTOCOL_CAPABILITIES.includes(capability.name); +} + +function withRegisteredProtocolCapabilities( + heartbeatCapabilities: FleetCapability[], + registeredCapabilities: FleetCapability[], +): FleetCapability[] { + return [ + ...heartbeatCapabilities.filter((capability) => !isNegotiatedProtocolCapability(capability)), + ...registeredCapabilities.filter(isNegotiatedProtocolCapability), + ]; +} + +/** + * Whether the provider's LIVE connection negotiated cursor-gated delivery at + * `node.register` — i.e. whether it was denied the register-time flush and so + * depends on a later certification to replay. + * + * The registry owns that answer: `node.register` hands it the negotiated mode + * per connection, and a reconnect starts a fresh one. Ask it first. Only a + * registry that does not expose the mode (an out-of-process owner on an older + * contract) falls back to the persisted advertisement — which heartbeats keep + * off-limits (see `heartbeatNode`) precisely so the fallback answers for the + * registration rather than for the latest roster snapshot. + * + * When the method DOES exist, its `undefined` is an answer too: the connection + * asking is not the provider's current owner (a superseded connection whose + * frame queued behind the replacement's register). Reading the persisted + * advertisement then would classify a stale connection's sync by whichever + * registration owns the provider now — replaying certified deliveries to a + * connection that has not certified those sessions. A registry that can + * answer gets the last word; the fallback is only for one that cannot. + */ +async function connectionNegotiatedDeliveryCursor( + db: EngineDb, + registry: NodeConnectionRegistry, + workspaceId: string, + nodeId: string, + providerName: string, + connectionId: string | undefined, +): Promise { + if (!supportsProviderDeliveryReadiness(registry)) return false; + if (registry.providerDeliveryReadinessMode) { + return registry.providerDeliveryReadinessMode(workspaceId, nodeId, providerName, connectionId) + === 'agent_scoped'; + } + return providerAdvertisesDeliveryCursor(db, workspaceId, nodeId, providerName); +} + function requestId(message: { id?: string }): string { return message.id ?? generateId(); } @@ -713,15 +801,23 @@ export async function heartbeatNode( await runAtomic(db, async (tx) => { if (message.capabilities !== undefined) { - const caps = normalizeCapabilities(message.capabilities); const [existingProvider] = await tx - .select({ instanceId: nodeProviders.instanceId, maxAgents: nodeProviders.maxAgents, version: nodeProviders.version }) + .select({ + instanceId: nodeProviders.instanceId, + maxAgents: nodeProviders.maxAgents, + version: nodeProviders.version, + capabilities: nodeProviders.capabilities, + }) .from(nodeProviders) .where(and( eq(nodeProviders.workspaceId, workspaceId), eq(nodeProviders.nodeId, nodeId), eq(nodeProviders.name, providerName), )); + const caps = withRegisteredProtocolCapabilities( + normalizeCapabilities(message.capabilities), + existingProvider?.capabilities ?? [], + ); await upsertProvider(tx, workspaceId, nodeId, { name: providerName, instanceId: existingProvider?.instanceId ?? `${providerName}:heartbeat`, @@ -1671,17 +1767,8 @@ export async function registerAgentViaNode( .from(nodes) .where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId))); if (!node) throw codedError(`Node "${nodeId}" not found`, 'node_not_found', 404); - const [provider] = await tx - .select({ capabilities: nodeProviders.capabilities }) - .from(nodeProviders) - .where(and( - eq(nodeProviders.workspaceId, workspaceId), - eq(nodeProviders.nodeId, nodeId), - eq(nodeProviders.name, providerName), - )); - const cursorHandshake = (options.deliveryCursorSupported ?? true) && (provider?.capabilities?.some( - (capability) => capability.name === FLEET_DELIVERY_CURSOR_CAPABILITY, - ) ?? false); + const cursorHandshake = (options.deliveryCursorSupported ?? true) + && await providerAdvertisesDeliveryCursor(tx, workspaceId, nodeId, providerName); // Preserve the established identity conflict even when this node is full. // A concurrent create after this read is still caught by the unique insert @@ -1800,17 +1887,8 @@ export async function recoverAgentViaNode( .where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId))); if (!node) throw codedError(`Node "${nodeId}" not found`, 'node_not_found', 404); - const [provider] = await tx - .select({ capabilities: nodeProviders.capabilities }) - .from(nodeProviders) - .where(and( - eq(nodeProviders.workspaceId, workspaceId), - eq(nodeProviders.nodeId, nodeId), - eq(nodeProviders.name, providerName), - )); - const cursorHandshake = (options.deliveryCursorSupported ?? true) && (provider?.capabilities?.some( - (capability) => capability.name === FLEET_DELIVERY_CURSOR_CAPABILITY, - ) ?? false); + const cursorHandshake = (options.deliveryCursorSupported ?? true) + && await providerAdvertisesDeliveryCursor(tx, workspaceId, nodeId, providerName); const [target] = await tx .select() @@ -2831,15 +2909,6 @@ export async function handleNodeControlMessage(args: HandleNodeControlMessageArg data: result.reply, }); if (!replySent) return; - const newlyReadyAgentIds = result.reconciledAgentIds.filter((agentId) => ( - !isProviderAgentDeliveryReady( - args.registry, - args.workspaceId, - args.nodeId, - frameProviderName, - agentId, - ) - )); args.registry.markProviderAgentsDeliveryReady?.( args.workspaceId, args.nodeId, @@ -2848,16 +2917,50 @@ export async function handleNodeControlMessage(args: HandleNodeControlMessageArg result.reconciledAgentIds, ); // Inventory represents sessions that survived a transport reconnect and - // therefore retain their in-memory cursors. Restrict replay to exactly - // those provider-owned identities; restarted sessions not in inventory - // become ready individually through `agent.register`. - const replayAgentIds = [...new Set([...newlyReadyAgentIds, ...result.newlyRoutedAgentIds])]; + // therefore retain their in-memory cursors. Restarted sessions not in + // the inventory stay excluded either way — they become ready + // individually through `agent.register`. + // + // A cursor-negotiated connection receives NO replay at `node.register` + // (each identity is seeded by its own cursor-bearing reply), so this + // certification is the whole node's only reconnect-replay trigger and + // must cover every certified identity. Scoping it to a readiness/routing + // TRANSITION instead stranded a reconnecting node's entire outage + // backlog whenever the socket owner already reported those identities as + // delivery-ready — an out-of-process owner whose ready-set is keyed per + // node+provider rather than per connection. The diff came back empty, + // the drain ran with an empty scope, and rows queued during the outage + // sat unsent until their mailbox TTL. + // + // A legacy immediate-delivery connection already had the whole node + // flushed to it at `node.register` and gates nothing afterwards, so only + // identities this sync newly routed here still need a push; re-sending + // the rest would duplicate that flush. + // + // Either way replay stays bounded, ordered oldest-first, gated per + // identity on delivery readiness inside the drain, and deduped by the + // cumulative delivery cursor, so an identity that is already drained and + // acked re-sends nothing. + // The mode is the CONNECTION's, recovered from the registry that + // `node.register` configured it on — not re-derived from roster state + // that a heartbeat can rewrite between registration and certification. + const cursorGated = await connectionNegotiatedDeliveryCursor( + args.db, + args.registry, + args.workspaceId, + args.nodeId, + frameProviderName, + args.connectionId, + ); await deliverPendingToNode( args.db, args.registry, args.workspaceId, args.nodeId, - { providerName: frameProviderName, agentIds: replayAgentIds }, + { + providerName: frameProviderName, + agentIds: cursorGated ? result.reconciledAgentIds : result.newlyRoutedAgentIds, + }, ); return; } diff --git a/packages/engine/src/ports/realtime.ts b/packages/engine/src/ports/realtime.ts index f7ff06e4..46a80963 100644 --- a/packages/engine/src/ports/realtime.ts +++ b/packages/engine/src/ports/realtime.ts @@ -238,6 +238,24 @@ export interface NodeConnectionRegistry { mode: 'immediate' | 'agent_scoped', ): void; + /** + * The mode {@link setProviderDeliveryReadiness} last configured for the + * provider's CURRENT connection, or `undefined` when no connection is bound + * (or the registry does not track it). This is the negotiated handshake of + * the live connection, so a later frame on that connection can recover what + * `node.register` agreed to instead of re-deriving it from roster state a + * heartbeat may have rewritten since. Implementations must resolve the + * connection exactly as {@link setProviderDeliveryReadiness} does, and return + * `undefined` for a `connectionId` that is no longer the provider's current + * connection. + */ + providerDeliveryReadinessMode?( + workspaceId: string, + nodeId: string, + providerName: string, + connectionId?: string | undefined, + ): 'immediate' | 'agent_scoped' | undefined; + /** Mark identities ready after their cursor-bearing reply is on the socket. */ markProviderAgentsDeliveryReady?( workspaceId: string,