From 8d384a2b736b4f5faca52c3c5dea697ef5640663 Mon Sep 17 00:00:00 2001 From: Andrew Barba Date: Wed, 29 Jul 2026 21:20:10 -0400 Subject: [PATCH 1/9] fix(eve): settle turn as cancelled when abort races step completion A cancellation observed while a durable turn step was already returning lost the race to an ordinary completion: the step body could miss the abort signal and finish done, leaving the session completed after the user had cancelled. Honor the claimed cancel whenever the abort signal fired and the step produced no pending runtime-action batch (those resolve in-line and observe the signal in their own wait). Surfaced by deterministic-model timing in the e2e world suites, where turns reach completion fast enough to expose the race (originally found in #1367). Signed-off-by: Andrew Barba --- .changeset/turn-cancel-race.md | 5 +++ .../eve/src/execution/turn-workflow.test.ts | 40 +++++++++++++++++++ packages/eve/src/execution/turn-workflow.ts | 18 ++++++--- 3 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 .changeset/turn-cancel-race.md diff --git a/.changeset/turn-cancel-race.md b/.changeset/turn-cancel-race.md new file mode 100644 index 000000000..54e534288 --- /dev/null +++ b/.changeset/turn-cancel-race.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +A turn cancellation observed while a durable turn step is returning now settles the turn as cancelled instead of losing the race to an ordinary completion. Previously the step could miss the abort signal and finish `done`, leaving the session in a completed state after the user had already cancelled. diff --git a/packages/eve/src/execution/turn-workflow.test.ts b/packages/eve/src/execution/turn-workflow.test.ts index 227743039..9f352ebd0 100644 --- a/packages/eve/src/execution/turn-workflow.test.ts +++ b/packages/eve/src/execution/turn-workflow.test.ts @@ -296,6 +296,46 @@ describe("turnWorkflow", () => { expect(resumeHookMock.mock.calls.filter((call) => call[1]?.kind === "turn-error")).toEqual([]); }); + it("honors cancellation observed while a durable turn step returns", async () => { + const sessionState = createSessionState(); + installInbox([], { cancelPayloads: [{}] }); + vi.mocked(turnStep).mockImplementationOnce(async (stepInput) => { + await vi.waitFor(() => expect(stepInput.abortSignal?.aborted).toBe(true)); + return { + action: "done", + output: "must not complete", + serializedContext: { state: "done" }, + sessionState, + }; + }); + + const { input } = createInput({ + driverCapabilities: { cancelledTurnSettle: true, turnInbox: true }, + sessionState, + }); + await turnWorkflow(input); + + expect(cancelDescendantTurnsStep).toHaveBeenCalledWith({ + serializedContext: { state: "start" }, + sessionState, + }); + expect(resumeHookMock).toHaveBeenCalledWith("turn-token", { + action: { + cancelled: true, + kind: "park", + serializedContext: { state: "start" }, + sessionState, + }, + kind: "turn-result", + }); + expect(resumeHookMock).not.toHaveBeenCalledWith( + "turn-token", + expect.objectContaining({ + action: expect.objectContaining({ kind: "done" }), + }), + ); + }); + it("runs uncancellable when the session cancel token is claimed by another run", async () => { const sessionState = createSessionState(); installInbox([], { cancelConflict: { runId: "wrun_stale_prior_turn" } }); diff --git a/packages/eve/src/execution/turn-workflow.ts b/packages/eve/src/execution/turn-workflow.ts index 62ceaf9df..1248afac5 100644 --- a/packages/eve/src/execution/turn-workflow.ts +++ b/packages/eve/src/execution/turn-workflow.ts @@ -106,8 +106,19 @@ async function runTurnOwnedWorkflow(input: TurnWorkflowInput): Promise { while (true) { const result = await turnStep(cursor.createStepInput(nextStepInput, cancellation?.signal)); + const pendingActionKeys = + result.action === "dispatch-workflow-runtime-actions" || result.action === "park" + ? result.pendingRuntimeActionKeys + : undefined; - if (result.action === "cancelled") { + // A cancel observed while the step was already returning must still win: + // the step body may have missed the abort and produced an ordinary + // completion, but the user's cancel was claimed. Pending runtime-action + // batches are exempt — their in-line wait below observes the signal. + if ( + result.action === "cancelled" || + (cancellation?.signal.aborted === true && pendingActionKeys === undefined) + ) { // No `canPark` check here: that gate rejects model-authored waits // (`next: null`) in task mode, whereas a cancelled turn parks by // design and its parkability was already established when the @@ -137,11 +148,6 @@ async function runTurnOwnedWorkflow(input: TurnWorkflowInput): Promise { // A pending runtime-action batch (model-driven `park` or dynamic-workflow // interrupt) is resolved in-line so the turn stays alive across the wait; // the two arms differ only in their dispatch path. - const pendingActionKeys = - result.action === "dispatch-workflow-runtime-actions" || result.action === "park" - ? result.pendingRuntimeActionKeys - : undefined; - if (pendingActionKeys !== undefined) { await cursor.adopt(result); const dispatch = From 5c735a71f49c60953e4317da0992f984340be19b Mon Sep 17 00:00:00 2001 From: Andrew Barba Date: Wed, 29 Jul 2026 22:09:05 -0400 Subject: [PATCH 2/9] fix(eve): harden descendant turn-cancel delivery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e cancellation flake has a second face beyond the settle race: on the Vercel world, a cancelled parent's cascade to its subagent child can drop entirely — the child's wait-for-cancellation tool rides out its fallback timeout and the child runs to completion with no turn.cancelled. Every drop path in cancelDescendantTurnsStep was silent: a missing pending batch, missing child session ids, and retry exhaustion all returned as if the cancel had been delivered. Retry descendant cancels with exponential backoff (~8s budget instead of a flat 3s), classify no_active_turn results with the error that marked the target inactive (HookNotFoundError vs EntityConflictError etc.), and warn on every path that gives up so an uncancelled child is at least observable in logs while a durable delivery design is worked out. Signed-off-by: Andrew Barba --- .changeset/descendant-cancel-delivery.md | 5 ++ packages/eve/src/channel/types.ts | 7 +++ .../cancel-descendant-turns-step.test.ts | 36 ++++++++++- .../execution/cancel-descendant-turns-step.ts | 61 ++++++++++++++++--- .../turn-cancellation.integration.test.ts | 8 ++- .../src/execution/workflow-runtime.test.ts | 16 +++-- .../eve/src/execution/workflow-runtime.ts | 18 +++--- 7 files changed, 122 insertions(+), 29 deletions(-) create mode 100644 .changeset/descendant-cancel-delivery.md diff --git a/.changeset/descendant-cancel-delivery.md b/.changeset/descendant-cancel-delivery.md new file mode 100644 index 000000000..b9b75d4df --- /dev/null +++ b/.changeset/descendant-cancel-delivery.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Cancelling a parent turn now delivers descendant cancellations more reliably: the cancel request retries with exponential backoff (~8s budget instead of 3s), `no_active_turn` results carry the error class that marked the target inactive, and every previously silent drop path (missing pending batch, missing child session ids, exhausted retries) now logs a warning so an uncancelled child no longer runs to completion without a trace. diff --git a/packages/eve/src/channel/types.ts b/packages/eve/src/channel/types.ts index 564b47dfd..7784ddb39 100644 --- a/packages/eve/src/channel/types.ts +++ b/packages/eve/src/channel/types.ts @@ -29,6 +29,13 @@ export interface CancelTurnInput { /** Result of requesting turn cancellation. Both statuses are successful. */ export interface CancelTurnResult { readonly status: CancelTurnStatus; + /** + * For `no_active_turn`: the error class that classified the target as + * inactive (e.g. `HookNotFoundError`, `EntityConflictError`). Lets callers + * that expected an active turn distinguish "already finished" from a + * transiently unreachable cancel hook. + */ + readonly reason?: string; } /** Identifies a session to transition permanently to a terminal state. */ diff --git a/packages/eve/src/execution/cancel-descendant-turns-step.test.ts b/packages/eve/src/execution/cancel-descendant-turns-step.test.ts index b4a740ec9..9dbe9ac62 100644 --- a/packages/eve/src/execution/cancel-descendant-turns-step.test.ts +++ b/packages/eve/src/execution/cancel-descendant-turns-step.test.ts @@ -140,11 +140,11 @@ describe("cancelDescendantTurnsStep", () => { serializedContext: {}, sessionState: createPendingState(), }); - await vi.advanceTimersByTimeAsync(3_000); + await vi.advanceTimersByTimeAsync(10_000); await expect(cancellation).resolves.toBeUndefined(); expect(requestWorkflowTurnCancellation).toHaveBeenCalledOnce(); - expect(cancelRemoteAgentTurn).toHaveBeenCalledTimes(12); + expect(cancelRemoteAgentTurn).toHaveBeenCalledTimes(8); expect(error).toHaveBeenCalledWith( "[eve:execution.cancel-descendant-turns] failed to cancel local descendant turn", expect.objectContaining({ callId: "call-local", childSessionId: "local-child" }), @@ -155,7 +155,34 @@ describe("cancelDescendantTurnsStep", () => { ); }); + it("warns loudly when a descendant cancel is never accepted", async () => { + vi.useFakeTimers(); + vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ + reason: "EntityConflictError", + status: "no_active_turn", + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + const cancellation = cancelDescendantTurnsStep({ + serializedContext: {}, + sessionState: createPendingState({ includeRemote: false }), + }); + await vi.advanceTimersByTimeAsync(10_000); + await expect(cancellation).resolves.toBeUndefined(); + + expect(requestWorkflowTurnCancellation).toHaveBeenCalledTimes(8); + expect(warn).toHaveBeenCalledWith( + "[eve:execution.cancel-descendant-turns] descendant cancel was never accepted; the child may run to completion", + expect.objectContaining({ + childSessionId: "local-child", + finalStatus: "no_active_turn", + reason: "EntityConflictError", + }), + ); + }); + it("treats pending batches from older deployments as having no descendants", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); const session = setPendingRuntimeActionBatch({ actions: [localAction], event: { sequence: 0, stepIndex: 0, turnId: "turn_0" }, @@ -171,6 +198,11 @@ describe("cancelDescendantTurnsStep", () => { expect(requestWorkflowTurnCancellation).not.toHaveBeenCalled(); expect(cancelRemoteAgentTurn).not.toHaveBeenCalled(); expect(deserializeContext).not.toHaveBeenCalled(); + // The skipped cancellation is observable, not silent. + expect(warn).toHaveBeenCalledWith( + "[eve:execution.cancel-descendant-turns] pending batch carries no child session ids; descendants cannot be cancelled", + expect.objectContaining({ sessionId: expect.any(String) }), + ); }); }); diff --git a/packages/eve/src/execution/cancel-descendant-turns-step.ts b/packages/eve/src/execution/cancel-descendant-turns-step.ts index 854e0cb89..b7d6a7265 100644 --- a/packages/eve/src/execution/cancel-descendant-turns-step.ts +++ b/packages/eve/src/execution/cancel-descendant-turns-step.ts @@ -16,8 +16,14 @@ import type { } from "#runtime/actions/types.js"; import type { RuntimeSubagentRegistry } from "#runtime/subagents/registry.js"; -const CANCEL_ATTEMPTS = 12; -const CANCEL_RETRY_DELAY_MS = 250; +// Descendants listed in a pending batch should be actively running, so a +// cancel that cannot be delivered is anomalous: retry with backoff long +// enough to ride out transient world contention (queue wakes, hook-claim +// conflicts), then log loudly rather than dropping the cancel silently — +// an uncancelled child otherwise runs to completion with no trace of why. +const CANCEL_ATTEMPTS = 8; +const CANCEL_RETRY_INITIAL_DELAY_MS = 250; +const CANCEL_RETRY_MAX_DELAY_MS = 1_500; const log = createLogger("execution.cancel-descendant-turns"); /** Cancels every successfully adopted child in the current pending batch. */ @@ -38,9 +44,20 @@ export async function cancelDescendantTurnsStep(input: { return; } - if (batch === undefined) return; + if (batch === undefined) { + log.warn("no pending batch found while cancelling descendants; nothing to cancel", { + sessionId: input.sessionState.sessionId, + }); + return; + } const childSessionIds = batch.childSessionIds; - if (childSessionIds === undefined) return; + if (childSessionIds === undefined) { + log.warn("pending batch carries no child session ids; descendants cannot be cancelled", { + actionCount: batch.actions.length, + sessionId: input.sessionState.sessionId, + }); + return; + } let remoteRegistry: Promise | undefined; const getRemoteRegistry = () => @@ -75,10 +92,19 @@ async function cancelLocalDescendant(input: { readonly childSessionId: string; }): Promise { try { - await requestCancellationWithRetry({ + const final = await requestCancellationWithRetry({ request: () => requestWorkflowTurnCancellation({ sessionId: input.childSessionId }), shouldRetryError: () => false, }); + if (final.status !== "accepted") { + log.warn("descendant cancel was never accepted; the child may run to completion", { + callId: input.action.callId, + childSessionId: input.childSessionId, + finalStatus: final.status, + reason: final.reason, + subagentName: input.action.subagentName, + }); + } } catch (error) { logError(log, "failed to cancel local descendant turn", error, { callId: input.action.callId, @@ -101,10 +127,19 @@ async function cancelRemoteDescendant(input: { registry, }); - await requestCancellationWithRetry({ + const final = await requestCancellationWithRetry({ request: () => cancelRemoteAgentTurn({ remote, sessionId: input.childSessionId }), shouldRetryError: isRetryableRemoteAgentCancelError, }); + if (final.status !== "accepted") { + log.warn("remote descendant cancel was never accepted; the child may run to completion", { + callId: input.action.callId, + childSessionId: input.childSessionId, + finalStatus: final.status, + reason: final.reason, + remoteAgentName: input.action.remoteAgentName, + }); + } } catch (error) { logError(log, "failed to cancel remote descendant turn", error, { callId: input.action.callId, @@ -117,15 +152,21 @@ async function cancelRemoteDescendant(input: { async function requestCancellationWithRetry(input: { readonly request: () => Promise; readonly shouldRetryError: (error: unknown) => boolean; -}): Promise { +}): Promise { + let delayMs = CANCEL_RETRY_INITIAL_DELAY_MS; + let lastResult: CancelTurnResult = { status: "no_active_turn" }; + for (let attempt = 1; attempt <= CANCEL_ATTEMPTS; attempt += 1) { try { - const result = await input.request(); - if (result.status === "accepted" || attempt === CANCEL_ATTEMPTS) return; + lastResult = await input.request(); + if (lastResult.status === "accepted" || attempt === CANCEL_ATTEMPTS) return lastResult; } catch (error) { if (!input.shouldRetryError(error) || attempt === CANCEL_ATTEMPTS) throw error; } - await new Promise((resolve) => setTimeout(resolve, CANCEL_RETRY_DELAY_MS)); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + delayMs = Math.min(delayMs * 2, CANCEL_RETRY_MAX_DELAY_MS); } + + return lastResult; } diff --git a/packages/eve/src/execution/turn-cancellation.integration.test.ts b/packages/eve/src/execution/turn-cancellation.integration.test.ts index 705bf08f7..1cef2066c 100644 --- a/packages/eve/src/execution/turn-cancellation.integration.test.ts +++ b/packages/eve/src/execution/turn-cancellation.integration.test.ts @@ -486,10 +486,14 @@ describe("turn cancellation integration", () => { expect(fixture.toolAborts()).toBe(1); // Session.cancel() addresses the same session by id. With the turn - // settled and its cancel hook swept, it reports the benign status. + // settled and its cancel hook swept, it reports the benign status, + // classified by the error that marked the target inactive. await waitForHookSweep(sessionCancelHookToken(run.runId)); const session = createSession(run.runId, rawToken, workflowRuntime); - await expect(session.cancel()).resolves.toEqual({ status: "no_active_turn" }); + await expect(session.cancel()).resolves.toEqual({ + reason: "HookNotFoundError", + status: "no_active_turn", + }); await waitForHook({ runId: run.runId }, { token: continuationToken }); await resumeHook(continuationToken, { diff --git a/packages/eve/src/execution/workflow-runtime.test.ts b/packages/eve/src/execution/workflow-runtime.test.ts index 79346d5cc..207f90f10 100644 --- a/packages/eve/src/execution/workflow-runtime.test.ts +++ b/packages/eve/src/execution/workflow-runtime.test.ts @@ -169,19 +169,23 @@ describe("createWorkflowRuntime#cancelTurn", () => { expect(resumeHookMock).toHaveBeenCalledWith("session-1:cancel", {}); }); - it("maps missing and terminal targets to 'no_active_turn'", async () => { + it("maps missing and terminal targets to 'no_active_turn' with a classified reason", async () => { const { EntityConflictError, HookNotFoundError, RunExpiredError, WorkflowRunNotFoundError } = await import("#compiled/@workflow/errors/index.js"); const errors = [ - new HookNotFoundError("session-1:cancel"), - new WorkflowRunNotFoundError("turn-run"), - new RunExpiredError("turn already completed"), - new EntityConflictError("turn completed during cancellation"), + { error: new HookNotFoundError("session-1:cancel"), reason: "HookNotFoundError" }, + { error: new WorkflowRunNotFoundError("turn-run"), reason: "WorkflowRunNotFoundError" }, + { error: new RunExpiredError("turn already completed"), reason: "RunExpiredError" }, + { + error: new EntityConflictError("turn completed during cancellation"), + reason: "EntityConflictError", + }, ]; - for (const error of errors) { + for (const { error, reason } of errors) { resumeHookMock.mockRejectedValueOnce(error); await expect(buildRuntime().cancelTurn({ sessionId: "session-1" })).resolves.toEqual({ + reason, status: "no_active_turn", }); } diff --git a/packages/eve/src/execution/workflow-runtime.ts b/packages/eve/src/execution/workflow-runtime.ts index d63ab146e..9711a203a 100644 --- a/packages/eve/src/execution/workflow-runtime.ts +++ b/packages/eve/src/execution/workflow-runtime.ts @@ -269,20 +269,20 @@ export async function requestWorkflowTurnCancellation( await resumeHook(sessionCancelHookToken(input.sessionId), payload); return { status: "accepted" }; } catch (error) { - if (isInactiveCancelTarget(error)) { - return { status: "no_active_turn" }; + const reason = classifyInactiveCancelTarget(error); + if (reason !== undefined) { + return { reason, status: "no_active_turn" }; } throw error; } } -function isInactiveCancelTarget(error: unknown): boolean { - return ( - HookNotFoundError.is(error) || - WorkflowRunNotFoundError.is(error) || - RunExpiredError.is(error) || - EntityConflictError.is(error) - ); +function classifyInactiveCancelTarget(error: unknown): string | undefined { + if (HookNotFoundError.is(error)) return "HookNotFoundError"; + if (WorkflowRunNotFoundError.is(error)) return "WorkflowRunNotFoundError"; + if (RunExpiredError.is(error)) return "RunExpiredError"; + if (EntityConflictError.is(error)) return "EntityConflictError"; + return undefined; } function isAlreadyTerminalSessionError(error: unknown): boolean { From 4e222be14452967025152e38d252620363590ab4 Mon Sep 17 00:00:00 2001 From: Andrew Barba Date: Wed, 29 Jul 2026 22:25:31 -0400 Subject: [PATCH 3/9] fix(eve): abort the turn cancel signal in the payload's own microtask A wake that replays a journaled cancel alongside a completed turn step drives both continuations in one microtask drain, payload first. The abort was chained on consumeMatchingCancel with .then, so it ran one hop behind the turn loop's continuation: the loop checked signal.aborted before the abort fired, took the done arm, and disposed the pending cancel. Fire the abort inside the matching read's own continuation so the signal is already flipped when any same-drain continuation checks it. Observed on the Vercel world as a subagent child completing normally after its parent's cascade cancel had been accepted. Signed-off-by: Andrew Barba --- .changeset/turn-cancel-race.md | 2 +- .../turn-cancellation-control.test.ts | 46 +++++++++++++++++++ .../execution/turn-cancellation-control.ts | 23 +++++++--- 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/.changeset/turn-cancel-race.md b/.changeset/turn-cancel-race.md index 54e534288..b311f9e2e 100644 --- a/.changeset/turn-cancel-race.md +++ b/.changeset/turn-cancel-race.md @@ -2,4 +2,4 @@ "eve": patch --- -A turn cancellation observed while a durable turn step is returning now settles the turn as cancelled instead of losing the race to an ordinary completion. Previously the step could miss the abort signal and finish `done`, leaving the session in a completed state after the user had already cancelled. +A turn cancellation observed while a durable turn step is returning now settles the turn as cancelled instead of losing the race to an ordinary completion. Previously the step could miss the abort signal and finish `done`, leaving the session in a completed state after the user had already cancelled. The cancel signal also now aborts in the same microtask that consumes the cancel payload, so a cancel replayed alongside a completed step can no longer lose the settle check by one task-queue hop. diff --git a/packages/eve/src/execution/turn-cancellation-control.test.ts b/packages/eve/src/execution/turn-cancellation-control.test.ts index cda4eb17b..2f78a40ca 100644 --- a/packages/eve/src/execution/turn-cancellation-control.test.ts +++ b/packages/eve/src/execution/turn-cancellation-control.test.ts @@ -113,6 +113,52 @@ describe("createTurnCancellationControl", () => { expect(control!.signal.aborted).toBe(false); }); + it("flips the signal in the same microtask that consumes the payload", async () => { + // Reproduces a wake replay where a journaled cancel and a completed + // turn step resolve in the same drive, payload first. The turn loop + // checks `signal.aborted` in the step's continuation — one microtask + // after the payload read — so the abort must fire inside the read + // continuation, not a `.then` chained behind it. + let releasePayload!: (result: IteratorResult) => void; + const firstRead = new Promise>((resolve) => { + releasePayload = resolve; + }); + let delivered = false; + createHookMock.mockReturnValue({ + token: "session-1:cancel", + getConflict: vi.fn(async () => null), + dispose: vi.fn(), + [Symbol.asyncIterator](): AsyncIterator { + return { + next: () => { + if (delivered) return new Promise>(() => {}); + delivered = true; + return firstRead; + }, + return: vi.fn(async () => ({ done: true, value: undefined })), + }; + }, + }); + + const control = await createTurnCancellationControl({ + expectedTurnId: "turn_0", + sessionId: "session-1", + }); + + let releaseStep!: () => void; + const step = new Promise((resolve) => { + releaseStep = resolve; + }); + const abortedAtStepContinuation = step.then(() => control!.signal.aborted); + + // Journal order: the cancel payload resolves before the step result. + releasePayload({ done: false, value: {} }); + releaseStep(); + + await expect(abortedAtStepContinuation).resolves.toBe(true); + await expect(control!.requested).resolves.toBe("cancel"); + }); + it("disposes idempotently", async () => { const { dispose } = installCancelHook({}); diff --git a/packages/eve/src/execution/turn-cancellation-control.ts b/packages/eve/src/execution/turn-cancellation-control.ts index 1869ce4f7..18f0a58e3 100644 --- a/packages/eve/src/execution/turn-cancellation-control.ts +++ b/packages/eve/src/execution/turn-cancellation-control.ts @@ -51,12 +51,15 @@ export async function createTurnCancellationControl(input: { } const controller = new AbortController(); - // The durable abort fires in the read's continuation so its call site - // is reached deterministically on every replay. - const requested = consumeMatchingCancel(iterator, input.expectedTurnId).then(() => { + // The abort fires inside the read continuation itself — not a chained + // `.then` — so the signal flips in the same microtask that consumes the + // payload. When a wake replays a journaled cancel alongside a completed + // step, the turn loop checks `signal.aborted` in the step's continuation; + // a chained abort would run one microtask later and lose to that check, + // letting an ordinary completion swallow the cancel. + const requested = consumeMatchingCancel(iterator, input.expectedTurnId, () => { controller.abort(new TurnCancelledError()); - return "cancel" as const; - }); + }).then(() => "cancel" as const); let disposed = false; return { @@ -73,15 +76,21 @@ export async function createTurnCancellationControl(input: { } // Mismatched turn guards are consumed as no-ops; each read is durable, -// so the skip sequence replays deterministically. +// so the skip sequence replays deterministically. `onCancel` runs +// synchronously within the matching read's continuation (see the abort +// ordering note in `createTurnCancellationControl`). async function consumeMatchingCancel( iterator: AsyncIterator, expectedTurnId: string, + onCancel: () => void, ): Promise { while (true) { const next = await iterator.next(); if (next.done) return await new Promise(() => {}); - if (matchesActiveTurn(next.value, expectedTurnId)) return; + if (matchesActiveTurn(next.value, expectedTurnId)) { + onCancel(); + return; + } } } From 66ca1a2874efc9b8635c77116e8a25c4c8b2229e Mon Sep 17 00:00:00 2001 From: Andrew Barba Date: Wed, 29 Jul 2026 23:09:48 -0400 Subject: [PATCH 4/9] fix(eve): fan turn cancellation out to descendants at request time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The descendant cascade previously lived only inside the parent's workflow (cancelDescendantTurnsStep), which cannot execute while the parent run is suspended — including while it is parked awaiting the very child it needs to cancel. When the workflow world deferred the parent's cancel wake until the child's completion delivery, the cascade was dead on arrival: the child ran its full fallback timeout and completed uncancelled (observed repeatedly in the cancel-subagent e2e eval). runtime.cancelTurn now cancels the target and immediately fans out to every dispatched-but-unresolved descendant, recursively, without any run having to wake. Descendants are derived from the session's own durable event stream — subagent.called records the dispatch and action.result marks it resolved — read up to the tail observed on entry so a live stream cannot block the request. Remote children are re-resolved from the registry by dispatch URL and cancelled over the standard remote cancel route, which fans out on its own deployment. The settle-time cascade remains as the durable backstop, but no longer burns its 8-attempt retry budget on children that were already cancelled at request time: terminal no_active_turn reasons (hook gone, run gone, run expired) return after a single attempt, and only hook-claim contention is retried. Signed-off-by: Andrew Barba --- .changeset/request-time-cancel-fanout.md | 5 + docs/concepts/sessions-runs-and-streaming.md | 2 +- docs/guides/remote-agents.md | 2 +- .../cancel-descendant-turns-step.test.ts | 49 ++- .../execution/cancel-descendant-turns-step.ts | 70 ++- .../turn-cancellation-fanout.test.ts | 290 ++++++++++++ .../src/execution/turn-cancellation-fanout.ts | 412 ++++++++++++++++++ .../execution/turn-cancellation-request.ts | 49 +++ .../eve/src/execution/workflow-runtime.ts | 42 +- 9 files changed, 859 insertions(+), 62 deletions(-) create mode 100644 .changeset/request-time-cancel-fanout.md create mode 100644 packages/eve/src/execution/turn-cancellation-fanout.test.ts create mode 100644 packages/eve/src/execution/turn-cancellation-fanout.ts create mode 100644 packages/eve/src/execution/turn-cancellation-request.ts diff --git a/.changeset/request-time-cancel-fanout.md b/.changeset/request-time-cancel-fanout.md new file mode 100644 index 000000000..d6755d2cc --- /dev/null +++ b/.changeset/request-time-cancel-fanout.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Cancelling a session now fans the cancellation out to its subagent descendants immediately at request time, instead of only when the parent's workflow run wakes and settles. A parent suspended on the very child it needs to cancel can no longer strand that child: descendants are derived from the session's durable event stream and cancelled recursively (local and remote) as soon as the cancel request is accepted, with the settle-time cascade remaining as a durable backstop. diff --git a/docs/concepts/sessions-runs-and-streaming.md b/docs/concepts/sessions-runs-and-streaming.md index 0859c5163..7feeff208 100644 --- a/docs/concepts/sessions-runs-and-streaming.md +++ b/docs/concepts/sessions-runs-and-streaming.md @@ -163,7 +163,7 @@ curl -X POST http://127.0.0.1:2000/eve/v1/session//cancel # {"ok":true,"sessionId":"","status":"accepted"} ``` -`"accepted"` means a cancellation hook accepted the request. Confirm cancellation on the stream as `turn.cancelled` followed by `session.waiting`; the session then accepts the next message normally. If the turn is waiting on active local or remote subagents, eve also requests cancellation of every adopted child, recursively, before settling the parent. Each child reports its own cancellation boundary on its child-session stream; the parent does not emit `subagent.completed` for cancelled work. `"no_active_turn"` means no resumable cancellation target exists, including an unknown session or an already-settled turn. Both statuses are success, so clients can fire and forget. See the [eve channel](../channels/eve) for the full route contract. +`"accepted"` means a cancellation hook accepted the request. Confirm cancellation on the stream as `turn.cancelled` followed by `session.waiting`; the session then accepts the next message normally. If the turn is waiting on active local or remote subagents, eve also requests cancellation of every adopted child, recursively. The fan-out happens twice: immediately when the cancel request is accepted — so children stop promptly even while the parent run is still suspended — and again as a durable backstop before the parent settles. Each child reports its own cancellation boundary on its child-session stream; the parent does not emit `subagent.completed` for cancelled work. `"no_active_turn"` means no resumable cancellation target exists, including an unknown session or an already-settled turn. Both statuses are success, so clients can fire and forget. See the [eve channel](../channels/eve) for the full route contract. Custom channel routes request the same cancellation without knowing the session id: the `cancel` route helper is addressed by the channel-local continuation token, and `Session.cancel()` by session id. See [custom channels](../channels/custom#cancel-a-turn). diff --git a/docs/guides/remote-agents.md b/docs/guides/remote-agents.md index 8f876c7cc..faab018df 100644 --- a/docs/guides/remote-agents.md +++ b/docs/guides/remote-agents.md @@ -93,7 +93,7 @@ A local subagent runs inline. A remote one runs in its own deployment, so dispat The parent stream carries the same `subagent.called`, `action.result`, and `subagent.completed` events as local delegation. For a remote call, `subagent.called.data.remote.url` records the target. -Cancelling the parent while a remote call is active sends an authenticated `POST /eve/v1/session/:childSessionId/cancel` to the remote and waits for that request to be accepted before the parent settles. eve resolves the remote's `headers` and `auth` again for every cancellation attempt, so rotating credentials work the same way as they do for session creation. Cancellation always uses the standard eve cancel path on `url`, even when `path` customizes only the create-session endpoint. The remote child reports `turn.cancelled` → `session.waiting` on its own stream; an older or unreachable remote is logged but cannot turn the parent's cancellation into a failure. +Cancelling the parent while a remote call is active sends an authenticated `POST /eve/v1/session/:childSessionId/cancel` to the remote — once immediately when the parent's cancel request is accepted, and again as a durable backstop before the parent settles. eve resolves the remote's `headers` and `auth` again for every cancellation attempt, so rotating credentials work the same way as they do for session creation. Cancellation always uses the standard eve cancel path on `url`, even when `path` customizes only the create-session endpoint. The remote child reports `turn.cancelled` → `session.waiting` on its own stream; an older or unreachable remote is logged but cannot turn the parent's cancellation into a failure. Both failure paths surface to the parent as a failed tool result, so the caller can explain or recover within the same session. A failed _start_ returns the error inline. A remote that starts and then fails posts a terminal failure callback, which the parent receives as an errored subagent result carrying the remote's error (or `REMOTE_AGENT_FAILED` when none is supplied). Terminal callback delivery runs as a durable step on the underlying workflow engine (see [Execution model & durability](../concepts/execution-model-and-durability)). A failed callback POST is rethrown rather than marking the task complete, so the engine retries it. diff --git a/packages/eve/src/execution/cancel-descendant-turns-step.test.ts b/packages/eve/src/execution/cancel-descendant-turns-step.test.ts index 9dbe9ac62..cbb86e680 100644 --- a/packages/eve/src/execution/cancel-descendant-turns-step.test.ts +++ b/packages/eve/src/execution/cancel-descendant-turns-step.test.ts @@ -8,7 +8,7 @@ import { isRetryableRemoteAgentCancelError, resolveRemoteAgentForAction, } from "#execution/remote-agent-dispatch.js"; -import { requestWorkflowTurnCancellation } from "#execution/workflow-runtime.js"; +import { requestWorkflowTurnCancellation } from "#execution/turn-cancellation-request.js"; import { recordPendingSubagentChild, setPendingRuntimeActionBatch, @@ -20,7 +20,8 @@ vi.mock("#context/serialize.js", () => ({ deserializeContext: vi.fn(), })); -vi.mock("./workflow-runtime.js", () => ({ +vi.mock("./turn-cancellation-request.js", () => ({ + isRetryableInactiveCancelReason: (reason: string | undefined) => reason === "EntityConflictError", requestWorkflowTurnCancellation: vi.fn(), })); @@ -103,16 +104,14 @@ describe("cancelDescendantTurnsStep", () => { expect(deserializeContext).not.toHaveBeenCalled(); }); - it("retries no-active-turn responses during the child adoption window", async () => { + it("retries hook-claim conflicts during the child adoption window", async () => { vi.useFakeTimers(); installRemoteRegistry(); vi.mocked(resolveRemoteAgentForAction).mockReturnValue(remote); vi.mocked(requestWorkflowTurnCancellation) - .mockResolvedValueOnce({ status: "no_active_turn" }) - .mockResolvedValueOnce({ status: "accepted" }); - vi.mocked(cancelRemoteAgentTurn) - .mockResolvedValueOnce({ status: "no_active_turn" }) + .mockResolvedValueOnce({ reason: "EntityConflictError", status: "no_active_turn" }) .mockResolvedValueOnce({ status: "accepted" }); + vi.mocked(cancelRemoteAgentTurn).mockResolvedValue({ status: "accepted" }); const cancellation = cancelDescendantTurnsStep({ serializedContext: {}, @@ -122,7 +121,41 @@ describe("cancelDescendantTurnsStep", () => { await cancellation; expect(requestWorkflowTurnCancellation).toHaveBeenCalledTimes(2); - expect(cancelRemoteAgentTurn).toHaveBeenCalledTimes(2); + }); + + it("treats a terminal no-active-turn child as settled after a single attempt", async () => { + // The request-time fan-out usually cancels descendants before this + // durable backstop runs; a missing hook must not burn the retry budget. + vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ + reason: "HookNotFoundError", + status: "no_active_turn", + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await cancelDescendantTurnsStep({ + serializedContext: {}, + sessionState: createPendingState({ includeRemote: false }), + }); + + expect(requestWorkflowTurnCancellation).toHaveBeenCalledOnce(); + expect(warn).not.toHaveBeenCalledWith( + expect.stringContaining("descendant cancel was never accepted"), + expect.anything(), + ); + }); + + it("does not retry a remote no-active-turn result", async () => { + installRemoteRegistry(); + vi.mocked(resolveRemoteAgentForAction).mockReturnValue(remote); + vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); + vi.mocked(cancelRemoteAgentTurn).mockResolvedValue({ status: "no_active_turn" }); + + await cancelDescendantTurnsStep({ + serializedContext: {}, + sessionState: createPendingState(), + }); + + expect(cancelRemoteAgentTurn).toHaveBeenCalledOnce(); }); it("contains unexpected local failures and exhausted remote failures", async () => { diff --git a/packages/eve/src/execution/cancel-descendant-turns-step.ts b/packages/eve/src/execution/cancel-descendant-turns-step.ts index b7d6a7265..37d6d43c5 100644 --- a/packages/eve/src/execution/cancel-descendant-turns-step.ts +++ b/packages/eve/src/execution/cancel-descendant-turns-step.ts @@ -7,7 +7,10 @@ import { isRetryableRemoteAgentCancelError, resolveRemoteAgentForAction, } from "#execution/remote-agent-dispatch.js"; -import { requestWorkflowTurnCancellation } from "#execution/workflow-runtime.js"; +import { + isRetryableInactiveCancelReason, + requestWorkflowTurnCancellation, +} from "#execution/turn-cancellation-request.js"; import { getPendingRuntimeActionBatch } from "#harness/runtime-actions.js"; import { createLogger, logError } from "#internal/logging.js"; import type { @@ -16,11 +19,13 @@ import type { } from "#runtime/actions/types.js"; import type { RuntimeSubagentRegistry } from "#runtime/subagents/registry.js"; -// Descendants listed in a pending batch should be actively running, so a -// cancel that cannot be delivered is anomalous: retry with backoff long -// enough to ride out transient world contention (queue wakes, hook-claim -// conflicts), then log loudly rather than dropping the cancel silently — -// an uncancelled child otherwise runs to completion with no trace of why. +// This step is the durable backstop behind the request-time fan-out +// (`cancelTurnWithDescendantFanout`), which usually reaches descendants +// before the parent's run ever wakes. By the time this step executes, a +// child with no active turn was most often already cancelled at request +// time — a terminal outcome, returned immediately. Only reasons that can +// heal with time (hook-claim conflicts during queue wakes) are retried, +// and an exhausted retry is logged loudly rather than dropped silently. const CANCEL_ATTEMPTS = 8; const CANCEL_RETRY_INITIAL_DELAY_MS = 250; const CANCEL_RETRY_MAX_DELAY_MS = 1_500; @@ -95,15 +100,25 @@ async function cancelLocalDescendant(input: { const final = await requestCancellationWithRetry({ request: () => requestWorkflowTurnCancellation({ sessionId: input.childSessionId }), shouldRetryError: () => false, + shouldRetryResult: (result) => isRetryableInactiveCancelReason(result.reason), }); if (final.status !== "accepted") { - log.warn("descendant cancel was never accepted; the child may run to completion", { - callId: input.action.callId, - childSessionId: input.childSessionId, - finalStatus: final.status, - reason: final.reason, - subagentName: input.action.subagentName, - }); + if (isRetryableInactiveCancelReason(final.reason)) { + log.warn("descendant cancel was never accepted; the child may run to completion", { + callId: input.action.callId, + childSessionId: input.childSessionId, + finalStatus: final.status, + reason: final.reason, + subagentName: input.action.subagentName, + }); + } else { + log.info("descendant had no active turn at settle time; already cancelled or finished", { + callId: input.action.callId, + childSessionId: input.childSessionId, + reason: final.reason, + subagentName: input.action.subagentName, + }); + } } } catch (error) { logError(log, "failed to cancel local descendant turn", error, { @@ -130,15 +145,21 @@ async function cancelRemoteDescendant(input: { const final = await requestCancellationWithRetry({ request: () => cancelRemoteAgentTurn({ remote, sessionId: input.childSessionId }), shouldRetryError: isRetryableRemoteAgentCancelError, + // The remote deployment already classified its own turn state; a + // remote no_active_turn is terminal. + shouldRetryResult: () => false, }); if (final.status !== "accepted") { - log.warn("remote descendant cancel was never accepted; the child may run to completion", { - callId: input.action.callId, - childSessionId: input.childSessionId, - finalStatus: final.status, - reason: final.reason, - remoteAgentName: input.action.remoteAgentName, - }); + log.info( + "remote descendant had no active turn at settle time; already cancelled or finished", + { + callId: input.action.callId, + childSessionId: input.childSessionId, + finalStatus: final.status, + reason: final.reason, + remoteAgentName: input.action.remoteAgentName, + }, + ); } } catch (error) { logError(log, "failed to cancel remote descendant turn", error, { @@ -152,6 +173,7 @@ async function cancelRemoteDescendant(input: { async function requestCancellationWithRetry(input: { readonly request: () => Promise; readonly shouldRetryError: (error: unknown) => boolean; + readonly shouldRetryResult: (result: CancelTurnResult) => boolean; }): Promise { let delayMs = CANCEL_RETRY_INITIAL_DELAY_MS; let lastResult: CancelTurnResult = { status: "no_active_turn" }; @@ -159,7 +181,13 @@ async function requestCancellationWithRetry(input: { for (let attempt = 1; attempt <= CANCEL_ATTEMPTS; attempt += 1) { try { lastResult = await input.request(); - if (lastResult.status === "accepted" || attempt === CANCEL_ATTEMPTS) return lastResult; + if ( + lastResult.status === "accepted" || + !input.shouldRetryResult(lastResult) || + attempt === CANCEL_ATTEMPTS + ) { + return lastResult; + } } catch (error) { if (!input.shouldRetryError(error) || attempt === CANCEL_ATTEMPTS) throw error; } diff --git a/packages/eve/src/execution/turn-cancellation-fanout.test.ts b/packages/eve/src/execution/turn-cancellation-fanout.test.ts new file mode 100644 index 000000000..3157cddcf --- /dev/null +++ b/packages/eve/src/execution/turn-cancellation-fanout.test.ts @@ -0,0 +1,290 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { cancelRemoteAgentTurn } from "#execution/remote-agent-dispatch.js"; +import { + cancelTurnWithDescendantFanout, + collectPendingDescendants, +} from "#execution/turn-cancellation-fanout.js"; +import { requestWorkflowTurnCancellation } from "#execution/turn-cancellation-request.js"; +import { getRun } from "#internal/workflow/runtime.js"; + +vi.mock("#internal/workflow/runtime.js", () => ({ + getRun: vi.fn(), +})); + +vi.mock("./turn-cancellation-request.js", () => ({ + requestWorkflowTurnCancellation: vi.fn(), +})); + +vi.mock("./remote-agent-dispatch.js", () => ({ + cancelRemoteAgentTurn: vi.fn(), + isRetryableRemoteAgentCancelError: vi.fn(() => false), +})); + +afterEach(() => { + vi.clearAllMocks(); +}); + +function subagentCalled(input: { + readonly callId: string; + readonly childSessionId: string; + readonly remote?: { readonly url: string }; + readonly turnId?: string; +}) { + return { + data: { + callId: input.callId, + childSessionId: input.childSessionId, + name: "child", + remote: input.remote, + sequence: 0, + sessionId: "parent", + toolName: "child", + turnId: input.turnId ?? "turn_0", + workflowId: "workflow//eve//workflowEntry", + }, + type: "subagent.called", + }; +} + +function actionResult(callId: string) { + return { + data: { + result: { callId, kind: "subagent-result", output: "done", subagentName: "child" }, + sequence: 0, + status: "completed", + stepIndex: 0, + turnId: "turn_0", + }, + type: "action.result", + }; +} + +/** + * Mimics a live run stream: journaled chunks are followed by an open tail + * that never closes, exactly like a parked durable session's event stream. + */ +function installEventStreams(streamsBySessionId: Record): void { + vi.mocked(getRun).mockImplementation( + (sessionId: string) => + ({ + getReadable: () => { + const events = streamsBySessionId[sessionId] ?? []; + const chunks = events.map((event) => + new TextEncoder().encode(`${JSON.stringify(event)}\n`), + ); + let index = 0; + const stream = new ReadableStream({ + pull(controller) { + const chunk = chunks[index]; + if (chunk !== undefined) { + controller.enqueue(chunk); + index += 1; + } + // Past the tail the stream stays open, like a live run. + }, + }) as ReadableStream & { getTailIndex: () => Promise }; + stream.getTailIndex = async () => chunks.length - 1; + return stream; + }, + }) as never, + ); +} + +describe("collectPendingDescendants", () => { + it("returns dispatched children without results and skips resolved ones", () => { + const records = collectPendingDescendants({ + events: [ + subagentCalled({ callId: "call-1", childSessionId: "child-1" }), + subagentCalled({ callId: "call-2", childSessionId: "child-2" }), + actionResult("call-1"), + ], + }); + + expect(records).toEqual([ + expect.objectContaining({ callId: "call-2", childSessionId: "child-2" }), + ]); + }); + + it("filters by turn id when provided", () => { + const records = collectPendingDescendants({ + events: [ + subagentCalled({ callId: "call-1", childSessionId: "child-1", turnId: "turn_0" }), + subagentCalled({ callId: "call-2", childSessionId: "child-2", turnId: "turn_1" }), + ], + turnId: "turn_1", + }); + + expect(records).toEqual([expect.objectContaining({ childSessionId: "child-2" })]); + }); + + it("captures remote identity and ignores malformed events", () => { + const records = collectPendingDescendants({ + events: [ + "not-an-event", + { type: "subagent.called", data: { callId: 42 } }, + subagentCalled({ + callId: "call-r", + childSessionId: "remote-child", + remote: { url: "https://remote.example.com" }, + }), + ], + }); + + expect(records).toEqual([ + expect.objectContaining({ + childSessionId: "remote-child", + remote: { url: "https://remote.example.com" }, + }), + ]); + }); +}); + +describe("cancelTurnWithDescendantFanout", () => { + it("cancels the target and every unresolved local descendant at request time", async () => { + installEventStreams({ + parent: [subagentCalled({ callId: "call-1", childSessionId: "child-1" })], + }); + vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); + + const result = await cancelTurnWithDescendantFanout({ sessionId: "parent" }); + + expect(result).toEqual({ status: "accepted" }); + expect(requestWorkflowTurnCancellation).toHaveBeenCalledWith({ sessionId: "parent" }); + expect(requestWorkflowTurnCancellation).toHaveBeenCalledWith({ sessionId: "child-1" }); + }); + + it("recurses into local descendants to cancel grandchildren", async () => { + installEventStreams({ + "child-1": [subagentCalled({ callId: "call-2", childSessionId: "grandchild-1" })], + grandchild: [], + parent: [subagentCalled({ callId: "call-1", childSessionId: "child-1" })], + }); + vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); + + await cancelTurnWithDescendantFanout({ sessionId: "parent" }); + + expect(requestWorkflowTurnCancellation).toHaveBeenCalledWith({ sessionId: "grandchild-1" }); + }); + + it("skips already-visited sessions so cycles cannot recurse forever", async () => { + installEventStreams({ + "child-1": [subagentCalled({ callId: "call-back", childSessionId: "parent" })], + parent: [subagentCalled({ callId: "call-1", childSessionId: "child-1" })], + }); + vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); + + await cancelTurnWithDescendantFanout({ sessionId: "parent" }); + + const cancelledSessions = vi + .mocked(requestWorkflowTurnCancellation) + .mock.calls.map(([input]) => input.sessionId); + expect(cancelledSessions).toEqual(["parent", "child-1"]); + }); + + it("cancels remote descendants through the url-matched registry node", async () => { + installEventStreams({ + parent: [ + subagentCalled({ + callId: "call-r", + childSessionId: "remote-child", + remote: { url: "https://remote.example.com" }, + }), + ], + }); + const remoteNode = { kind: "remote", name: "child", url: "https://remote.example.com" }; + const registry = new Map([["subagents/remote", { definition: remoteNode }]]); + vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); + vi.mocked(cancelRemoteAgentTurn).mockResolvedValue({ status: "accepted" }); + + await cancelTurnWithDescendantFanout({ + resolveRemoteRegistry: async () => registry as never, + sessionId: "parent", + }); + + expect(cancelRemoteAgentTurn).toHaveBeenCalledWith({ + remote: remoteNode, + sessionId: "remote-child", + }); + // Remote descendants are cancelled by their own deployment; no local recursion. + expect(requestWorkflowTurnCancellation).not.toHaveBeenCalledWith({ + sessionId: "remote-child", + }); + }); + + it("defers a remote descendant whose url matches no registered agent", async () => { + installEventStreams({ + parent: [ + subagentCalled({ + callId: "call-r", + childSessionId: "remote-child", + remote: { url: "https://unknown.example.com" }, + }), + ], + }); + vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await cancelTurnWithDescendantFanout({ + resolveRemoteRegistry: async () => new Map() as never, + sessionId: "parent", + }); + + expect(cancelRemoteAgentTurn).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("matched no registered remote agent"), + expect.anything(), + ); + }); + + it("retries a local child caught in its dispatch gap", async () => { + vi.useFakeTimers(); + try { + installEventStreams({ + parent: [subagentCalled({ callId: "call-1", childSessionId: "child-1" })], + }); + vi.mocked(requestWorkflowTurnCancellation) + .mockResolvedValueOnce({ status: "accepted" }) // parent + .mockResolvedValueOnce({ reason: "HookNotFoundError", status: "no_active_turn" }) + .mockResolvedValueOnce({ status: "accepted" }); + + const fanout = cancelTurnWithDescendantFanout({ sessionId: "parent" }); + await vi.advanceTimersByTimeAsync(1_000); + await fanout; + + const childCancels = vi + .mocked(requestWorkflowTurnCancellation) + .mock.calls.filter(([input]) => input.sessionId === "child-1"); + expect(childCancels).toHaveLength(2); + } finally { + vi.useRealTimers(); + } + }); + + it("returns the target result even when fan-out fails", async () => { + vi.mocked(getRun).mockImplementation(() => { + throw new Error("world unavailable"); + }); + vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + + const result = await cancelTurnWithDescendantFanout({ sessionId: "parent" }); + + expect(result).toEqual({ status: "accepted" }); + expect(error).toHaveBeenCalled(); + }); + + it("still fans out when the target itself has no active turn", async () => { + installEventStreams({ + parent: [subagentCalled({ callId: "call-1", childSessionId: "orphan-child" })], + }); + vi.mocked(requestWorkflowTurnCancellation) + .mockResolvedValueOnce({ reason: "HookNotFoundError", status: "no_active_turn" }) + .mockResolvedValue({ status: "accepted" }); + + const result = await cancelTurnWithDescendantFanout({ sessionId: "parent" }); + + expect(result).toEqual({ reason: "HookNotFoundError", status: "no_active_turn" }); + expect(requestWorkflowTurnCancellation).toHaveBeenCalledWith({ sessionId: "orphan-child" }); + }); +}); diff --git a/packages/eve/src/execution/turn-cancellation-fanout.ts b/packages/eve/src/execution/turn-cancellation-fanout.ts new file mode 100644 index 000000000..2bc0ee4ec --- /dev/null +++ b/packages/eve/src/execution/turn-cancellation-fanout.ts @@ -0,0 +1,412 @@ +/** + * Request-time descendant cancellation. + * + * The durable settle-time cascade + * ({@link import("#execution/cancel-descendant-turns-step.js").cancelDescendantTurnsStep}) + * runs inside the parent's workflow, which cannot execute while the parent + * run is suspended — including while it is parked awaiting the very child it + * needs to cancel. A cancel request therefore fans out to descendants + * immediately at the API boundary, before any run has to wake. + * + * The session's own event stream is the durable record of dispatched + * children: `subagent.called` carries `{ callId, childSessionId, + * remote?.url, turnId }` and `action.result` marks the call resolved. Both + * are readable by session id from any process. Cancelling an + * already-finished child resolves to `no_active_turn` and is harmless, so + * the fold errs toward over-cancelling. + */ + +import type { CancelTurnResult } from "#channel/types.js"; +import { createLogger, logError } from "#internal/logging.js"; +import { getRun } from "#internal/workflow/runtime.js"; +import { + cancelRemoteAgentTurn, + isRetryableRemoteAgentCancelError, +} from "#execution/remote-agent-dispatch.js"; +import { requestWorkflowTurnCancellation } from "#execution/turn-cancellation-request.js"; +import type { RuntimeSubagentRegistry } from "#runtime/subagents/registry.js"; +import type { ResolvedRuntimeRemoteAgentNode } from "#runtime/types.js"; + +const log = createLogger("execution.turn-cancellation-fanout"); + +// Fan-out runs inline in the cancel request handler, so every bound is +// small: a short per-child retry covers the dispatch gap where a child run +// exists but its cancel hook is not yet armed, and the stream read deadline +// keeps a wedged world from stalling the response. The durable settle-time +// cascade remains the backstop for anything missed here. +const FANOUT_ATTEMPTS = 3; +const FANOUT_RETRY_DELAY_MS = 250; +const MAX_DESCENDANT_DEPTH = 8; +const STREAM_READ_DEADLINE_MS = 5_000; + +type RemoteRegistry = RuntimeSubagentRegistry["subagentsByNodeId"]; + +/** One dispatched-but-unresolved child derived from a session's events. */ +export interface PendingDescendantRecord { + readonly callId: string; + readonly childSessionId: string; + readonly remote?: { readonly url: string }; + readonly toolName: string; + readonly turnId: string; +} + +/** + * Cancels a session's active turn and immediately fans the cancellation out + * to every dispatched-but-unresolved descendant, without waiting for the + * target's workflow run to wake. Fan-out failures are logged, never thrown: + * the returned result reflects the target session alone. + */ +export async function cancelTurnWithDescendantFanout(input: { + readonly sessionId: string; + readonly turnId?: string; + readonly resolveRemoteRegistry?: () => Promise; +}): Promise { + const cancelInput: { sessionId: string; turnId?: string } = { sessionId: input.sessionId }; + if (input.turnId !== undefined) cancelInput.turnId = input.turnId; + const result = await requestWorkflowTurnCancellation(cancelInput); + + // Fan out even when the target reports no active turn: descendants that + // outlived an earlier cancel (or a crashed parent) are exactly the ones + // only a fresh request can still reach. + try { + await cancelDescendantSessions({ + depth: 0, + resolveRemoteRegistry: input.resolveRemoteRegistry, + sessionId: input.sessionId, + turnId: input.turnId, + visited: new Set([input.sessionId]), + }); + } catch (error) { + logError(log, "descendant cancel fan-out failed", error, { sessionId: input.sessionId }); + } + + return result; +} + +/** + * Folds a session's events into its dispatched-but-unresolved children. + * When `turnId` is set, only children dispatched by that turn are returned; + * malformed entries are skipped. + */ +export function collectPendingDescendants(input: { + readonly events: Iterable; + readonly turnId?: string; +}): PendingDescendantRecord[] { + const pending = new Map(); + + for (const event of input.events) { + if (typeof event !== "object" || event === null) continue; + const candidate = event as { readonly data?: unknown; readonly type?: unknown }; + + if (candidate.type === "subagent.called") { + const record = parseSubagentCalledData(candidate.data); + if (record !== undefined) pending.set(record.callId, record); + continue; + } + + if (candidate.type === "action.result") { + const callId = parseActionResultCallId(candidate.data); + if (callId !== undefined) pending.delete(callId); + } + } + + const records = [...pending.values()]; + return input.turnId === undefined + ? records + : records.filter((record) => record.turnId === input.turnId); +} + +async function cancelDescendantSessions(input: { + readonly depth: number; + readonly resolveRemoteRegistry?: () => Promise; + readonly sessionId: string; + readonly turnId?: string; + readonly visited: Set; +}): Promise { + if (input.depth >= MAX_DESCENDANT_DEPTH) { + log.warn("descendant cancel fan-out hit the depth cap", { + depth: input.depth, + sessionId: input.sessionId, + }); + return; + } + + let records: PendingDescendantRecord[]; + try { + records = collectPendingDescendants({ + events: await readSessionEventsUpToTail(input.sessionId), + turnId: input.turnId, + }); + } catch (error) { + logError(log, "failed to read session events during cancel fan-out", error, { + sessionId: input.sessionId, + }); + return; + } + + await Promise.all( + records.map(async (record) => { + if (input.visited.has(record.childSessionId)) return; + input.visited.add(record.childSessionId); + + if (record.remote !== undefined) { + await cancelRemoteDescendantAtRequest({ + record, + resolveRemoteRegistry: input.resolveRemoteRegistry, + }); + return; + } + + await cancelLocalDescendantAtRequest(record); + // Grandchildren are cancelled without a turn filter: whatever is still + // unresolved under a cancelled child must not keep running. + await cancelDescendantSessions({ + depth: input.depth + 1, + resolveRemoteRegistry: input.resolveRemoteRegistry, + sessionId: record.childSessionId, + visited: input.visited, + }); + }), + ); +} + +async function cancelLocalDescendantAtRequest(record: PendingDescendantRecord): Promise { + try { + const final = await requestCancellationWithRetry({ + request: () => requestWorkflowTurnCancellation({ sessionId: record.childSessionId }), + shouldRetryError: () => false, + shouldRetryResult: (result) => + // A child in its dispatch gap has no cancel hook yet, which is + // indistinguishable from an already-finished child — retrying briefly + // covers the former and only costs no-ops on the latter. + result.status === "no_active_turn", + }); + if (final.status !== "accepted") { + log.info("descendant had no active turn during request-time cancel fan-out", { + callId: record.callId, + childSessionId: record.childSessionId, + reason: final.reason, + toolName: record.toolName, + }); + } + } catch (error) { + logError(log, "failed to cancel local descendant at request time", error, { + callId: record.callId, + childSessionId: record.childSessionId, + toolName: record.toolName, + }); + } +} + +async function cancelRemoteDescendantAtRequest(input: { + readonly record: PendingDescendantRecord; + readonly resolveRemoteRegistry?: () => Promise; +}): Promise { + const { record } = input; + if (record.remote === undefined || input.resolveRemoteRegistry === undefined) { + log.warn("remote descendant cannot be cancelled at request time; deferring to settle-time", { + callId: record.callId, + childSessionId: record.childSessionId, + hasRegistry: input.resolveRemoteRegistry !== undefined, + toolName: record.toolName, + }); + return; + } + + try { + const remote = findRemoteAgentByUrl({ + registry: await input.resolveRemoteRegistry(), + url: record.remote.url, + }); + if (remote === undefined) { + log.warn("remote descendant url matched no registered remote agent; deferring", { + callId: record.callId, + childSessionId: record.childSessionId, + toolName: record.toolName, + url: record.remote.url, + }); + return; + } + const final = await requestCancellationWithRetry({ + request: () => cancelRemoteAgentTurn({ remote, sessionId: record.childSessionId }), + shouldRetryError: isRetryableRemoteAgentCancelError, + // The remote deployment already classified its own turn state; a + // remote no_active_turn is terminal. + shouldRetryResult: () => false, + }); + if (final.status !== "accepted") { + log.info("remote descendant had no active turn during request-time cancel fan-out", { + callId: record.callId, + childSessionId: record.childSessionId, + toolName: record.toolName, + }); + } + } catch (error) { + logError(log, "failed to cancel remote descendant at request time", error, { + callId: record.callId, + childSessionId: record.childSessionId, + toolName: record.toolName, + }); + } +} + +async function requestCancellationWithRetry(input: { + readonly request: () => Promise; + readonly shouldRetryError: (error: unknown) => boolean; + readonly shouldRetryResult: (result: CancelTurnResult) => boolean; +}): Promise { + let lastResult: CancelTurnResult = { status: "no_active_turn" }; + + for (let attempt = 1; attempt <= FANOUT_ATTEMPTS; attempt += 1) { + try { + lastResult = await input.request(); + if (lastResult.status === "accepted" || !input.shouldRetryResult(lastResult)) { + return lastResult; + } + } catch (error) { + if (!input.shouldRetryError(error) || attempt === FANOUT_ATTEMPTS) throw error; + } + if (attempt < FANOUT_ATTEMPTS) { + await new Promise((resolve) => setTimeout(resolve, FANOUT_RETRY_DELAY_MS * attempt)); + } + } + + return lastResult; +} + +/** + * Reads every event currently journaled on the session's stream and stops at + * the tail observed on entry, so a live (never-closing) run stream cannot + * block the cancel request. Chunks map 1:1 to stream writes; each carries + * one or more NDJSON lines. + */ +async function readSessionEventsUpToTail(sessionId: string): Promise { + const stream = getRun(sessionId).getReadable(); + const tailIndex = await stream.getTailIndex(); + if (tailIndex < 0) { + await stream.cancel("eve cancel fan-out: session has no events").catch(() => {}); + return []; + } + + const reader = stream.getReader(); + const decoder = new TextDecoder(); + const events: unknown[] = []; + let buffer = ""; + let chunksRead = 0; + let deadlineHit = false; + const deadline = setTimeout(() => { + deadlineHit = true; + void reader.cancel("eve cancel fan-out: stream read deadline reached").catch(() => {}); + }, STREAM_READ_DEADLINE_MS); + + const drainBuffer = () => { + for ( + let newlineIndex = buffer.indexOf("\n"); + newlineIndex !== -1; + newlineIndex = buffer.indexOf("\n") + ) { + const line = buffer.slice(0, newlineIndex).trim(); + buffer = buffer.slice(newlineIndex + 1); + if (line.length === 0) continue; + try { + events.push(JSON.parse(line)); + } catch { + // A single undecodable line must not abort the whole fan-out. + } + } + }; + + try { + while (chunksRead <= tailIndex) { + const { done, value } = await reader.read(); + if (done) break; + chunksRead += 1; + buffer += decoder.decode(value, { stream: true }); + drainBuffer(); + } + buffer += decoder.decode(); + buffer += "\n"; + drainBuffer(); + } finally { + clearTimeout(deadline); + await reader.cancel("eve cancel fan-out: read complete").catch(() => {}); + reader.releaseLock(); + } + + if (deadlineHit) { + log.warn("session event read hit the fan-out deadline; descendants may be incomplete", { + chunksRead, + sessionId, + tailIndex, + }); + } + + return events; +} + +function parseSubagentCalledData(data: unknown): PendingDescendantRecord | undefined { + if (typeof data !== "object" || data === null) return undefined; + const value = data as { + readonly callId?: unknown; + readonly childSessionId?: unknown; + readonly remote?: unknown; + readonly toolName?: unknown; + readonly turnId?: unknown; + }; + if ( + typeof value.callId !== "string" || + typeof value.childSessionId !== "string" || + typeof value.toolName !== "string" || + typeof value.turnId !== "string" + ) { + return undefined; + } + + const record: { + callId: string; + childSessionId: string; + remote?: { url: string }; + toolName: string; + turnId: string; + } = { + callId: value.callId, + childSessionId: value.childSessionId, + toolName: value.toolName, + turnId: value.turnId, + }; + + if ( + typeof value.remote === "object" && + value.remote !== null && + typeof (value.remote as { readonly url?: unknown }).url === "string" + ) { + record.remote = { url: (value.remote as { readonly url: string }).url }; + } + + return record; +} + +/** + * The `subagent.called` event records a remote child's dispatch URL but not + * its registry node, so the cancel request re-resolves the node (and with it + * the remote's auth and headers) by URL. Distinct nodes sharing a URL target + * the same deployment, so any match cancels the same session. + */ +function findRemoteAgentByUrl(input: { + readonly registry: RemoteRegistry; + readonly url: string; +}): ResolvedRuntimeRemoteAgentNode | undefined { + for (const registered of input.registry.values()) { + const definition = registered.definition; + if (definition.kind === "remote" && definition.url === input.url) return definition; + } + return undefined; +} + +function parseActionResultCallId(data: unknown): string | undefined { + if (typeof data !== "object" || data === null) return undefined; + const result = (data as { readonly result?: unknown }).result; + if (typeof result !== "object" || result === null) return undefined; + const callId = (result as { readonly callId?: unknown }).callId; + return typeof callId === "string" ? callId : undefined; +} diff --git a/packages/eve/src/execution/turn-cancellation-request.ts b/packages/eve/src/execution/turn-cancellation-request.ts new file mode 100644 index 000000000..2f5e1d9e6 --- /dev/null +++ b/packages/eve/src/execution/turn-cancellation-request.ts @@ -0,0 +1,49 @@ +import { + EntityConflictError, + HookNotFoundError, + RunExpiredError, + WorkflowRunNotFoundError, +} from "#compiled/@workflow/errors/index.js"; + +import type { CancelTurnInput, CancelTurnResult } from "#channel/types.js"; +import { resumeHook } from "#internal/workflow/runtime.js"; +import { + sessionCancelHookToken, + type TurnCancelPayload, +} from "#execution/turn-cancellation-token.js"; + +/** Requests cancellation through a session's stable workflow hook. */ +export async function requestWorkflowTurnCancellation( + input: CancelTurnInput, +): Promise { + const payload: TurnCancelPayload = input.turnId === undefined ? {} : { turnId: input.turnId }; + + try { + await resumeHook(sessionCancelHookToken(input.sessionId), payload); + return { status: "accepted" }; + } catch (error) { + const reason = classifyInactiveCancelTarget(error); + if (reason !== undefined) { + return { reason, status: "no_active_turn" }; + } + throw error; + } +} + +/** + * Returns true when a `no_active_turn` reason can heal with time — world + * contention such as a hook-claim conflict during a queue wake. Terminal + * reasons (hook gone, run gone, run expired) mean the target turn already + * settled and will never accept a cancel again. + */ +export function isRetryableInactiveCancelReason(reason: string | undefined): boolean { + return reason === "EntityConflictError"; +} + +function classifyInactiveCancelTarget(error: unknown): string | undefined { + if (HookNotFoundError.is(error)) return "HookNotFoundError"; + if (WorkflowRunNotFoundError.is(error)) return "WorkflowRunNotFoundError"; + if (RunExpiredError.is(error)) return "RunExpiredError"; + if (EntityConflictError.is(error)) return "EntityConflictError"; + return undefined; +} diff --git a/packages/eve/src/execution/workflow-runtime.ts b/packages/eve/src/execution/workflow-runtime.ts index 9711a203a..bf1a61d9a 100644 --- a/packages/eve/src/execution/workflow-runtime.ts +++ b/packages/eve/src/execution/workflow-runtime.ts @@ -46,12 +46,9 @@ import { getCompiledRuntimeAgentBundle } from "#runtime/sessions/compiled-agent- import { buildRunContext } from "#execution/runtime-context.js"; import { parseNdjsonStream } from "#execution/ndjson-stream.js"; import { RuntimeNoActiveSessionError } from "#execution/runtime-errors.js"; +import { cancelTurnWithDescendantFanout } from "#execution/turn-cancellation-fanout.js"; import type { WorkflowEntryInput } from "#execution/workflow-entry.js"; import { walkCauseChain } from "#shared/errors.js"; -import { - sessionCancelHookToken, - type TurnCancelPayload, -} from "#execution/turn-cancellation-token.js"; const WORKFLOW_ENTRY_NAME = "workflowEntry"; const TURN_WORKFLOW_NAME = "turnWorkflow"; @@ -183,7 +180,16 @@ export function createWorkflowRuntime(config: { }, async cancelTurn(input: CancelTurnInput): Promise { - return await requestWorkflowTurnCancellation(input); + return await cancelTurnWithDescendantFanout({ + ...input, + resolveRemoteRegistry: async () => { + const bundle = await getCompiledRuntimeAgentBundle({ + compiledArtifactsSource: config.compiledArtifactsSource, + nodeId: config.nodeId, + }); + return bundle.subagentRegistry.subagentsByNodeId; + }, + }); }, async terminateSession(input: TerminateSessionInput): Promise { @@ -259,32 +265,6 @@ export function createWorkflowRuntime(config: { }; } -/** Requests cancellation through a session's stable workflow hook. */ -export async function requestWorkflowTurnCancellation( - input: CancelTurnInput, -): Promise { - const payload: TurnCancelPayload = input.turnId === undefined ? {} : { turnId: input.turnId }; - - try { - await resumeHook(sessionCancelHookToken(input.sessionId), payload); - return { status: "accepted" }; - } catch (error) { - const reason = classifyInactiveCancelTarget(error); - if (reason !== undefined) { - return { reason, status: "no_active_turn" }; - } - throw error; - } -} - -function classifyInactiveCancelTarget(error: unknown): string | undefined { - if (HookNotFoundError.is(error)) return "HookNotFoundError"; - if (WorkflowRunNotFoundError.is(error)) return "WorkflowRunNotFoundError"; - if (RunExpiredError.is(error)) return "RunExpiredError"; - if (EntityConflictError.is(error)) return "EntityConflictError"; - return undefined; -} - function isAlreadyTerminalSessionError(error: unknown): boolean { for (const candidate of walkCauseChain(error)) { if ( From a23dd001ccaf64d1b844312e71326c75a7193f7b Mon Sep 17 00:00:00 2001 From: Andrew Barba Date: Thu, 30 Jul 2026 08:27:57 -0400 Subject: [PATCH 5/9] fix(eve): re-enqueue the session run after an accepted cancel resume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workflow world does not reliably reschedule a suspended run when one of its hooks is resumed. With descendants now cancelled at request time, the parked parent no longer gets the masking wake from its child's completion delivery: on the latest e2e run the parent held an accepted cancel for 237 seconds without a single execution while its child had already settled cancelled (parent wrun_41KYRG44MR0GPGP0E13WGGP4JW, run 30510438463). requestWorkflowTurnCancellation now follows every accepted resumeHook with an explicit reenqueueRun nudge. The resume payload is already durable, so a failed nudge only re-exposes the world's wake race — never a lost cancel — and a redundant nudge is a harmless replay. Signed-off-by: Andrew Barba --- .changeset/cancel-reenqueue-nudge.md | 5 ++ .../turn-cancellation-request.test.ts | 73 +++++++++++++++++++ .../execution/turn-cancellation-request.ts | 22 +++++- 3 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 .changeset/cancel-reenqueue-nudge.md create mode 100644 packages/eve/src/execution/turn-cancellation-request.test.ts diff --git a/.changeset/cancel-reenqueue-nudge.md b/.changeset/cancel-reenqueue-nudge.md new file mode 100644 index 000000000..578fd734e --- /dev/null +++ b/.changeset/cancel-reenqueue-nudge.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +An accepted turn-cancel request now explicitly re-enqueues the target session's workflow run instead of relying on the workflow world to reschedule it when the cancel hook is resumed. A parked session whose wake was dropped by the scheduler previously held an accepted cancel indefinitely without emitting `turn.cancelled`. diff --git a/packages/eve/src/execution/turn-cancellation-request.test.ts b/packages/eve/src/execution/turn-cancellation-request.test.ts new file mode 100644 index 000000000..a5b29038e --- /dev/null +++ b/packages/eve/src/execution/turn-cancellation-request.test.ts @@ -0,0 +1,73 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { HookNotFoundError } from "#compiled/@workflow/errors/index.js"; +import { + isRetryableInactiveCancelReason, + requestWorkflowTurnCancellation, +} from "#execution/turn-cancellation-request.js"; +import { getWorld, reenqueueRun, resumeHook } from "#internal/workflow/runtime.js"; + +vi.mock("#internal/workflow/runtime.js", () => ({ + getWorld: vi.fn(), + reenqueueRun: vi.fn(), + resumeHook: vi.fn(), +})); + +afterEach(() => { + vi.clearAllMocks(); +}); + +describe("requestWorkflowTurnCancellation", () => { + it("resumes the session cancel hook and nudges the run scheduler", async () => { + const world = { kind: "test-world" }; + vi.mocked(resumeHook).mockResolvedValue({ runId: "session-1" } as never); + vi.mocked(getWorld).mockResolvedValue(world as never); + vi.mocked(reenqueueRun).mockResolvedValue(undefined as never); + + const result = await requestWorkflowTurnCancellation({ sessionId: "session-1" }); + + expect(result).toEqual({ status: "accepted" }); + expect(resumeHook).toHaveBeenCalledWith("session-1:cancel", {}); + expect(reenqueueRun).toHaveBeenCalledWith(world, "session-1"); + }); + + it("stays accepted when the scheduler nudge fails", async () => { + // The resume payload is durable; a failed re-enqueue only re-exposes the + // world's wake race and must not fail the cancel request. + vi.mocked(resumeHook).mockResolvedValue({ runId: "session-1" } as never); + vi.mocked(getWorld).mockRejectedValue(new Error("world unavailable")); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + + const result = await requestWorkflowTurnCancellation({ sessionId: "session-1" }); + + expect(result).toEqual({ status: "accepted" }); + expect(error).toHaveBeenCalled(); + }); + + it("classifies a missing hook as no_active_turn without nudging", async () => { + vi.mocked(resumeHook).mockRejectedValue(new HookNotFoundError("no hook")); + + const result = await requestWorkflowTurnCancellation({ sessionId: "session-1" }); + + expect(result).toEqual({ reason: "HookNotFoundError", status: "no_active_turn" }); + expect(reenqueueRun).not.toHaveBeenCalled(); + }); + + it("forwards the turn guard in the hook payload", async () => { + vi.mocked(resumeHook).mockResolvedValue({ runId: "session-1" } as never); + vi.mocked(getWorld).mockResolvedValue({} as never); + + await requestWorkflowTurnCancellation({ sessionId: "session-1", turnId: "turn_3" }); + + expect(resumeHook).toHaveBeenCalledWith("session-1:cancel", { turnId: "turn_3" }); + }); +}); + +describe("isRetryableInactiveCancelReason", () => { + it("retries only hook-claim contention", () => { + expect(isRetryableInactiveCancelReason("EntityConflictError")).toBe(true); + expect(isRetryableInactiveCancelReason("HookNotFoundError")).toBe(false); + expect(isRetryableInactiveCancelReason("WorkflowRunNotFoundError")).toBe(false); + expect(isRetryableInactiveCancelReason(undefined)).toBe(false); + }); +}); diff --git a/packages/eve/src/execution/turn-cancellation-request.ts b/packages/eve/src/execution/turn-cancellation-request.ts index 2f5e1d9e6..2567e1b31 100644 --- a/packages/eve/src/execution/turn-cancellation-request.ts +++ b/packages/eve/src/execution/turn-cancellation-request.ts @@ -6,12 +6,15 @@ import { } from "#compiled/@workflow/errors/index.js"; import type { CancelTurnInput, CancelTurnResult } from "#channel/types.js"; -import { resumeHook } from "#internal/workflow/runtime.js"; +import { createLogger, logError } from "#internal/logging.js"; +import { getWorld, reenqueueRun, resumeHook } from "#internal/workflow/runtime.js"; import { sessionCancelHookToken, type TurnCancelPayload, } from "#execution/turn-cancellation-token.js"; +const log = createLogger("execution.turn-cancellation-request"); + /** Requests cancellation through a session's stable workflow hook. */ export async function requestWorkflowTurnCancellation( input: CancelTurnInput, @@ -20,7 +23,6 @@ export async function requestWorkflowTurnCancellation( try { await resumeHook(sessionCancelHookToken(input.sessionId), payload); - return { status: "accepted" }; } catch (error) { const reason = classifyInactiveCancelTarget(error); if (reason !== undefined) { @@ -28,6 +30,22 @@ export async function requestWorkflowTurnCancellation( } throw error; } + + // The world does not reliably reschedule a suspended run when one of its + // hooks is resumed: a parked parent has been observed holding an accepted + // cancel for minutes without a single execution. Nudge the scheduler + // explicitly. The resume payload above is already durable, so a failed + // nudge only re-exposes that wake race — never a lost cancel — and a + // redundant one is a harmless replay. + try { + await reenqueueRun(await getWorld(), input.sessionId); + } catch (error) { + logError(log, "failed to re-enqueue the cancelled session's run", error, { + sessionId: input.sessionId, + }); + } + + return { status: "accepted" }; } /** From a58d59828cff455a09fd58f163c4dfb4bdf12f65 Mon Sep 17 00:00:00 2001 From: Andrew Barba Date: Thu, 30 Jul 2026 08:59:51 -0400 Subject: [PATCH 6/9] fix(eve): nudge the cancel hook's owning run, not the session driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reenqueue nudge added in a23dd001 targeted input.sessionId — the session's driver run. But the cancel hook is created inside the turn workflow run (createTurnCancellationControl runs in the turn workflow body), so the run suspended racing the cancel payload is the hook's own run, which resumeHook returns. Nudging the driver woke a run with nothing to do while the turn run stayed asleep — confirmed by the next e2e failure (parent wrun_41KYSG2A1B0GHHEEMG4ECX217X, run 30542663066): child cancelled at request time in 1.4s, parent never emitted turn.cancelled through the eval timeout. Re-enqueue hook.runId, falling back to the session id only when the world returns no run id on the hook. Signed-off-by: Andrew Barba --- .../turn-cancellation-request.test.ts | 18 ++++++++++++-- .../execution/turn-cancellation-request.ts | 24 +++++++++++++------ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/packages/eve/src/execution/turn-cancellation-request.test.ts b/packages/eve/src/execution/turn-cancellation-request.test.ts index a5b29038e..80f733ba7 100644 --- a/packages/eve/src/execution/turn-cancellation-request.test.ts +++ b/packages/eve/src/execution/turn-cancellation-request.test.ts @@ -18,9 +18,11 @@ afterEach(() => { }); describe("requestWorkflowTurnCancellation", () => { - it("resumes the session cancel hook and nudges the run scheduler", async () => { + it("resumes the session cancel hook and nudges the hook's owning run", async () => { + // The cancel hook is created by the turn workflow run, not the session's + // driver run, so the scheduler nudge must target the hook's runId. const world = { kind: "test-world" }; - vi.mocked(resumeHook).mockResolvedValue({ runId: "session-1" } as never); + vi.mocked(resumeHook).mockResolvedValue({ runId: "turn-run-1" } as never); vi.mocked(getWorld).mockResolvedValue(world as never); vi.mocked(reenqueueRun).mockResolvedValue(undefined as never); @@ -28,6 +30,18 @@ describe("requestWorkflowTurnCancellation", () => { expect(result).toEqual({ status: "accepted" }); expect(resumeHook).toHaveBeenCalledWith("session-1:cancel", {}); + expect(reenqueueRun).toHaveBeenCalledWith(world, "turn-run-1"); + }); + + it("falls back to the session id when the hook carries no run id", async () => { + const world = { kind: "test-world" }; + vi.mocked(resumeHook).mockResolvedValue({} as never); + vi.mocked(getWorld).mockResolvedValue(world as never); + vi.mocked(reenqueueRun).mockResolvedValue(undefined as never); + + const result = await requestWorkflowTurnCancellation({ sessionId: "session-1" }); + + expect(result).toEqual({ status: "accepted" }); expect(reenqueueRun).toHaveBeenCalledWith(world, "session-1"); }); diff --git a/packages/eve/src/execution/turn-cancellation-request.ts b/packages/eve/src/execution/turn-cancellation-request.ts index 2567e1b31..3c85c7ef8 100644 --- a/packages/eve/src/execution/turn-cancellation-request.ts +++ b/packages/eve/src/execution/turn-cancellation-request.ts @@ -21,8 +21,9 @@ export async function requestWorkflowTurnCancellation( ): Promise { const payload: TurnCancelPayload = input.turnId === undefined ? {} : { turnId: input.turnId }; + let hookRunId: string | undefined; try { - await resumeHook(sessionCancelHookToken(input.sessionId), payload); + hookRunId = readHookRunId(await resumeHook(sessionCancelHookToken(input.sessionId), payload)); } catch (error) { const reason = classifyInactiveCancelTarget(error); if (reason !== undefined) { @@ -32,15 +33,18 @@ export async function requestWorkflowTurnCancellation( } // The world does not reliably reschedule a suspended run when one of its - // hooks is resumed: a parked parent has been observed holding an accepted + // hooks is resumed: a parked run has been observed holding an accepted // cancel for minutes without a single execution. Nudge the scheduler - // explicitly. The resume payload above is already durable, so a failed - // nudge only re-exposes that wake race — never a lost cancel — and a - // redundant one is a harmless replay. + // explicitly — targeting the run that owns the hook (the turn workflow + // run, not the session's driver run named by `input.sessionId`), since + // that is the run suspended racing the cancel payload. The resume above + // is already durable, so a failed nudge only re-exposes that wake race — + // never a lost cancel — and a redundant one is a harmless replay. try { - await reenqueueRun(await getWorld(), input.sessionId); + await reenqueueRun(await getWorld(), hookRunId ?? input.sessionId); } catch (error) { - logError(log, "failed to re-enqueue the cancelled session's run", error, { + logError(log, "failed to re-enqueue the cancel hook's owning run", error, { + hookRunId, sessionId: input.sessionId, }); } @@ -58,6 +62,12 @@ export function isRetryableInactiveCancelReason(reason: string | undefined): boo return reason === "EntityConflictError"; } +function readHookRunId(hook: unknown): string | undefined { + if (typeof hook !== "object" || hook === null || !("runId" in hook)) return undefined; + const runId = (hook as { readonly runId?: unknown }).runId; + return typeof runId === "string" && runId.length > 0 ? runId : undefined; +} + function classifyInactiveCancelTarget(error: unknown): string | undefined { if (HookNotFoundError.is(error)) return "HookNotFoundError"; if (WorkflowRunNotFoundError.is(error)) return "WorkflowRunNotFoundError"; From db8030751c0bebca5db71585b389455296ae1892 Mon Sep 17 00:00:00 2001 From: Andrew Barba Date: Thu, 30 Jul 2026 12:08:22 -0400 Subject: [PATCH 7/9] fix(eve): upgrade Workflow DevKit to beta.38 and remove wake workarounds @workflow/core 5.0.0-beta.38 contains the fix for vercel/workflow#3183 (idle observed while a committed hook delivery is parked behind its deferral), the root cause of cancelled sessions going dormant: a run suspended racing its inbox and cancel hooks could ack a wake without consuming the journaled cancel payload, holding an accepted cancel until unrelated traffic woke it. With the root cause fixed upstream, remove the two workarounds that existed only to compensate for it: - the request-time descendant fan-out (cancelTurnWithDescendantFanout): the durable settle-time cascade is the principled owner of descendant cancellation and reaches children promptly now that the parent's cancel wake is reliable - the reenqueueRun nudge after an accepted resume: resumeHook already enqueues the wake; the defect was in the replay, not scheduling requestWorkflowTurnCancellation returns to workflow-runtime, and the settle-time cascade keeps its full adoption-window retry. The turn-loop fixes (cancel wins the settle race; abort flips in the payload's own microtask) are independent of the world bug and stay. @workflow/utils beta.8 dropped getWorldImport (static world-target injection revert); eve now owns the trivial world-name-to-package mapping in internal/workflow/world-target.ts. Signed-off-by: Andrew Barba --- .changeset/cancel-reenqueue-nudge.md | 5 - .changeset/request-time-cancel-fanout.md | 5 - .changeset/workflow-beta-38.md | 5 + docs/concepts/sessions-runs-and-streaming.md | 2 +- docs/guides/remote-agents.md | 2 +- packages/eve/package.json | 10 +- .../cancel-descendant-turns-step.test.ts | 49 +-- .../execution/cancel-descendant-turns-step.ts | 70 +-- .../turn-cancellation-fanout.test.ts | 290 ------------ .../src/execution/turn-cancellation-fanout.ts | 412 ------------------ .../turn-cancellation-request.test.ts | 87 ---- .../execution/turn-cancellation-request.ts | 77 ---- .../eve/src/execution/workflow-runtime.ts | 42 +- .../application/compiled-artifacts.ts | 4 +- .../workflow/development-world-protocol.ts | 8 +- .../internal/workflow/world-target.test.ts | 15 + .../eve/src/internal/workflow/world-target.ts | 14 + pnpm-lock.yaml | 66 +-- 18 files changed, 138 insertions(+), 1025 deletions(-) delete mode 100644 .changeset/cancel-reenqueue-nudge.md delete mode 100644 .changeset/request-time-cancel-fanout.md create mode 100644 .changeset/workflow-beta-38.md delete mode 100644 packages/eve/src/execution/turn-cancellation-fanout.test.ts delete mode 100644 packages/eve/src/execution/turn-cancellation-fanout.ts delete mode 100644 packages/eve/src/execution/turn-cancellation-request.test.ts delete mode 100644 packages/eve/src/execution/turn-cancellation-request.ts create mode 100644 packages/eve/src/internal/workflow/world-target.test.ts create mode 100644 packages/eve/src/internal/workflow/world-target.ts diff --git a/.changeset/cancel-reenqueue-nudge.md b/.changeset/cancel-reenqueue-nudge.md deleted file mode 100644 index 578fd734e..000000000 --- a/.changeset/cancel-reenqueue-nudge.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"eve": patch ---- - -An accepted turn-cancel request now explicitly re-enqueues the target session's workflow run instead of relying on the workflow world to reschedule it when the cancel hook is resumed. A parked session whose wake was dropped by the scheduler previously held an accepted cancel indefinitely without emitting `turn.cancelled`. diff --git a/.changeset/request-time-cancel-fanout.md b/.changeset/request-time-cancel-fanout.md deleted file mode 100644 index d6755d2cc..000000000 --- a/.changeset/request-time-cancel-fanout.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"eve": patch ---- - -Cancelling a session now fans the cancellation out to its subagent descendants immediately at request time, instead of only when the parent's workflow run wakes and settles. A parent suspended on the very child it needs to cancel can no longer strand that child: descendants are derived from the session's durable event stream and cancelled recursively (local and remote) as soon as the cancel request is accepted, with the settle-time cascade remaining as a durable backstop. diff --git a/.changeset/workflow-beta-38.md b/.changeset/workflow-beta-38.md new file mode 100644 index 000000000..5e245bb5d --- /dev/null +++ b/.changeset/workflow-beta-38.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Upgrade the vendored Workflow DevKit to `@workflow/core@5.0.0-beta.38`. This picks up the upstream fix for runs going dormant after an accepted hook resume (vercel/workflow#3183): a session cancelled while parked on subagents could previously hold the accepted cancel indefinitely — its turn only settling as cancelled when unrelated traffic happened to wake the run. diff --git a/docs/concepts/sessions-runs-and-streaming.md b/docs/concepts/sessions-runs-and-streaming.md index 7feeff208..0859c5163 100644 --- a/docs/concepts/sessions-runs-and-streaming.md +++ b/docs/concepts/sessions-runs-and-streaming.md @@ -163,7 +163,7 @@ curl -X POST http://127.0.0.1:2000/eve/v1/session//cancel # {"ok":true,"sessionId":"","status":"accepted"} ``` -`"accepted"` means a cancellation hook accepted the request. Confirm cancellation on the stream as `turn.cancelled` followed by `session.waiting`; the session then accepts the next message normally. If the turn is waiting on active local or remote subagents, eve also requests cancellation of every adopted child, recursively. The fan-out happens twice: immediately when the cancel request is accepted — so children stop promptly even while the parent run is still suspended — and again as a durable backstop before the parent settles. Each child reports its own cancellation boundary on its child-session stream; the parent does not emit `subagent.completed` for cancelled work. `"no_active_turn"` means no resumable cancellation target exists, including an unknown session or an already-settled turn. Both statuses are success, so clients can fire and forget. See the [eve channel](../channels/eve) for the full route contract. +`"accepted"` means a cancellation hook accepted the request. Confirm cancellation on the stream as `turn.cancelled` followed by `session.waiting`; the session then accepts the next message normally. If the turn is waiting on active local or remote subagents, eve also requests cancellation of every adopted child, recursively, before settling the parent. Each child reports its own cancellation boundary on its child-session stream; the parent does not emit `subagent.completed` for cancelled work. `"no_active_turn"` means no resumable cancellation target exists, including an unknown session or an already-settled turn. Both statuses are success, so clients can fire and forget. See the [eve channel](../channels/eve) for the full route contract. Custom channel routes request the same cancellation without knowing the session id: the `cancel` route helper is addressed by the channel-local continuation token, and `Session.cancel()` by session id. See [custom channels](../channels/custom#cancel-a-turn). diff --git a/docs/guides/remote-agents.md b/docs/guides/remote-agents.md index faab018df..8f876c7cc 100644 --- a/docs/guides/remote-agents.md +++ b/docs/guides/remote-agents.md @@ -93,7 +93,7 @@ A local subagent runs inline. A remote one runs in its own deployment, so dispat The parent stream carries the same `subagent.called`, `action.result`, and `subagent.completed` events as local delegation. For a remote call, `subagent.called.data.remote.url` records the target. -Cancelling the parent while a remote call is active sends an authenticated `POST /eve/v1/session/:childSessionId/cancel` to the remote — once immediately when the parent's cancel request is accepted, and again as a durable backstop before the parent settles. eve resolves the remote's `headers` and `auth` again for every cancellation attempt, so rotating credentials work the same way as they do for session creation. Cancellation always uses the standard eve cancel path on `url`, even when `path` customizes only the create-session endpoint. The remote child reports `turn.cancelled` → `session.waiting` on its own stream; an older or unreachable remote is logged but cannot turn the parent's cancellation into a failure. +Cancelling the parent while a remote call is active sends an authenticated `POST /eve/v1/session/:childSessionId/cancel` to the remote and waits for that request to be accepted before the parent settles. eve resolves the remote's `headers` and `auth` again for every cancellation attempt, so rotating credentials work the same way as they do for session creation. Cancellation always uses the standard eve cancel path on `url`, even when `path` customizes only the create-session endpoint. The remote child reports `turn.cancelled` → `session.waiting` on its own stream; an older or unreachable remote is logged but cannot turn the parent's cancellation into a failure. Both failure paths surface to the parent as a failed tool result, so the caller can explain or recover within the same session. A failed _start_ returns the error inline. A remote that starts and then fails posts a terminal failure callback, which the parent receives as an errored subagent result carrying the remote's error (or `REMOTE_AGENT_FAILED` when none is supplied). Terminal callback delivery runs as a durable step on the underlying workflow engine (see [Execution model & durability](../concepts/execution-model-and-durability)). A failed callback POST is rethrown rather than marking the task complete, so the engine retries it. diff --git a/packages/eve/package.json b/packages/eve/package.json index 4f0b64d73..34d62f8e0 100644 --- a/packages/eve/package.json +++ b/packages/eve/package.json @@ -328,13 +328,13 @@ "@vercel/otel": "catalog:", "@vercel/sandbox": "catalog:", "@vercel/sdk": "1.28.8", - "@workflow/core": "5.0.0-beta.37", - "@workflow/errors": "5.0.0-beta.13", + "@workflow/core": "5.0.0-beta.38", + "@workflow/errors": "5.0.0-beta.14", "@workflow/serde": "5.0.0-beta.2", - "@workflow/utils": "5.0.0-beta.7", + "@workflow/utils": "5.0.0-beta.8", "@workflow/world": "5.0.0-beta.23", - "@workflow/world-local": "5.0.0-beta.31", - "@workflow/world-vercel": "5.0.0-beta.33", + "@workflow/world-local": "5.0.0-beta.32", + "@workflow/world-vercel": "5.0.0-beta.34", "ai": "catalog:", "autoevals": "0.0.132", "chat": "4.34.0", diff --git a/packages/eve/src/execution/cancel-descendant-turns-step.test.ts b/packages/eve/src/execution/cancel-descendant-turns-step.test.ts index cbb86e680..9dbe9ac62 100644 --- a/packages/eve/src/execution/cancel-descendant-turns-step.test.ts +++ b/packages/eve/src/execution/cancel-descendant-turns-step.test.ts @@ -8,7 +8,7 @@ import { isRetryableRemoteAgentCancelError, resolveRemoteAgentForAction, } from "#execution/remote-agent-dispatch.js"; -import { requestWorkflowTurnCancellation } from "#execution/turn-cancellation-request.js"; +import { requestWorkflowTurnCancellation } from "#execution/workflow-runtime.js"; import { recordPendingSubagentChild, setPendingRuntimeActionBatch, @@ -20,8 +20,7 @@ vi.mock("#context/serialize.js", () => ({ deserializeContext: vi.fn(), })); -vi.mock("./turn-cancellation-request.js", () => ({ - isRetryableInactiveCancelReason: (reason: string | undefined) => reason === "EntityConflictError", +vi.mock("./workflow-runtime.js", () => ({ requestWorkflowTurnCancellation: vi.fn(), })); @@ -104,14 +103,16 @@ describe("cancelDescendantTurnsStep", () => { expect(deserializeContext).not.toHaveBeenCalled(); }); - it("retries hook-claim conflicts during the child adoption window", async () => { + it("retries no-active-turn responses during the child adoption window", async () => { vi.useFakeTimers(); installRemoteRegistry(); vi.mocked(resolveRemoteAgentForAction).mockReturnValue(remote); vi.mocked(requestWorkflowTurnCancellation) - .mockResolvedValueOnce({ reason: "EntityConflictError", status: "no_active_turn" }) + .mockResolvedValueOnce({ status: "no_active_turn" }) + .mockResolvedValueOnce({ status: "accepted" }); + vi.mocked(cancelRemoteAgentTurn) + .mockResolvedValueOnce({ status: "no_active_turn" }) .mockResolvedValueOnce({ status: "accepted" }); - vi.mocked(cancelRemoteAgentTurn).mockResolvedValue({ status: "accepted" }); const cancellation = cancelDescendantTurnsStep({ serializedContext: {}, @@ -121,41 +122,7 @@ describe("cancelDescendantTurnsStep", () => { await cancellation; expect(requestWorkflowTurnCancellation).toHaveBeenCalledTimes(2); - }); - - it("treats a terminal no-active-turn child as settled after a single attempt", async () => { - // The request-time fan-out usually cancels descendants before this - // durable backstop runs; a missing hook must not burn the retry budget. - vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ - reason: "HookNotFoundError", - status: "no_active_turn", - }); - const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); - - await cancelDescendantTurnsStep({ - serializedContext: {}, - sessionState: createPendingState({ includeRemote: false }), - }); - - expect(requestWorkflowTurnCancellation).toHaveBeenCalledOnce(); - expect(warn).not.toHaveBeenCalledWith( - expect.stringContaining("descendant cancel was never accepted"), - expect.anything(), - ); - }); - - it("does not retry a remote no-active-turn result", async () => { - installRemoteRegistry(); - vi.mocked(resolveRemoteAgentForAction).mockReturnValue(remote); - vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); - vi.mocked(cancelRemoteAgentTurn).mockResolvedValue({ status: "no_active_turn" }); - - await cancelDescendantTurnsStep({ - serializedContext: {}, - sessionState: createPendingState(), - }); - - expect(cancelRemoteAgentTurn).toHaveBeenCalledOnce(); + expect(cancelRemoteAgentTurn).toHaveBeenCalledTimes(2); }); it("contains unexpected local failures and exhausted remote failures", async () => { diff --git a/packages/eve/src/execution/cancel-descendant-turns-step.ts b/packages/eve/src/execution/cancel-descendant-turns-step.ts index 37d6d43c5..b7d6a7265 100644 --- a/packages/eve/src/execution/cancel-descendant-turns-step.ts +++ b/packages/eve/src/execution/cancel-descendant-turns-step.ts @@ -7,10 +7,7 @@ import { isRetryableRemoteAgentCancelError, resolveRemoteAgentForAction, } from "#execution/remote-agent-dispatch.js"; -import { - isRetryableInactiveCancelReason, - requestWorkflowTurnCancellation, -} from "#execution/turn-cancellation-request.js"; +import { requestWorkflowTurnCancellation } from "#execution/workflow-runtime.js"; import { getPendingRuntimeActionBatch } from "#harness/runtime-actions.js"; import { createLogger, logError } from "#internal/logging.js"; import type { @@ -19,13 +16,11 @@ import type { } from "#runtime/actions/types.js"; import type { RuntimeSubagentRegistry } from "#runtime/subagents/registry.js"; -// This step is the durable backstop behind the request-time fan-out -// (`cancelTurnWithDescendantFanout`), which usually reaches descendants -// before the parent's run ever wakes. By the time this step executes, a -// child with no active turn was most often already cancelled at request -// time — a terminal outcome, returned immediately. Only reasons that can -// heal with time (hook-claim conflicts during queue wakes) are retried, -// and an exhausted retry is logged loudly rather than dropped silently. +// Descendants listed in a pending batch should be actively running, so a +// cancel that cannot be delivered is anomalous: retry with backoff long +// enough to ride out transient world contention (queue wakes, hook-claim +// conflicts), then log loudly rather than dropping the cancel silently — +// an uncancelled child otherwise runs to completion with no trace of why. const CANCEL_ATTEMPTS = 8; const CANCEL_RETRY_INITIAL_DELAY_MS = 250; const CANCEL_RETRY_MAX_DELAY_MS = 1_500; @@ -100,25 +95,15 @@ async function cancelLocalDescendant(input: { const final = await requestCancellationWithRetry({ request: () => requestWorkflowTurnCancellation({ sessionId: input.childSessionId }), shouldRetryError: () => false, - shouldRetryResult: (result) => isRetryableInactiveCancelReason(result.reason), }); if (final.status !== "accepted") { - if (isRetryableInactiveCancelReason(final.reason)) { - log.warn("descendant cancel was never accepted; the child may run to completion", { - callId: input.action.callId, - childSessionId: input.childSessionId, - finalStatus: final.status, - reason: final.reason, - subagentName: input.action.subagentName, - }); - } else { - log.info("descendant had no active turn at settle time; already cancelled or finished", { - callId: input.action.callId, - childSessionId: input.childSessionId, - reason: final.reason, - subagentName: input.action.subagentName, - }); - } + log.warn("descendant cancel was never accepted; the child may run to completion", { + callId: input.action.callId, + childSessionId: input.childSessionId, + finalStatus: final.status, + reason: final.reason, + subagentName: input.action.subagentName, + }); } } catch (error) { logError(log, "failed to cancel local descendant turn", error, { @@ -145,21 +130,15 @@ async function cancelRemoteDescendant(input: { const final = await requestCancellationWithRetry({ request: () => cancelRemoteAgentTurn({ remote, sessionId: input.childSessionId }), shouldRetryError: isRetryableRemoteAgentCancelError, - // The remote deployment already classified its own turn state; a - // remote no_active_turn is terminal. - shouldRetryResult: () => false, }); if (final.status !== "accepted") { - log.info( - "remote descendant had no active turn at settle time; already cancelled or finished", - { - callId: input.action.callId, - childSessionId: input.childSessionId, - finalStatus: final.status, - reason: final.reason, - remoteAgentName: input.action.remoteAgentName, - }, - ); + log.warn("remote descendant cancel was never accepted; the child may run to completion", { + callId: input.action.callId, + childSessionId: input.childSessionId, + finalStatus: final.status, + reason: final.reason, + remoteAgentName: input.action.remoteAgentName, + }); } } catch (error) { logError(log, "failed to cancel remote descendant turn", error, { @@ -173,7 +152,6 @@ async function cancelRemoteDescendant(input: { async function requestCancellationWithRetry(input: { readonly request: () => Promise; readonly shouldRetryError: (error: unknown) => boolean; - readonly shouldRetryResult: (result: CancelTurnResult) => boolean; }): Promise { let delayMs = CANCEL_RETRY_INITIAL_DELAY_MS; let lastResult: CancelTurnResult = { status: "no_active_turn" }; @@ -181,13 +159,7 @@ async function requestCancellationWithRetry(input: { for (let attempt = 1; attempt <= CANCEL_ATTEMPTS; attempt += 1) { try { lastResult = await input.request(); - if ( - lastResult.status === "accepted" || - !input.shouldRetryResult(lastResult) || - attempt === CANCEL_ATTEMPTS - ) { - return lastResult; - } + if (lastResult.status === "accepted" || attempt === CANCEL_ATTEMPTS) return lastResult; } catch (error) { if (!input.shouldRetryError(error) || attempt === CANCEL_ATTEMPTS) throw error; } diff --git a/packages/eve/src/execution/turn-cancellation-fanout.test.ts b/packages/eve/src/execution/turn-cancellation-fanout.test.ts deleted file mode 100644 index 3157cddcf..000000000 --- a/packages/eve/src/execution/turn-cancellation-fanout.test.ts +++ /dev/null @@ -1,290 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { cancelRemoteAgentTurn } from "#execution/remote-agent-dispatch.js"; -import { - cancelTurnWithDescendantFanout, - collectPendingDescendants, -} from "#execution/turn-cancellation-fanout.js"; -import { requestWorkflowTurnCancellation } from "#execution/turn-cancellation-request.js"; -import { getRun } from "#internal/workflow/runtime.js"; - -vi.mock("#internal/workflow/runtime.js", () => ({ - getRun: vi.fn(), -})); - -vi.mock("./turn-cancellation-request.js", () => ({ - requestWorkflowTurnCancellation: vi.fn(), -})); - -vi.mock("./remote-agent-dispatch.js", () => ({ - cancelRemoteAgentTurn: vi.fn(), - isRetryableRemoteAgentCancelError: vi.fn(() => false), -})); - -afterEach(() => { - vi.clearAllMocks(); -}); - -function subagentCalled(input: { - readonly callId: string; - readonly childSessionId: string; - readonly remote?: { readonly url: string }; - readonly turnId?: string; -}) { - return { - data: { - callId: input.callId, - childSessionId: input.childSessionId, - name: "child", - remote: input.remote, - sequence: 0, - sessionId: "parent", - toolName: "child", - turnId: input.turnId ?? "turn_0", - workflowId: "workflow//eve//workflowEntry", - }, - type: "subagent.called", - }; -} - -function actionResult(callId: string) { - return { - data: { - result: { callId, kind: "subagent-result", output: "done", subagentName: "child" }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn_0", - }, - type: "action.result", - }; -} - -/** - * Mimics a live run stream: journaled chunks are followed by an open tail - * that never closes, exactly like a parked durable session's event stream. - */ -function installEventStreams(streamsBySessionId: Record): void { - vi.mocked(getRun).mockImplementation( - (sessionId: string) => - ({ - getReadable: () => { - const events = streamsBySessionId[sessionId] ?? []; - const chunks = events.map((event) => - new TextEncoder().encode(`${JSON.stringify(event)}\n`), - ); - let index = 0; - const stream = new ReadableStream({ - pull(controller) { - const chunk = chunks[index]; - if (chunk !== undefined) { - controller.enqueue(chunk); - index += 1; - } - // Past the tail the stream stays open, like a live run. - }, - }) as ReadableStream & { getTailIndex: () => Promise }; - stream.getTailIndex = async () => chunks.length - 1; - return stream; - }, - }) as never, - ); -} - -describe("collectPendingDescendants", () => { - it("returns dispatched children without results and skips resolved ones", () => { - const records = collectPendingDescendants({ - events: [ - subagentCalled({ callId: "call-1", childSessionId: "child-1" }), - subagentCalled({ callId: "call-2", childSessionId: "child-2" }), - actionResult("call-1"), - ], - }); - - expect(records).toEqual([ - expect.objectContaining({ callId: "call-2", childSessionId: "child-2" }), - ]); - }); - - it("filters by turn id when provided", () => { - const records = collectPendingDescendants({ - events: [ - subagentCalled({ callId: "call-1", childSessionId: "child-1", turnId: "turn_0" }), - subagentCalled({ callId: "call-2", childSessionId: "child-2", turnId: "turn_1" }), - ], - turnId: "turn_1", - }); - - expect(records).toEqual([expect.objectContaining({ childSessionId: "child-2" })]); - }); - - it("captures remote identity and ignores malformed events", () => { - const records = collectPendingDescendants({ - events: [ - "not-an-event", - { type: "subagent.called", data: { callId: 42 } }, - subagentCalled({ - callId: "call-r", - childSessionId: "remote-child", - remote: { url: "https://remote.example.com" }, - }), - ], - }); - - expect(records).toEqual([ - expect.objectContaining({ - childSessionId: "remote-child", - remote: { url: "https://remote.example.com" }, - }), - ]); - }); -}); - -describe("cancelTurnWithDescendantFanout", () => { - it("cancels the target and every unresolved local descendant at request time", async () => { - installEventStreams({ - parent: [subagentCalled({ callId: "call-1", childSessionId: "child-1" })], - }); - vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); - - const result = await cancelTurnWithDescendantFanout({ sessionId: "parent" }); - - expect(result).toEqual({ status: "accepted" }); - expect(requestWorkflowTurnCancellation).toHaveBeenCalledWith({ sessionId: "parent" }); - expect(requestWorkflowTurnCancellation).toHaveBeenCalledWith({ sessionId: "child-1" }); - }); - - it("recurses into local descendants to cancel grandchildren", async () => { - installEventStreams({ - "child-1": [subagentCalled({ callId: "call-2", childSessionId: "grandchild-1" })], - grandchild: [], - parent: [subagentCalled({ callId: "call-1", childSessionId: "child-1" })], - }); - vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); - - await cancelTurnWithDescendantFanout({ sessionId: "parent" }); - - expect(requestWorkflowTurnCancellation).toHaveBeenCalledWith({ sessionId: "grandchild-1" }); - }); - - it("skips already-visited sessions so cycles cannot recurse forever", async () => { - installEventStreams({ - "child-1": [subagentCalled({ callId: "call-back", childSessionId: "parent" })], - parent: [subagentCalled({ callId: "call-1", childSessionId: "child-1" })], - }); - vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); - - await cancelTurnWithDescendantFanout({ sessionId: "parent" }); - - const cancelledSessions = vi - .mocked(requestWorkflowTurnCancellation) - .mock.calls.map(([input]) => input.sessionId); - expect(cancelledSessions).toEqual(["parent", "child-1"]); - }); - - it("cancels remote descendants through the url-matched registry node", async () => { - installEventStreams({ - parent: [ - subagentCalled({ - callId: "call-r", - childSessionId: "remote-child", - remote: { url: "https://remote.example.com" }, - }), - ], - }); - const remoteNode = { kind: "remote", name: "child", url: "https://remote.example.com" }; - const registry = new Map([["subagents/remote", { definition: remoteNode }]]); - vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); - vi.mocked(cancelRemoteAgentTurn).mockResolvedValue({ status: "accepted" }); - - await cancelTurnWithDescendantFanout({ - resolveRemoteRegistry: async () => registry as never, - sessionId: "parent", - }); - - expect(cancelRemoteAgentTurn).toHaveBeenCalledWith({ - remote: remoteNode, - sessionId: "remote-child", - }); - // Remote descendants are cancelled by their own deployment; no local recursion. - expect(requestWorkflowTurnCancellation).not.toHaveBeenCalledWith({ - sessionId: "remote-child", - }); - }); - - it("defers a remote descendant whose url matches no registered agent", async () => { - installEventStreams({ - parent: [ - subagentCalled({ - callId: "call-r", - childSessionId: "remote-child", - remote: { url: "https://unknown.example.com" }, - }), - ], - }); - vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); - const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); - - await cancelTurnWithDescendantFanout({ - resolveRemoteRegistry: async () => new Map() as never, - sessionId: "parent", - }); - - expect(cancelRemoteAgentTurn).not.toHaveBeenCalled(); - expect(warn).toHaveBeenCalledWith( - expect.stringContaining("matched no registered remote agent"), - expect.anything(), - ); - }); - - it("retries a local child caught in its dispatch gap", async () => { - vi.useFakeTimers(); - try { - installEventStreams({ - parent: [subagentCalled({ callId: "call-1", childSessionId: "child-1" })], - }); - vi.mocked(requestWorkflowTurnCancellation) - .mockResolvedValueOnce({ status: "accepted" }) // parent - .mockResolvedValueOnce({ reason: "HookNotFoundError", status: "no_active_turn" }) - .mockResolvedValueOnce({ status: "accepted" }); - - const fanout = cancelTurnWithDescendantFanout({ sessionId: "parent" }); - await vi.advanceTimersByTimeAsync(1_000); - await fanout; - - const childCancels = vi - .mocked(requestWorkflowTurnCancellation) - .mock.calls.filter(([input]) => input.sessionId === "child-1"); - expect(childCancels).toHaveLength(2); - } finally { - vi.useRealTimers(); - } - }); - - it("returns the target result even when fan-out fails", async () => { - vi.mocked(getRun).mockImplementation(() => { - throw new Error("world unavailable"); - }); - vi.mocked(requestWorkflowTurnCancellation).mockResolvedValue({ status: "accepted" }); - const error = vi.spyOn(console, "error").mockImplementation(() => {}); - - const result = await cancelTurnWithDescendantFanout({ sessionId: "parent" }); - - expect(result).toEqual({ status: "accepted" }); - expect(error).toHaveBeenCalled(); - }); - - it("still fans out when the target itself has no active turn", async () => { - installEventStreams({ - parent: [subagentCalled({ callId: "call-1", childSessionId: "orphan-child" })], - }); - vi.mocked(requestWorkflowTurnCancellation) - .mockResolvedValueOnce({ reason: "HookNotFoundError", status: "no_active_turn" }) - .mockResolvedValue({ status: "accepted" }); - - const result = await cancelTurnWithDescendantFanout({ sessionId: "parent" }); - - expect(result).toEqual({ reason: "HookNotFoundError", status: "no_active_turn" }); - expect(requestWorkflowTurnCancellation).toHaveBeenCalledWith({ sessionId: "orphan-child" }); - }); -}); diff --git a/packages/eve/src/execution/turn-cancellation-fanout.ts b/packages/eve/src/execution/turn-cancellation-fanout.ts deleted file mode 100644 index 2bc0ee4ec..000000000 --- a/packages/eve/src/execution/turn-cancellation-fanout.ts +++ /dev/null @@ -1,412 +0,0 @@ -/** - * Request-time descendant cancellation. - * - * The durable settle-time cascade - * ({@link import("#execution/cancel-descendant-turns-step.js").cancelDescendantTurnsStep}) - * runs inside the parent's workflow, which cannot execute while the parent - * run is suspended — including while it is parked awaiting the very child it - * needs to cancel. A cancel request therefore fans out to descendants - * immediately at the API boundary, before any run has to wake. - * - * The session's own event stream is the durable record of dispatched - * children: `subagent.called` carries `{ callId, childSessionId, - * remote?.url, turnId }` and `action.result` marks the call resolved. Both - * are readable by session id from any process. Cancelling an - * already-finished child resolves to `no_active_turn` and is harmless, so - * the fold errs toward over-cancelling. - */ - -import type { CancelTurnResult } from "#channel/types.js"; -import { createLogger, logError } from "#internal/logging.js"; -import { getRun } from "#internal/workflow/runtime.js"; -import { - cancelRemoteAgentTurn, - isRetryableRemoteAgentCancelError, -} from "#execution/remote-agent-dispatch.js"; -import { requestWorkflowTurnCancellation } from "#execution/turn-cancellation-request.js"; -import type { RuntimeSubagentRegistry } from "#runtime/subagents/registry.js"; -import type { ResolvedRuntimeRemoteAgentNode } from "#runtime/types.js"; - -const log = createLogger("execution.turn-cancellation-fanout"); - -// Fan-out runs inline in the cancel request handler, so every bound is -// small: a short per-child retry covers the dispatch gap where a child run -// exists but its cancel hook is not yet armed, and the stream read deadline -// keeps a wedged world from stalling the response. The durable settle-time -// cascade remains the backstop for anything missed here. -const FANOUT_ATTEMPTS = 3; -const FANOUT_RETRY_DELAY_MS = 250; -const MAX_DESCENDANT_DEPTH = 8; -const STREAM_READ_DEADLINE_MS = 5_000; - -type RemoteRegistry = RuntimeSubagentRegistry["subagentsByNodeId"]; - -/** One dispatched-but-unresolved child derived from a session's events. */ -export interface PendingDescendantRecord { - readonly callId: string; - readonly childSessionId: string; - readonly remote?: { readonly url: string }; - readonly toolName: string; - readonly turnId: string; -} - -/** - * Cancels a session's active turn and immediately fans the cancellation out - * to every dispatched-but-unresolved descendant, without waiting for the - * target's workflow run to wake. Fan-out failures are logged, never thrown: - * the returned result reflects the target session alone. - */ -export async function cancelTurnWithDescendantFanout(input: { - readonly sessionId: string; - readonly turnId?: string; - readonly resolveRemoteRegistry?: () => Promise; -}): Promise { - const cancelInput: { sessionId: string; turnId?: string } = { sessionId: input.sessionId }; - if (input.turnId !== undefined) cancelInput.turnId = input.turnId; - const result = await requestWorkflowTurnCancellation(cancelInput); - - // Fan out even when the target reports no active turn: descendants that - // outlived an earlier cancel (or a crashed parent) are exactly the ones - // only a fresh request can still reach. - try { - await cancelDescendantSessions({ - depth: 0, - resolveRemoteRegistry: input.resolveRemoteRegistry, - sessionId: input.sessionId, - turnId: input.turnId, - visited: new Set([input.sessionId]), - }); - } catch (error) { - logError(log, "descendant cancel fan-out failed", error, { sessionId: input.sessionId }); - } - - return result; -} - -/** - * Folds a session's events into its dispatched-but-unresolved children. - * When `turnId` is set, only children dispatched by that turn are returned; - * malformed entries are skipped. - */ -export function collectPendingDescendants(input: { - readonly events: Iterable; - readonly turnId?: string; -}): PendingDescendantRecord[] { - const pending = new Map(); - - for (const event of input.events) { - if (typeof event !== "object" || event === null) continue; - const candidate = event as { readonly data?: unknown; readonly type?: unknown }; - - if (candidate.type === "subagent.called") { - const record = parseSubagentCalledData(candidate.data); - if (record !== undefined) pending.set(record.callId, record); - continue; - } - - if (candidate.type === "action.result") { - const callId = parseActionResultCallId(candidate.data); - if (callId !== undefined) pending.delete(callId); - } - } - - const records = [...pending.values()]; - return input.turnId === undefined - ? records - : records.filter((record) => record.turnId === input.turnId); -} - -async function cancelDescendantSessions(input: { - readonly depth: number; - readonly resolveRemoteRegistry?: () => Promise; - readonly sessionId: string; - readonly turnId?: string; - readonly visited: Set; -}): Promise { - if (input.depth >= MAX_DESCENDANT_DEPTH) { - log.warn("descendant cancel fan-out hit the depth cap", { - depth: input.depth, - sessionId: input.sessionId, - }); - return; - } - - let records: PendingDescendantRecord[]; - try { - records = collectPendingDescendants({ - events: await readSessionEventsUpToTail(input.sessionId), - turnId: input.turnId, - }); - } catch (error) { - logError(log, "failed to read session events during cancel fan-out", error, { - sessionId: input.sessionId, - }); - return; - } - - await Promise.all( - records.map(async (record) => { - if (input.visited.has(record.childSessionId)) return; - input.visited.add(record.childSessionId); - - if (record.remote !== undefined) { - await cancelRemoteDescendantAtRequest({ - record, - resolveRemoteRegistry: input.resolveRemoteRegistry, - }); - return; - } - - await cancelLocalDescendantAtRequest(record); - // Grandchildren are cancelled without a turn filter: whatever is still - // unresolved under a cancelled child must not keep running. - await cancelDescendantSessions({ - depth: input.depth + 1, - resolveRemoteRegistry: input.resolveRemoteRegistry, - sessionId: record.childSessionId, - visited: input.visited, - }); - }), - ); -} - -async function cancelLocalDescendantAtRequest(record: PendingDescendantRecord): Promise { - try { - const final = await requestCancellationWithRetry({ - request: () => requestWorkflowTurnCancellation({ sessionId: record.childSessionId }), - shouldRetryError: () => false, - shouldRetryResult: (result) => - // A child in its dispatch gap has no cancel hook yet, which is - // indistinguishable from an already-finished child — retrying briefly - // covers the former and only costs no-ops on the latter. - result.status === "no_active_turn", - }); - if (final.status !== "accepted") { - log.info("descendant had no active turn during request-time cancel fan-out", { - callId: record.callId, - childSessionId: record.childSessionId, - reason: final.reason, - toolName: record.toolName, - }); - } - } catch (error) { - logError(log, "failed to cancel local descendant at request time", error, { - callId: record.callId, - childSessionId: record.childSessionId, - toolName: record.toolName, - }); - } -} - -async function cancelRemoteDescendantAtRequest(input: { - readonly record: PendingDescendantRecord; - readonly resolveRemoteRegistry?: () => Promise; -}): Promise { - const { record } = input; - if (record.remote === undefined || input.resolveRemoteRegistry === undefined) { - log.warn("remote descendant cannot be cancelled at request time; deferring to settle-time", { - callId: record.callId, - childSessionId: record.childSessionId, - hasRegistry: input.resolveRemoteRegistry !== undefined, - toolName: record.toolName, - }); - return; - } - - try { - const remote = findRemoteAgentByUrl({ - registry: await input.resolveRemoteRegistry(), - url: record.remote.url, - }); - if (remote === undefined) { - log.warn("remote descendant url matched no registered remote agent; deferring", { - callId: record.callId, - childSessionId: record.childSessionId, - toolName: record.toolName, - url: record.remote.url, - }); - return; - } - const final = await requestCancellationWithRetry({ - request: () => cancelRemoteAgentTurn({ remote, sessionId: record.childSessionId }), - shouldRetryError: isRetryableRemoteAgentCancelError, - // The remote deployment already classified its own turn state; a - // remote no_active_turn is terminal. - shouldRetryResult: () => false, - }); - if (final.status !== "accepted") { - log.info("remote descendant had no active turn during request-time cancel fan-out", { - callId: record.callId, - childSessionId: record.childSessionId, - toolName: record.toolName, - }); - } - } catch (error) { - logError(log, "failed to cancel remote descendant at request time", error, { - callId: record.callId, - childSessionId: record.childSessionId, - toolName: record.toolName, - }); - } -} - -async function requestCancellationWithRetry(input: { - readonly request: () => Promise; - readonly shouldRetryError: (error: unknown) => boolean; - readonly shouldRetryResult: (result: CancelTurnResult) => boolean; -}): Promise { - let lastResult: CancelTurnResult = { status: "no_active_turn" }; - - for (let attempt = 1; attempt <= FANOUT_ATTEMPTS; attempt += 1) { - try { - lastResult = await input.request(); - if (lastResult.status === "accepted" || !input.shouldRetryResult(lastResult)) { - return lastResult; - } - } catch (error) { - if (!input.shouldRetryError(error) || attempt === FANOUT_ATTEMPTS) throw error; - } - if (attempt < FANOUT_ATTEMPTS) { - await new Promise((resolve) => setTimeout(resolve, FANOUT_RETRY_DELAY_MS * attempt)); - } - } - - return lastResult; -} - -/** - * Reads every event currently journaled on the session's stream and stops at - * the tail observed on entry, so a live (never-closing) run stream cannot - * block the cancel request. Chunks map 1:1 to stream writes; each carries - * one or more NDJSON lines. - */ -async function readSessionEventsUpToTail(sessionId: string): Promise { - const stream = getRun(sessionId).getReadable(); - const tailIndex = await stream.getTailIndex(); - if (tailIndex < 0) { - await stream.cancel("eve cancel fan-out: session has no events").catch(() => {}); - return []; - } - - const reader = stream.getReader(); - const decoder = new TextDecoder(); - const events: unknown[] = []; - let buffer = ""; - let chunksRead = 0; - let deadlineHit = false; - const deadline = setTimeout(() => { - deadlineHit = true; - void reader.cancel("eve cancel fan-out: stream read deadline reached").catch(() => {}); - }, STREAM_READ_DEADLINE_MS); - - const drainBuffer = () => { - for ( - let newlineIndex = buffer.indexOf("\n"); - newlineIndex !== -1; - newlineIndex = buffer.indexOf("\n") - ) { - const line = buffer.slice(0, newlineIndex).trim(); - buffer = buffer.slice(newlineIndex + 1); - if (line.length === 0) continue; - try { - events.push(JSON.parse(line)); - } catch { - // A single undecodable line must not abort the whole fan-out. - } - } - }; - - try { - while (chunksRead <= tailIndex) { - const { done, value } = await reader.read(); - if (done) break; - chunksRead += 1; - buffer += decoder.decode(value, { stream: true }); - drainBuffer(); - } - buffer += decoder.decode(); - buffer += "\n"; - drainBuffer(); - } finally { - clearTimeout(deadline); - await reader.cancel("eve cancel fan-out: read complete").catch(() => {}); - reader.releaseLock(); - } - - if (deadlineHit) { - log.warn("session event read hit the fan-out deadline; descendants may be incomplete", { - chunksRead, - sessionId, - tailIndex, - }); - } - - return events; -} - -function parseSubagentCalledData(data: unknown): PendingDescendantRecord | undefined { - if (typeof data !== "object" || data === null) return undefined; - const value = data as { - readonly callId?: unknown; - readonly childSessionId?: unknown; - readonly remote?: unknown; - readonly toolName?: unknown; - readonly turnId?: unknown; - }; - if ( - typeof value.callId !== "string" || - typeof value.childSessionId !== "string" || - typeof value.toolName !== "string" || - typeof value.turnId !== "string" - ) { - return undefined; - } - - const record: { - callId: string; - childSessionId: string; - remote?: { url: string }; - toolName: string; - turnId: string; - } = { - callId: value.callId, - childSessionId: value.childSessionId, - toolName: value.toolName, - turnId: value.turnId, - }; - - if ( - typeof value.remote === "object" && - value.remote !== null && - typeof (value.remote as { readonly url?: unknown }).url === "string" - ) { - record.remote = { url: (value.remote as { readonly url: string }).url }; - } - - return record; -} - -/** - * The `subagent.called` event records a remote child's dispatch URL but not - * its registry node, so the cancel request re-resolves the node (and with it - * the remote's auth and headers) by URL. Distinct nodes sharing a URL target - * the same deployment, so any match cancels the same session. - */ -function findRemoteAgentByUrl(input: { - readonly registry: RemoteRegistry; - readonly url: string; -}): ResolvedRuntimeRemoteAgentNode | undefined { - for (const registered of input.registry.values()) { - const definition = registered.definition; - if (definition.kind === "remote" && definition.url === input.url) return definition; - } - return undefined; -} - -function parseActionResultCallId(data: unknown): string | undefined { - if (typeof data !== "object" || data === null) return undefined; - const result = (data as { readonly result?: unknown }).result; - if (typeof result !== "object" || result === null) return undefined; - const callId = (result as { readonly callId?: unknown }).callId; - return typeof callId === "string" ? callId : undefined; -} diff --git a/packages/eve/src/execution/turn-cancellation-request.test.ts b/packages/eve/src/execution/turn-cancellation-request.test.ts deleted file mode 100644 index 80f733ba7..000000000 --- a/packages/eve/src/execution/turn-cancellation-request.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { HookNotFoundError } from "#compiled/@workflow/errors/index.js"; -import { - isRetryableInactiveCancelReason, - requestWorkflowTurnCancellation, -} from "#execution/turn-cancellation-request.js"; -import { getWorld, reenqueueRun, resumeHook } from "#internal/workflow/runtime.js"; - -vi.mock("#internal/workflow/runtime.js", () => ({ - getWorld: vi.fn(), - reenqueueRun: vi.fn(), - resumeHook: vi.fn(), -})); - -afterEach(() => { - vi.clearAllMocks(); -}); - -describe("requestWorkflowTurnCancellation", () => { - it("resumes the session cancel hook and nudges the hook's owning run", async () => { - // The cancel hook is created by the turn workflow run, not the session's - // driver run, so the scheduler nudge must target the hook's runId. - const world = { kind: "test-world" }; - vi.mocked(resumeHook).mockResolvedValue({ runId: "turn-run-1" } as never); - vi.mocked(getWorld).mockResolvedValue(world as never); - vi.mocked(reenqueueRun).mockResolvedValue(undefined as never); - - const result = await requestWorkflowTurnCancellation({ sessionId: "session-1" }); - - expect(result).toEqual({ status: "accepted" }); - expect(resumeHook).toHaveBeenCalledWith("session-1:cancel", {}); - expect(reenqueueRun).toHaveBeenCalledWith(world, "turn-run-1"); - }); - - it("falls back to the session id when the hook carries no run id", async () => { - const world = { kind: "test-world" }; - vi.mocked(resumeHook).mockResolvedValue({} as never); - vi.mocked(getWorld).mockResolvedValue(world as never); - vi.mocked(reenqueueRun).mockResolvedValue(undefined as never); - - const result = await requestWorkflowTurnCancellation({ sessionId: "session-1" }); - - expect(result).toEqual({ status: "accepted" }); - expect(reenqueueRun).toHaveBeenCalledWith(world, "session-1"); - }); - - it("stays accepted when the scheduler nudge fails", async () => { - // The resume payload is durable; a failed re-enqueue only re-exposes the - // world's wake race and must not fail the cancel request. - vi.mocked(resumeHook).mockResolvedValue({ runId: "session-1" } as never); - vi.mocked(getWorld).mockRejectedValue(new Error("world unavailable")); - const error = vi.spyOn(console, "error").mockImplementation(() => {}); - - const result = await requestWorkflowTurnCancellation({ sessionId: "session-1" }); - - expect(result).toEqual({ status: "accepted" }); - expect(error).toHaveBeenCalled(); - }); - - it("classifies a missing hook as no_active_turn without nudging", async () => { - vi.mocked(resumeHook).mockRejectedValue(new HookNotFoundError("no hook")); - - const result = await requestWorkflowTurnCancellation({ sessionId: "session-1" }); - - expect(result).toEqual({ reason: "HookNotFoundError", status: "no_active_turn" }); - expect(reenqueueRun).not.toHaveBeenCalled(); - }); - - it("forwards the turn guard in the hook payload", async () => { - vi.mocked(resumeHook).mockResolvedValue({ runId: "session-1" } as never); - vi.mocked(getWorld).mockResolvedValue({} as never); - - await requestWorkflowTurnCancellation({ sessionId: "session-1", turnId: "turn_3" }); - - expect(resumeHook).toHaveBeenCalledWith("session-1:cancel", { turnId: "turn_3" }); - }); -}); - -describe("isRetryableInactiveCancelReason", () => { - it("retries only hook-claim contention", () => { - expect(isRetryableInactiveCancelReason("EntityConflictError")).toBe(true); - expect(isRetryableInactiveCancelReason("HookNotFoundError")).toBe(false); - expect(isRetryableInactiveCancelReason("WorkflowRunNotFoundError")).toBe(false); - expect(isRetryableInactiveCancelReason(undefined)).toBe(false); - }); -}); diff --git a/packages/eve/src/execution/turn-cancellation-request.ts b/packages/eve/src/execution/turn-cancellation-request.ts deleted file mode 100644 index 3c85c7ef8..000000000 --- a/packages/eve/src/execution/turn-cancellation-request.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { - EntityConflictError, - HookNotFoundError, - RunExpiredError, - WorkflowRunNotFoundError, -} from "#compiled/@workflow/errors/index.js"; - -import type { CancelTurnInput, CancelTurnResult } from "#channel/types.js"; -import { createLogger, logError } from "#internal/logging.js"; -import { getWorld, reenqueueRun, resumeHook } from "#internal/workflow/runtime.js"; -import { - sessionCancelHookToken, - type TurnCancelPayload, -} from "#execution/turn-cancellation-token.js"; - -const log = createLogger("execution.turn-cancellation-request"); - -/** Requests cancellation through a session's stable workflow hook. */ -export async function requestWorkflowTurnCancellation( - input: CancelTurnInput, -): Promise { - const payload: TurnCancelPayload = input.turnId === undefined ? {} : { turnId: input.turnId }; - - let hookRunId: string | undefined; - try { - hookRunId = readHookRunId(await resumeHook(sessionCancelHookToken(input.sessionId), payload)); - } catch (error) { - const reason = classifyInactiveCancelTarget(error); - if (reason !== undefined) { - return { reason, status: "no_active_turn" }; - } - throw error; - } - - // The world does not reliably reschedule a suspended run when one of its - // hooks is resumed: a parked run has been observed holding an accepted - // cancel for minutes without a single execution. Nudge the scheduler - // explicitly — targeting the run that owns the hook (the turn workflow - // run, not the session's driver run named by `input.sessionId`), since - // that is the run suspended racing the cancel payload. The resume above - // is already durable, so a failed nudge only re-exposes that wake race — - // never a lost cancel — and a redundant one is a harmless replay. - try { - await reenqueueRun(await getWorld(), hookRunId ?? input.sessionId); - } catch (error) { - logError(log, "failed to re-enqueue the cancel hook's owning run", error, { - hookRunId, - sessionId: input.sessionId, - }); - } - - return { status: "accepted" }; -} - -/** - * Returns true when a `no_active_turn` reason can heal with time — world - * contention such as a hook-claim conflict during a queue wake. Terminal - * reasons (hook gone, run gone, run expired) mean the target turn already - * settled and will never accept a cancel again. - */ -export function isRetryableInactiveCancelReason(reason: string | undefined): boolean { - return reason === "EntityConflictError"; -} - -function readHookRunId(hook: unknown): string | undefined { - if (typeof hook !== "object" || hook === null || !("runId" in hook)) return undefined; - const runId = (hook as { readonly runId?: unknown }).runId; - return typeof runId === "string" && runId.length > 0 ? runId : undefined; -} - -function classifyInactiveCancelTarget(error: unknown): string | undefined { - if (HookNotFoundError.is(error)) return "HookNotFoundError"; - if (WorkflowRunNotFoundError.is(error)) return "WorkflowRunNotFoundError"; - if (RunExpiredError.is(error)) return "RunExpiredError"; - if (EntityConflictError.is(error)) return "EntityConflictError"; - return undefined; -} diff --git a/packages/eve/src/execution/workflow-runtime.ts b/packages/eve/src/execution/workflow-runtime.ts index bf1a61d9a..9711a203a 100644 --- a/packages/eve/src/execution/workflow-runtime.ts +++ b/packages/eve/src/execution/workflow-runtime.ts @@ -46,9 +46,12 @@ import { getCompiledRuntimeAgentBundle } from "#runtime/sessions/compiled-agent- import { buildRunContext } from "#execution/runtime-context.js"; import { parseNdjsonStream } from "#execution/ndjson-stream.js"; import { RuntimeNoActiveSessionError } from "#execution/runtime-errors.js"; -import { cancelTurnWithDescendantFanout } from "#execution/turn-cancellation-fanout.js"; import type { WorkflowEntryInput } from "#execution/workflow-entry.js"; import { walkCauseChain } from "#shared/errors.js"; +import { + sessionCancelHookToken, + type TurnCancelPayload, +} from "#execution/turn-cancellation-token.js"; const WORKFLOW_ENTRY_NAME = "workflowEntry"; const TURN_WORKFLOW_NAME = "turnWorkflow"; @@ -180,16 +183,7 @@ export function createWorkflowRuntime(config: { }, async cancelTurn(input: CancelTurnInput): Promise { - return await cancelTurnWithDescendantFanout({ - ...input, - resolveRemoteRegistry: async () => { - const bundle = await getCompiledRuntimeAgentBundle({ - compiledArtifactsSource: config.compiledArtifactsSource, - nodeId: config.nodeId, - }); - return bundle.subagentRegistry.subagentsByNodeId; - }, - }); + return await requestWorkflowTurnCancellation(input); }, async terminateSession(input: TerminateSessionInput): Promise { @@ -265,6 +259,32 @@ export function createWorkflowRuntime(config: { }; } +/** Requests cancellation through a session's stable workflow hook. */ +export async function requestWorkflowTurnCancellation( + input: CancelTurnInput, +): Promise { + const payload: TurnCancelPayload = input.turnId === undefined ? {} : { turnId: input.turnId }; + + try { + await resumeHook(sessionCancelHookToken(input.sessionId), payload); + return { status: "accepted" }; + } catch (error) { + const reason = classifyInactiveCancelTarget(error); + if (reason !== undefined) { + return { reason, status: "no_active_turn" }; + } + throw error; + } +} + +function classifyInactiveCancelTarget(error: unknown): string | undefined { + if (HookNotFoundError.is(error)) return "HookNotFoundError"; + if (WorkflowRunNotFoundError.is(error)) return "WorkflowRunNotFoundError"; + if (RunExpiredError.is(error)) return "RunExpiredError"; + if (EntityConflictError.is(error)) return "EntityConflictError"; + return undefined; +} + function isAlreadyTerminalSessionError(error: unknown): boolean { for (const candidate of walkCauseChain(error)) { if ( diff --git a/packages/eve/src/internal/application/compiled-artifacts.ts b/packages/eve/src/internal/application/compiled-artifacts.ts index 81114d440..4ce9c7e30 100644 --- a/packages/eve/src/internal/application/compiled-artifacts.ts +++ b/packages/eve/src/internal/application/compiled-artifacts.ts @@ -5,7 +5,6 @@ import { join } from "node:path"; import type { CompileMetadata } from "#compiler/artifacts.js"; import type { CompileAgentResult } from "#compiler/compile-agent.js"; import { createCompiledModuleMapSource } from "#compiler/module-map.js"; -import { getWorldImport } from "@workflow/utils"; import { stringifyEsmImportSpecifier } from "#internal/application/import-specifier.js"; import { resolvePackageCompiledFilePath, @@ -15,6 +14,7 @@ import { buildPackageUserAgent } from "#internal/user-agent.js"; import type { AgentWorkflowWorldDefinition } from "#shared/agent-definition.js"; import { readMaterializedAuthoredModuleIndex } from "#internal/materialized-authored-modules.js"; import { usesParentDevelopmentWorkflowWorld } from "#internal/workflow/development-world-protocol.js"; +import { resolveWorkflowWorldImport } from "#internal/workflow/world-target.js"; export type BuiltInWorkflowWorldTarget = "local" | "vercel"; @@ -313,7 +313,7 @@ export function createWorkflowWorldPluginSource(input: { defaultWorld: BuiltInWorkflowWorldTarget; }): string { const targetWorld = input.configuredWorld ?? input.defaultWorld; - const packageName = getWorldImport({ WORKFLOW_TARGET_WORLD: targetWorld }); + const packageName = resolveWorkflowWorldImport(targetWorld); const wiring = resolveWorkflowWorldWiring(packageName); const workflowRuntimeImportSpecifier = resolvePackageCompiledFilePath( "src/compiled/@workflow/core/runtime.js", diff --git a/packages/eve/src/internal/workflow/development-world-protocol.ts b/packages/eve/src/internal/workflow/development-world-protocol.ts index 4756b84a6..8249cf56c 100644 --- a/packages/eve/src/internal/workflow/development-world-protocol.ts +++ b/packages/eve/src/internal/workflow/development-world-protocol.ts @@ -1,6 +1,5 @@ -import { getWorldImport } from "@workflow/utils"; - import type { AgentWorkflowWorldDefinition } from "#shared/agent-definition.js"; +import { resolveWorkflowWorldImport } from "#internal/workflow/world-target.js"; export const DEVELOPMENT_WORKFLOW_WORLD_ROUTE = "/eve/v1/dev/internal/workflow-world"; @@ -15,10 +14,7 @@ export const DEVELOPMENT_WORKFLOW_WORLD_ROUTE = "/eve/v1/dev/internal/workflow-w export function usesParentDevelopmentWorkflowWorld( configuredWorld: AgentWorkflowWorldDefinition | undefined, ): boolean { - return ( - getWorldImport({ WORKFLOW_TARGET_WORLD: configuredWorld ?? "local" }) === - "@workflow/world-local" - ); + return resolveWorkflowWorldImport(configuredWorld ?? "local") === "@workflow/world-local"; } export const DEVELOPMENT_WORKFLOW_SECRET_ENV = "EVE_DEV_WORKFLOW_TRANSPORT_SECRET"; export const DEVELOPMENT_WORKER_APP_ROOT_ENV = "EVE_DEV_WORKER_APP_ROOT"; diff --git a/packages/eve/src/internal/workflow/world-target.test.ts b/packages/eve/src/internal/workflow/world-target.test.ts new file mode 100644 index 000000000..c740fbdb3 --- /dev/null +++ b/packages/eve/src/internal/workflow/world-target.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from "vitest"; + +import { resolveWorkflowWorldImport } from "#internal/workflow/world-target.js"; + +describe("resolveWorkflowWorldImport", () => { + it("maps the built-in world shorthands to their packages", () => { + expect(resolveWorkflowWorldImport("local")).toBe("@workflow/world-local"); + expect(resolveWorkflowWorldImport("vercel")).toBe("@workflow/world-vercel"); + }); + + it("passes custom world specifiers through unchanged", () => { + expect(resolveWorkflowWorldImport("@acme/world-redis")).toBe("@acme/world-redis"); + expect(resolveWorkflowWorldImport("./relative/world.js")).toBe("./relative/world.js"); + }); +}); diff --git a/packages/eve/src/internal/workflow/world-target.ts b/packages/eve/src/internal/workflow/world-target.ts new file mode 100644 index 000000000..d9643a564 --- /dev/null +++ b/packages/eve/src/internal/workflow/world-target.ts @@ -0,0 +1,14 @@ +/** + * Maps a configured workflow world target to its package import specifier. + * + * Mirrors the normalization the Workflow DevKit applies to + * `WORKFLOW_TARGET_WORLD` (`"local"` and `"vercel"` are shorthands for the + * first-party world packages; anything else is already a specifier). Owned + * by eve so builds do not depend on `@workflow/utils` keeping the helper + * exported. + */ +export function resolveWorkflowWorldImport(targetWorld: string): string { + if (targetWorld === "local") return "@workflow/world-local"; + if (targetWorld === "vercel") return "@workflow/world-vercel"; + return targetWorld; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index da0a62e02..d1381c7fd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1150,26 +1150,26 @@ importers: specifier: 1.28.8 version: 1.28.8 '@workflow/core': - specifier: 5.0.0-beta.37 - version: 5.0.0-beta.37(@opentelemetry/api@1.9.1)(supports-color@10.2.2)(ws@8.21.1(bufferutil@4.1.0)) + specifier: 5.0.0-beta.38 + version: 5.0.0-beta.38(@opentelemetry/api@1.9.1)(supports-color@10.2.2)(ws@8.21.1(bufferutil@4.1.0)) '@workflow/errors': - specifier: 5.0.0-beta.13 - version: 5.0.0-beta.13 + specifier: 5.0.0-beta.14 + version: 5.0.0-beta.14 '@workflow/serde': specifier: 5.0.0-beta.2 version: 5.0.0-beta.2 '@workflow/utils': - specifier: 5.0.0-beta.7 - version: 5.0.0-beta.7 + specifier: 5.0.0-beta.8 + version: 5.0.0-beta.8 '@workflow/world': specifier: 5.0.0-beta.23 version: 5.0.0-beta.23 '@workflow/world-local': - specifier: 5.0.0-beta.31 - version: 5.0.0-beta.31(@opentelemetry/api@1.9.1) + specifier: 5.0.0-beta.32 + version: 5.0.0-beta.32(@opentelemetry/api@1.9.1) '@workflow/world-vercel': - specifier: 5.0.0-beta.33 - version: 5.0.0-beta.33(@opentelemetry/api@1.9.1) + specifier: 5.0.0-beta.34 + version: 5.0.0-beta.34(@opentelemetry/api@1.9.1) ai: specifier: 'catalog:' version: 7.0.38(zod@4.4.3) @@ -7282,16 +7282,16 @@ packages: '@webgpu/types@0.1.71': resolution: {integrity: sha512-mMy8/ODcKhab808co15eW+yN+HgXoQxRQHTiBV9Mrvl1r0ufnid7YOcI+gi4eUWSWl9ezD6TW2KXccrL8HCh2A==} - '@workflow/core@5.0.0-beta.37': - resolution: {integrity: sha512-QaBWdRXpQj2dSMkl7sjsTL3TNTfo2yDcXcLd8rFJ6u6By7xj5rC4nu4qsk8Cn6f13Dh9jrQTXGbDrDVZaeLgpQ==} + '@workflow/core@5.0.0-beta.38': + resolution: {integrity: sha512-jXQFXQ6o5Px0UNzQwH7eCnNeR9XeVIYIjI99PoMmgXcZZ+3828/ij/jvsMI+5xW7yo8IJMm24gl8rxsVI0tngQ==} peerDependencies: '@opentelemetry/api': '1' peerDependenciesMeta: '@opentelemetry/api': optional: true - '@workflow/errors@5.0.0-beta.13': - resolution: {integrity: sha512-xRiG9MEunLxkb7hw7n17+/cKKH4QfFvnQly4cAF2ImxPRDnL0HvWot9eHOPS3S41LEyZ/LSs/sYrIN07XUZbgw==} + '@workflow/errors@5.0.0-beta.14': + resolution: {integrity: sha512-1RwF7LG7JbIp78zD+eb1A8ifDacI12wzpdMAFbzMODc/Ok0u+1IXbT1DIVKLtBLnUOPrWLxnXBYDnkvkvG9tAw==} '@workflow/serde@4.1.0': resolution: {integrity: sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==} @@ -7302,19 +7302,19 @@ packages: '@workflow/serde@5.0.0-beta.2': resolution: {integrity: sha512-INB+FcEKQkkFZa+s53sEMTNRFiBa2g9vzjsNuEDagvWxtI0t/cnCyoc57i7kS7nj+3/vGeRjO4Yn5ccbT0hyBQ==} - '@workflow/utils@5.0.0-beta.7': - resolution: {integrity: sha512-3vDiA8xea4USslIbz6m9HAhxDhvqLbETaXFxTNqVm/m8eji0rLKwnfF6JLaH0Qr1GsovysoOUlvyibokkNHZrg==} + '@workflow/utils@5.0.0-beta.8': + resolution: {integrity: sha512-dDiQ/LT05g9HyOicqWNNX/Gw/fmO7Ma2kZ+ddUXXpm/EFTMRS9eiI69suSe1+1NnhMzdV6ObI8wrO5lvQqPIbg==} - '@workflow/world-local@5.0.0-beta.31': - resolution: {integrity: sha512-Z9nq5lzjXnig4ONfLnVwTqx6EbxJjhTB5SBKT8ERcKe6/SFm30s1BIZ03bZBJox2Y4jJAgsNgvey29xWGnzMAg==} + '@workflow/world-local@5.0.0-beta.32': + resolution: {integrity: sha512-D8kb1IHHrkDbKnpUw3cqZnlkis3YCYPF2BoV6vWpGUGTeSGsnTZjMsrqsh5luKeZSUqK5BCO/YC27S9AdkBO0w==} peerDependencies: '@opentelemetry/api': '1' peerDependenciesMeta: '@opentelemetry/api': optional: true - '@workflow/world-vercel@5.0.0-beta.33': - resolution: {integrity: sha512-6MWaWw85145AInDBYql33XPETAjrOvu40wJYje/JwJwkFtOrGSx2B2vjbNxscafsN9TIZYUCRdK53spcqsMNzg==} + '@workflow/world-vercel@5.0.0-beta.34': + resolution: {integrity: sha512-bSBxUcs90wbhY8NlZYojoPAL7UwDuiTxr1R6eCb3ZCHCly5pTFXnkMIsM2hScPFM6ea/KImMl1KQVJc6OCPjIQ==} peerDependencies: '@opentelemetry/api': '1' peerDependenciesMeta: @@ -21035,19 +21035,19 @@ snapshots: '@webgpu/types@0.1.71': {} - '@workflow/core@5.0.0-beta.37(@opentelemetry/api@1.9.1)(supports-color@10.2.2)(ws@8.21.1(bufferutil@4.1.0))': + '@workflow/core@5.0.0-beta.38(@opentelemetry/api@1.9.1)(supports-color@10.2.2)(ws@8.21.1(bufferutil@4.1.0))': dependencies: '@aws-sdk/credential-provider-web-identity': 3.972.49 '@jridgewell/trace-mapping': 0.3.31 '@standard-schema/spec': 1.0.0 '@types/ms': 2.1.0 '@vercel/functions': 3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)) - '@workflow/errors': 5.0.0-beta.13 + '@workflow/errors': 5.0.0-beta.14 '@workflow/serde': 5.0.0-beta.2 - '@workflow/utils': 5.0.0-beta.7 + '@workflow/utils': 5.0.0-beta.8 '@workflow/world': 5.0.0-beta.23 - '@workflow/world-local': 5.0.0-beta.31(@opentelemetry/api@1.9.1) - '@workflow/world-vercel': 5.0.0-beta.33(@opentelemetry/api@1.9.1) + '@workflow/world-local': 5.0.0-beta.32(@opentelemetry/api@1.9.1) + '@workflow/world-vercel': 5.0.0-beta.34(@opentelemetry/api@1.9.1) debug: 4.4.3(supports-color@10.2.2) devalue: 5.8.1 ms: 2.1.3 @@ -21062,9 +21062,9 @@ snapshots: - supports-color - ws - '@workflow/errors@5.0.0-beta.13': + '@workflow/errors@5.0.0-beta.14': dependencies: - '@workflow/utils': 5.0.0-beta.7 + '@workflow/utils': 5.0.0-beta.8 ms: 2.1.3 '@workflow/serde@4.1.0': {} @@ -21073,15 +21073,15 @@ snapshots: '@workflow/serde@5.0.0-beta.2': {} - '@workflow/utils@5.0.0-beta.7': + '@workflow/utils@5.0.0-beta.8': dependencies: ms: 2.1.3 - '@workflow/world-local@5.0.0-beta.31(@opentelemetry/api@1.9.1)': + '@workflow/world-local@5.0.0-beta.32(@opentelemetry/api@1.9.1)': dependencies: '@vercel/queue': 0.4.0(@opentelemetry/api@1.9.1) - '@workflow/errors': 5.0.0-beta.13 - '@workflow/utils': 5.0.0-beta.7 + '@workflow/errors': 5.0.0-beta.14 + '@workflow/utils': 5.0.0-beta.8 '@workflow/world': 5.0.0-beta.23 async-sema: 3.1.1 ulid: 3.0.2 @@ -21090,11 +21090,11 @@ snapshots: optionalDependencies: '@opentelemetry/api': 1.9.1 - '@workflow/world-vercel@5.0.0-beta.33(@opentelemetry/api@1.9.1)': + '@workflow/world-vercel@5.0.0-beta.34(@opentelemetry/api@1.9.1)': dependencies: '@vercel/oidc': 3.2.0 '@vercel/queue': 0.4.0(@opentelemetry/api@1.9.1) - '@workflow/errors': 5.0.0-beta.13 + '@workflow/errors': 5.0.0-beta.14 '@workflow/world': 5.0.0-beta.23 cbor-x: 1.6.0 ulid: 3.0.2 From 38b9c95e96cbfe16191a019a586f9a8225d22572 Mon Sep 17 00:00:00 2001 From: Andrew Barba Date: Thu, 30 Jul 2026 12:14:56 -0400 Subject: [PATCH 8/9] docs(eve): tighten cancellation comments to their load-bearing why Signed-off-by: Andrew Barba --- packages/eve/src/channel/types.ts | 5 ++--- .../src/execution/cancel-descendant-turns-step.ts | 8 +++----- .../execution/turn-cancellation-control.test.ts | 8 +++----- .../src/execution/turn-cancellation-control.ts | 15 ++++++--------- .../turn-cancellation.integration.test.ts | 3 +-- packages/eve/src/execution/turn-workflow.ts | 7 +++---- .../eve/src/internal/workflow/world-target.ts | 11 ++++------- 7 files changed, 22 insertions(+), 35 deletions(-) diff --git a/packages/eve/src/channel/types.ts b/packages/eve/src/channel/types.ts index 7784ddb39..b21987e82 100644 --- a/packages/eve/src/channel/types.ts +++ b/packages/eve/src/channel/types.ts @@ -31,9 +31,8 @@ export interface CancelTurnResult { readonly status: CancelTurnStatus; /** * For `no_active_turn`: the error class that classified the target as - * inactive (e.g. `HookNotFoundError`, `EntityConflictError`). Lets callers - * that expected an active turn distinguish "already finished" from a - * transiently unreachable cancel hook. + * inactive, distinguishing "already finished" (`HookNotFoundError`) from a + * transiently unreachable cancel hook (`EntityConflictError`). */ readonly reason?: string; } diff --git a/packages/eve/src/execution/cancel-descendant-turns-step.ts b/packages/eve/src/execution/cancel-descendant-turns-step.ts index b7d6a7265..9924b7173 100644 --- a/packages/eve/src/execution/cancel-descendant-turns-step.ts +++ b/packages/eve/src/execution/cancel-descendant-turns-step.ts @@ -16,11 +16,9 @@ import type { } from "#runtime/actions/types.js"; import type { RuntimeSubagentRegistry } from "#runtime/subagents/registry.js"; -// Descendants listed in a pending batch should be actively running, so a -// cancel that cannot be delivered is anomalous: retry with backoff long -// enough to ride out transient world contention (queue wakes, hook-claim -// conflicts), then log loudly rather than dropping the cancel silently — -// an uncancelled child otherwise runs to completion with no trace of why. +// Retry through transient world contention (queue wakes, hook-claim +// conflicts), then log loudly: a silently dropped cancel leaves the child +// running to completion with no trace of why. const CANCEL_ATTEMPTS = 8; const CANCEL_RETRY_INITIAL_DELAY_MS = 250; const CANCEL_RETRY_MAX_DELAY_MS = 1_500; diff --git a/packages/eve/src/execution/turn-cancellation-control.test.ts b/packages/eve/src/execution/turn-cancellation-control.test.ts index 2f78a40ca..731b53fd1 100644 --- a/packages/eve/src/execution/turn-cancellation-control.test.ts +++ b/packages/eve/src/execution/turn-cancellation-control.test.ts @@ -114,11 +114,9 @@ describe("createTurnCancellationControl", () => { }); it("flips the signal in the same microtask that consumes the payload", async () => { - // Reproduces a wake replay where a journaled cancel and a completed - // turn step resolve in the same drive, payload first. The turn loop - // checks `signal.aborted` in the step's continuation — one microtask - // after the payload read — so the abort must fire inside the read - // continuation, not a `.then` chained behind it. + // A wake replays a journaled cancel and a completed step in one drain, + // payload first; the settle check reads `signal.aborted` one microtask + // later, so a `.then`-chained abort would lose to it. let releasePayload!: (result: IteratorResult) => void; const firstRead = new Promise>((resolve) => { releasePayload = resolve; diff --git a/packages/eve/src/execution/turn-cancellation-control.ts b/packages/eve/src/execution/turn-cancellation-control.ts index 18f0a58e3..2451e821e 100644 --- a/packages/eve/src/execution/turn-cancellation-control.ts +++ b/packages/eve/src/execution/turn-cancellation-control.ts @@ -51,12 +51,10 @@ export async function createTurnCancellationControl(input: { } const controller = new AbortController(); - // The abort fires inside the read continuation itself — not a chained - // `.then` — so the signal flips in the same microtask that consumes the - // payload. When a wake replays a journaled cancel alongside a completed - // step, the turn loop checks `signal.aborted` in the step's continuation; - // a chained abort would run one microtask later and lose to that check, - // letting an ordinary completion swallow the cancel. + // The abort must fire inside the read continuation — not a chained + // `.then` — so the signal is already flipped when a same-drain + // continuation (the turn loop's settle check) reads it; one microtask + // later and an ordinary completion swallows the cancel. const requested = consumeMatchingCancel(iterator, input.expectedTurnId, () => { controller.abort(new TurnCancelledError()); }).then(() => "cancel" as const); @@ -76,9 +74,8 @@ export async function createTurnCancellationControl(input: { } // Mismatched turn guards are consumed as no-ops; each read is durable, -// so the skip sequence replays deterministically. `onCancel` runs -// synchronously within the matching read's continuation (see the abort -// ordering note in `createTurnCancellationControl`). +// so the skip sequence replays deterministically. `onCancel` fires inside +// the matching read's continuation (see the abort ordering note above). async function consumeMatchingCancel( iterator: AsyncIterator, expectedTurnId: string, diff --git a/packages/eve/src/execution/turn-cancellation.integration.test.ts b/packages/eve/src/execution/turn-cancellation.integration.test.ts index 1cef2066c..857750b00 100644 --- a/packages/eve/src/execution/turn-cancellation.integration.test.ts +++ b/packages/eve/src/execution/turn-cancellation.integration.test.ts @@ -486,8 +486,7 @@ describe("turn cancellation integration", () => { expect(fixture.toolAborts()).toBe(1); // Session.cancel() addresses the same session by id. With the turn - // settled and its cancel hook swept, it reports the benign status, - // classified by the error that marked the target inactive. + // settled and its cancel hook swept, it reports the benign status. await waitForHookSweep(sessionCancelHookToken(run.runId)); const session = createSession(run.runId, rawToken, workflowRuntime); await expect(session.cancel()).resolves.toEqual({ diff --git a/packages/eve/src/execution/turn-workflow.ts b/packages/eve/src/execution/turn-workflow.ts index 1248afac5..e1250ff19 100644 --- a/packages/eve/src/execution/turn-workflow.ts +++ b/packages/eve/src/execution/turn-workflow.ts @@ -111,10 +111,9 @@ async function runTurnOwnedWorkflow(input: TurnWorkflowInput): Promise { ? result.pendingRuntimeActionKeys : undefined; - // A cancel observed while the step was already returning must still win: - // the step body may have missed the abort and produced an ordinary - // completion, but the user's cancel was claimed. Pending runtime-action - // batches are exempt — their in-line wait below observes the signal. + // A cancel observed while the step was returning must still win: the + // step may have missed the abort and completed normally. Pending + // runtime-action batches are exempt — their wait observes the signal. if ( result.action === "cancelled" || (cancellation?.signal.aborted === true && pendingActionKeys === undefined) diff --git a/packages/eve/src/internal/workflow/world-target.ts b/packages/eve/src/internal/workflow/world-target.ts index d9643a564..b9e297500 100644 --- a/packages/eve/src/internal/workflow/world-target.ts +++ b/packages/eve/src/internal/workflow/world-target.ts @@ -1,11 +1,8 @@ /** - * Maps a configured workflow world target to its package import specifier. - * - * Mirrors the normalization the Workflow DevKit applies to - * `WORKFLOW_TARGET_WORLD` (`"local"` and `"vercel"` are shorthands for the - * first-party world packages; anything else is already a specifier). Owned - * by eve so builds do not depend on `@workflow/utils` keeping the helper - * exported. + * Maps a configured workflow world target to its package import specifier, + * mirroring the Workflow DevKit's `WORKFLOW_TARGET_WORLD` normalization: + * `"local"` and `"vercel"` are shorthands for the first-party world + * packages; anything else is already a specifier. */ export function resolveWorkflowWorldImport(targetWorld: string): string { if (targetWorld === "local") return "@workflow/world-local"; From 0c5c379e4f10c05904f9c4a1a9d67dcd3c6c2e24 Mon Sep 17 00:00:00 2001 From: Andrew Barba Date: Thu, 30 Jul 2026 12:38:16 -0400 Subject: [PATCH 9/9] fix(eve): stub core's world factories at vendor time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @workflow/core@5.0.0-beta.38 restored runtime world selection: its createWorld() statically imports both first-party world packages, which dragged @workflow/world-local into hosted Vercel output through the vendored core runtime (caught by the app-runtime-dependencies scenario in CI). eve never calls that factory — worlds are selected and wired explicitly at build time — so the vendor step now resolves those imports to a throwing stub inside core. Each world package keeps its own vendored entry. The scenario's world-local canary moves from WORKFLOW_LOCAL_DATA_DIR (now named by world-agnostic core code) to the [world-local] log prefix, which only appears in world-local's own bundle. Also mock the vendored @vercel/oidc token lookup in the sandbox binding unit tests: on a machine with Vercel CLI auth and a .vercel project link, the credential fallback injected real project ids into the asserted SDK calls. Signed-off-by: Andrew Barba --- .changeset/workflow-beta-38.md | 2 +- .../vendor-compiled/@workflow/core.mjs | 31 +++++++++++++++++++ .../execution/sandbox/bindings/vercel.test.ts | 9 ++++++ .../app-runtime-dependencies.scenario.test.ts | 11 ++++--- 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/.changeset/workflow-beta-38.md b/.changeset/workflow-beta-38.md index 5e245bb5d..0d9365033 100644 --- a/.changeset/workflow-beta-38.md +++ b/.changeset/workflow-beta-38.md @@ -2,4 +2,4 @@ "eve": patch --- -Upgrade the vendored Workflow DevKit to `@workflow/core@5.0.0-beta.38`. This picks up the upstream fix for runs going dormant after an accepted hook resume (vercel/workflow#3183): a session cancelled while parked on subagents could previously hold the accepted cancel indefinitely — its turn only settling as cancelled when unrelated traffic happened to wake the run. +Upgrade the vendored Workflow DevKit to `@workflow/core@5.0.0-beta.38`. This picks up the upstream fix for runs going dormant after an accepted hook resume (vercel/workflow#3183): a session cancelled while parked on subagents could previously hold the accepted cancel indefinitely — its turn only settling as cancelled when unrelated traffic happened to wake the run. beta.38 also restored runtime world selection in core's `createWorld()`, which statically imports both first-party worlds; eve stubs those imports at vendor time so hosted Vercel bundles keep excluding local-world infrastructure. diff --git a/packages/eve/scripts/vendor-compiled/@workflow/core.mjs b/packages/eve/scripts/vendor-compiled/@workflow/core.mjs index ecb4b1890..5b980369a 100644 --- a/packages/eve/scripts/vendor-compiled/@workflow/core.mjs +++ b/packages/eve/scripts/vendor-compiled/@workflow/core.mjs @@ -94,10 +94,41 @@ const copyDeclarations = createDeclarationCopier({ }, }); +// @workflow/core@5.0.0-beta.38 restored runtime world selection: its +// `createWorld()` statically imports both first-party world packages. eve +// never calls that factory — worlds are selected and wired explicitly at +// build time — so those imports would only drag the local world into hosted +// Vercel output (and the Vercel world into local-only builds). Stub them +// inside core; each world package keeps its own vendored entry. +const CORE_WORLD_FACTORY_STUB_ID = "\0eve-core-world-factory-stub"; + +function stubCoreWorldFactories() { + return { + name: "eve:stub-core-world-factories", + resolveId(source, importer) { + if (source !== "@workflow/world-local" && source !== "@workflow/world-vercel") return null; + if (typeof importer !== "string") return null; + if (!importer.replaceAll("\\", "/").includes("/@workflow/core/")) return null; + return CORE_WORLD_FACTORY_STUB_ID; + }, + load(id) { + if (id !== CORE_WORLD_FACTORY_STUB_ID) return null; + return [ + "export function createWorld() {", + ' throw new Error("Unsupported in eve: @workflow/core createWorld(). ' + + 'eve selects and wires the workflow world explicitly at build time.");', + "}", + "", + ].join("\n"); + }, + }; +} + export default { packageName: "@workflow/core", compiledPath: "@workflow/core", chunkGroup: "workflow", + plugins: [stubCoreWorldFactories()], entries: [ { outputPath: "index", diff --git a/packages/eve/src/execution/sandbox/bindings/vercel.test.ts b/packages/eve/src/execution/sandbox/bindings/vercel.test.ts index 1a1f85928..38c23c880 100644 --- a/packages/eve/src/execution/sandbox/bindings/vercel.test.ts +++ b/packages/eve/src/execution/sandbox/bindings/vercel.test.ts @@ -6,6 +6,15 @@ import { SandboxTemplateNotProvisionedError } from "#public/definitions/sandbox- import { vercel } from "#public/sandbox/backends/vercel.js"; import { createVercelSandbox } from "#execution/sandbox/bindings/vercel.js"; +// The credential fallback consults the developer's Vercel CLI auth and the +// repo's `.vercel` project link; on a linked, logged-in machine it would +// inject real project credentials into the asserted SDK calls. +vi.mock("#compiled/@vercel/oidc/index.js", () => ({ + getVercelOidcToken: vi.fn(async () => { + throw new Error("No ambient Vercel OIDC token in unit tests."); + }), +})); + function createMockCommandResult() { return { exitCode: 0, diff --git a/packages/eve/test/scenarios/app-runtime-dependencies.scenario.test.ts b/packages/eve/test/scenarios/app-runtime-dependencies.scenario.test.ts index 20be75bbe..8c8a032d2 100644 --- a/packages/eve/test/scenarios/app-runtime-dependencies.scenario.test.ts +++ b/packages/eve/test/scenarios/app-runtime-dependencies.scenario.test.ts @@ -847,11 +847,12 @@ describe("app runtime dependency tracing", () => { expect(vercelFunctionsSource).not.toContain("chokidar"); expect(vercelFunctionsSource).not.toContain("[eve:dev]"); expect(vercelFunctionsSource).not.toContain("rollup:reload"); - // The world-local canary is its config env var, not its error-class - // names: the semantic-error catalog legitimately embeds names like - // `DataDirAccessError` as matcher strings without bundling any - // world-local code. - expect(vercelFunctionsSource).not.toContain("WORKFLOW_LOCAL_DATA_DIR"); + // The world-local canary is its log prefix, not names other packages + // legitimately mention without bundling world-local code: the + // semantic-error catalog embeds error-class names like + // `DataDirAccessError`, and @workflow/core's runtime world factory + // (stubbed out at vendor time) names `WORKFLOW_LOCAL_DATA_DIR`. + expect(vercelFunctionsSource).not.toContain("[world-local]"); }, 30_000); it("loads instrumentation runtime dependencies from hosted Vercel output", async () => {