Skip to content

Commit 3360bff

Browse files
committed
fix(sdk,api,local): surface executions interrupted by a daemon restart
1 parent fff7ed6 commit 3360bff

8 files changed

Lines changed: 534 additions & 1 deletion

File tree

‎.changeset/execution-tombstones.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
---
2+
"@executor-js/sdk": patch
3+
"@executor-js/api": patch
4+
"@executor-js/local-app": patch
5+
---
6+
7+
fix: surface executions interrupted by a daemon restart instead of losing them silently
8+
9+
A paused execution lives as an in-memory fiber inside the running engine. When
10+
the local service restarts (login, crash, upgrade), every fiber is gone and a
11+
later `executor resume` read as "approval expired" — silently discarding work
12+
the agent believed was still pending.
13+
14+
Executions now write a lightweight durable tombstone (id + status +
15+
timestamp, no arguments, no results, no secrets) at pause time. On boot the
16+
service marks every non-terminal tombstone `interrupted`; resuming an
17+
interrupted execution returns an explicit `InterruptedExecutionError` telling
18+
the agent to re-trigger the action, which is safe because nothing ran.
19+
20+
Also adds the `@executor-js/sdk` execution-record store used by hosts that
21+
need the same guarantee (cloud, self-host).

‎apps/local/src/executor.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,28 @@ const createLocalExecutorLayer = (options: LocalExecutorOptions = {}) => {
232232
},
233233
});
234234

235+
// Boot sweep: any execution still running|paused from a previous
236+
// process is unrecoverable (its fiber died with that process). Mark
237+
// them interrupted so resume surfaces honestly instead of "not found".
238+
// Runs after storage opens and before the API/MCP surfaces accept
239+
// resume calls; never fails boot (a sweep hiccup is logged, not fatal).
240+
yield* executor.executionRecords.sweepInterrupted().pipe(
241+
Effect.map(({ interrupted }) => {
242+
if (interrupted > 0) {
243+
console.warn(
244+
`[executor] Marked ${interrupted} execution(s) interrupted after restart; re-trigger them to resume.`,
245+
);
246+
}
247+
}),
248+
Effect.catch(() =>
249+
Effect.sync(() =>
250+
console.warn(
251+
"[executor] Execution tombstone sweep failed; interrupted state may be stale.",
252+
),
253+
),
254+
),
255+
);
256+
235257
if (migration.migrated) {
236258
console.warn(
237259
`[executor] Migrated local Executor data to v2; moved old DB to ${migration.backupPath}.`,

‎packages/core/api/src/executions/api.ts‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,20 @@ const ApprovalExpiredError = Schema.TaggedStruct("ApprovalExpiredError", {
7575
"The approval window closed before the action was approved. Nothing ran; trigger the action again.",
7676
});
7777

78+
/**
79+
* The execution was interrupted by a daemon restart before it settled.
80+
*
81+
* Distinct from `ApprovalExpiredError` (the human never answered) and
82+
* `ExecutionNotFoundError` (an id that was never ours): an interrupted
83+
* execution is one the agent believed was still pending, but the service
84+
* restarted and the fiber is unrecoverable. The honest outcome is
85+
* "re-trigger the action" — nothing ran, so re-triggering is safe.
86+
* See execution-records.ts.
87+
*/
88+
const InterruptedExecutionError = Schema.TaggedStruct("InterruptedExecutionError", {
89+
executionId: Schema.String,
90+
}).annotate({ httpApiStatus: 404 });
91+
7892
/**
7993
* An artifact-originated execution that could not be turned into a call: the
8094
* code was not the shell proxy's emission, the artifact is not this caller's,
@@ -125,6 +139,11 @@ export const ExecutionsApi = HttpApiGroup.make("executions")
125139
params: ExecutionParams,
126140
payload: ResumeRequest,
127141
success: ResumeResponse,
128-
error: [InternalError, ExecutionNotFoundError, ApprovalExpiredError],
142+
error: [
143+
InternalError,
144+
ExecutionNotFoundError,
145+
ApprovalExpiredError,
146+
InterruptedExecutionError,
147+
],
129148
}),
130149
);
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
import { describe, expect, it } from "@effect/vitest";
2+
import { Context, Effect, Layer, Predicate } from "effect";
3+
import { HttpRouter, HttpServer } from "effect/unstable/http";
4+
import { HttpApi, HttpApiBuilder } from "effect/unstable/httpapi";
5+
6+
import type { Executor } from "@executor-js/sdk";
7+
8+
import { ExecutionsApi } from "../executions/api";
9+
import { ExecutionsHandlers } from "./executions";
10+
import { ExecutionEngineService, ExecutorService } from "../services";
11+
12+
// ---------------------------------------------------------------------------
13+
// Focused tests — spec execution-tombstones, AC4 (resume-time surface).
14+
//
15+
// When the daemon restarts, the paused fiber is gone. A resume must NOT read
16+
// as a generic "approval expired": if a tombstone exists for the execution
17+
// (written before the restart), the resume surfaces the honest
18+
// "interrupted — re-trigger" outcome (InterruptedExecutionError).
19+
// ---------------------------------------------------------------------------
20+
21+
const stubExecutor = (record: { executionId: string; status: string } | null): Executor =>
22+
// oxlint-disable-next-line executor/no-double-cast -- minimal executor double: executionRecords.get and pendingApprovals.consume are exercised
23+
({
24+
executionRecords: {
25+
get: () => Effect.succeed(record),
26+
put: () => Effect.void,
27+
sweepInterrupted: () => Effect.succeed({ interrupted: 0 }),
28+
},
29+
// resumeFromPendingApproval consumes a stored approval before reaching
30+
// the tombstone check; absent approvals are the restart scenario.
31+
pendingApprovals: {
32+
consume: () => Effect.succeed(null),
33+
discard: () => Effect.void,
34+
put: () => Effect.void,
35+
},
36+
}) as unknown as Executor;
37+
38+
// The engine remembers nothing (fresh process): live resume returns null, and
39+
// there is no pending-approval record — this is the restart scenario.
40+
// oxlint-disable-next-line executor/no-double-cast -- minimal engine double: only resume's null return (fresh process) is exercised
41+
const emptyEngine = {
42+
resume: () => Effect.succeed(null),
43+
} as unknown as ExecutionEngineService["Service"];
44+
45+
const runResume = (executor: Executor) => {
46+
const handler = HttpRouter.toWebHandler(
47+
HttpApiBuilder.layer(HttpApi.make("executor").add(ExecutionsApi)).pipe(
48+
Layer.provide(ExecutionsHandlers),
49+
Layer.provide(Layer.succeed(ExecutorService)(executor)),
50+
Layer.provide(Layer.succeed(ExecutionEngineService)(emptyEngine)),
51+
Layer.provideMerge(HttpServer.layerServices),
52+
Layer.provideMerge(Layer.succeed(HttpRouter.RouterConfig)({ maxParamLength: 1000 })),
53+
),
54+
{ disableLogger: false },
55+
).handler;
56+
// The handler's inferred type demands a Context<ExecutorService |
57+
// ExecutionEngineService> second argument at the type level (a beta.59
58+
// inference quirk of toWebHandler's ReqR) even though the layer above
59+
// provides both at runtime. Pass the runtime-provided context explicitly.
60+
// The handler's inferred type demands a Context<ExecutorService |
61+
// ExecutionEngineService> second argument at the type level (a beta.59
62+
// inference quirk of toWebHandler's ReqR) even though the layer above
63+
// provides both at runtime. Passing the real services explicitly also
64+
// satisfies the runtime — the stubs here are self-sufficient.
65+
const context = Context.make(ExecutorService, executor).pipe(
66+
Context.add(ExecutionEngineService, emptyEngine),
67+
);
68+
return handler(
69+
new Request("https://executor.test/executions/exec_1/resume", {
70+
method: "POST",
71+
headers: { "content-type": "application/json" },
72+
body: JSON.stringify({ action: "accept" }),
73+
}),
74+
context,
75+
);
76+
};
77+
78+
describe("resume after daemon restart (tombstone path)", () => {
79+
it("surfaces interrupted (404 + InterruptedExecutionError) when a tombstone exists", async () => {
80+
const res = await runResume(stubExecutor({ executionId: "exec_1", status: "interrupted" }));
81+
expect(res.status).toBe(404);
82+
const body = (await res.json()) as { _tag?: string; executionId?: string };
83+
expect(Predicate.isTagged(body, "InterruptedExecutionError")).toBe(true);
84+
expect(body.executionId).toBe("exec_1");
85+
});
86+
87+
it("surfaces interrupted for a stale paused tombstone (sweep missed it — not a live execution)", async () => {
88+
const res = await runResume(stubExecutor({ executionId: "exec_1", status: "paused" }));
89+
expect(res.status).toBe(404);
90+
expect(JSON.stringify(await res.json())).toContain("InterruptedExecutionError");
91+
});
92+
93+
it("falls through to approval-expired when no tombstone exists", async () => {
94+
const res = await runResume(stubExecutor(null));
95+
// ApprovalExpiredError is annotated httpApiStatus: 410 (Gone).
96+
expect(res.status).toBe(410);
97+
expect(JSON.stringify(await res.json())).toContain("ApprovalExpiredError");
98+
});
99+
100+
it("completed tombstones do not resurrect (completed is immutable)", async () => {
101+
const res = await runResume(stubExecutor({ executionId: "exec_1", status: "completed" }));
102+
// No tombstone hit for completed (immutable) — falls through to expired (410).
103+
expect(res.status).toBe(410);
104+
expect(JSON.stringify(await res.json())).toContain("ApprovalExpiredError");
105+
});
106+
});

‎packages/core/api/src/handlers/executions.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,29 @@ class ApprovalExpiredError extends Schema.TaggedErrorClass<ApprovalExpiredError>
6060
}
6161
}
6262

63+
/**
64+
* The execution was interrupted by a daemon restart before it settled.
65+
*
66+
* Distinct from `ApprovalExpiredError` (the human never answered) and
67+
* `ExecutionNotFoundError` (an id that was never ours): an interrupted
68+
* execution is one the agent believed was still pending, but the service
69+
* restarted and the fiber is unrecoverable. The honest outcome is
70+
* "re-trigger the action" — nothing ran, so re-triggering is safe.
71+
* 404, because the live execution no longer exists on this host.
72+
* See execution-records.ts (tombstones).
73+
*/
74+
class InterruptedExecutionError extends Schema.TaggedErrorClass<InterruptedExecutionError>()(
75+
"InterruptedExecutionError",
76+
{
77+
executionId: Schema.String,
78+
},
79+
{ httpApiStatus: 404 },
80+
) {
81+
override get message(): string {
82+
return "This execution was interrupted by a restart. Re-trigger the action.";
83+
}
84+
}
85+
6386
/**
6487
* Parse and bind one artifact-originated call, or fail with something the shell
6588
* can render inside the component that made it.
@@ -157,6 +180,16 @@ const resumeFromPendingApproval = (executionId: string, action: "accept" | "decl
157180
);
158181

159182
if (outcome.status === "completed") {
183+
// Terminal tombstone write — same rationale as the resume handler:
184+
// stop the record from being sweep-eligible once the work is done.
185+
// Best-effort, like every tombstone write.
186+
yield* executor.executionRecords
187+
.put({
188+
executionId,
189+
status: "completed",
190+
updatedAt: Date.now(),
191+
})
192+
.pipe(Effect.catchCause(() => Effect.void));
160193
const formatted = formatExecuteResult(outcome.result);
161194
return {
162195
status: "completed" as const,
@@ -196,6 +229,7 @@ export const ExecutionsHandlers = HttpApiBuilder.group(ExecutorApi, "executions"
196229
capture(
197230
Effect.gen(function* () {
198231
const engine = yield* ExecutionEngineService;
232+
const executor = yield* ExecutorService;
199233
// An artifact-originated request is not arbitrary code. It is parsed
200234
// against the shell proxy's one grammar and rewritten through the
201235
// artifact's connection bindings, exactly as `execute-action` does in
@@ -232,6 +266,15 @@ export const ExecutionsHandlers = HttpApiBuilder.group(ExecutorApi, "executions"
232266
code,
233267
address: String(outcome.execution.elicitationContext.address),
234268
});
269+
// Tombstone the pause so a restart marks it interrupted rather
270+
// than losing it silently. Best-effort, like the approval record.
271+
yield* executor.executionRecords
272+
.put({
273+
executionId: outcome.execution.id,
274+
status: "paused",
275+
updatedAt: Date.now(),
276+
})
277+
.pipe(Effect.catchCause(() => Effect.void));
235278
}
236279

237280
const formatted = formatPausedExecution(outcome.execution);
@@ -247,6 +290,7 @@ export const ExecutionsHandlers = HttpApiBuilder.group(ExecutorApi, "executions"
247290
capture(
248291
Effect.gen(function* () {
249292
const engine = yield* ExecutionEngineService;
293+
const executor = yield* ExecutorService;
250294
const result = yield* captureEngineError(
251295
engine.resume(path.executionId, {
252296
action: payload.action,
@@ -260,10 +304,34 @@ export const ExecutionsHandlers = HttpApiBuilder.group(ExecutorApi, "executions"
260304
if (!result) {
261305
const honoured = yield* resumeFromPendingApproval(path.executionId, payload.action);
262306
if (honoured) return honoured;
307+
308+
// No live pause and no pending-approval record. If a tombstone
309+
// exists for this execution, the daemon may have restarted since
310+
// it paused — surface that honestly instead of a generic
311+
// "approval expired". A running|paused tombstone read at resume
312+
// time means the boot sweep missed it (host couldn't enumerate);
313+
// treat it as interrupted here — it is not a live execution.
314+
const record = yield* executor.executionRecords.get(path.executionId);
315+
if (record !== null && record.status !== "completed") {
316+
return yield* new InterruptedExecutionError({ executionId: path.executionId });
317+
}
318+
263319
return yield* new ApprovalExpiredError({ executionId: path.executionId });
264320
}
265321

266322
if (result.status === "completed") {
323+
// Tombstone the terminal outcome so the record stops being
324+
// sweep-eligible: without this write, a resumed-to-completed
325+
// execution keeps its pre-restart "paused" tombstone and the
326+
// next boot sweep would mark it "interrupted" — factually wrong
327+
// for work that finished. Best-effort, like the pause write.
328+
yield* executor.executionRecords
329+
.put({
330+
executionId: path.executionId,
331+
status: "completed",
332+
updatedAt: Date.now(),
333+
})
334+
.pipe(Effect.catchCause(() => Effect.void));
267335
const formatted = formatExecuteResult(result.result);
268336
return {
269337
status: "completed" as const,
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
import { describe, expect, it } from "@effect/vitest";
2+
import { Effect } from "effect";
3+
4+
import { makeInMemoryBlobStore } from "./blob";
5+
import { makeExecutionRecordStore, type ExecutionRecord } from "./execution-records";
6+
7+
const record = (overrides?: Partial<ExecutionRecord>): ExecutionRecord => ({
8+
executionId: "exec_1",
9+
status: "paused",
10+
updatedAt: 1_000,
11+
...overrides,
12+
});
13+
14+
const partition = "u:t:s";
15+
16+
describe("makeExecutionRecordStore", () => {
17+
it.effect(
18+
"round-trips a record and reads it through a second store over the same partition",
19+
() =>
20+
Effect.gen(function* () {
21+
// The whole point of the tombstone: the engine that paused is gone, so
22+
// the record has to be readable by a caller that never saw the pause.
23+
const blobs = makeInMemoryBlobStore();
24+
yield* makeExecutionRecordStore(blobs, partition).put(record());
25+
26+
const restarted = makeExecutionRecordStore(blobs, partition);
27+
expect(yield* restarted.get("exec_1")).toStrictEqual(record());
28+
}),
29+
);
30+
31+
it.effect("reads absent for an unknown execution id", () =>
32+
Effect.gen(function* () {
33+
const store = makeExecutionRecordStore(makeInMemoryBlobStore(), partition);
34+
expect(yield* store.get("never_existed")).toBeNull();
35+
}),
36+
);
37+
38+
it.effect("is owner-scoped: a different partition does not see the record", () =>
39+
Effect.gen(function* () {
40+
const blobs = makeInMemoryBlobStore();
41+
yield* makeExecutionRecordStore(blobs, "u:t:other-subject").put(record());
42+
43+
// The namespace includes the partition — this caller's store reads a
44+
// different namespace and simply does not see the record.
45+
const otherStore = makeExecutionRecordStore(blobs, "u:t:s");
46+
expect(yield* otherStore.get("exec_1")).toBeNull();
47+
}),
48+
);
49+
50+
it.effect("treats a corrupt record as absent (never surfaces garbage)", () =>
51+
Effect.gen(function* () {
52+
const blobs = makeInMemoryBlobStore();
53+
yield* blobs.put("u:t:s/@execution-records", "exec_1", "not-json{{");
54+
55+
const store = makeExecutionRecordStore(blobs, partition);
56+
expect(yield* store.get("exec_1")).toBeNull();
57+
}),
58+
);
59+
60+
it.effect("sweep marks running|paused records interrupted and clears the live index", () =>
61+
Effect.gen(function* () {
62+
const blobs = makeInMemoryBlobStore();
63+
const store = makeExecutionRecordStore(blobs, partition);
64+
yield* store.put(record({ executionId: "exec_running", status: "running", updatedAt: 1 }));
65+
yield* store.put(record({ executionId: "exec_paused", status: "paused", updatedAt: 2 }));
66+
yield* store.put(record({ executionId: "exec_done", status: "completed", updatedAt: 3 }));
67+
68+
const swept = yield* store.sweepInterrupted();
69+
expect(swept.interrupted).toBe(2);
70+
71+
expect((yield* store.get("exec_running"))?.status).toBe("interrupted");
72+
expect((yield* store.get("exec_paused"))?.status).toBe("interrupted");
73+
// completed is immutable
74+
expect((yield* store.get("exec_done"))?.status).toBe("completed");
75+
76+
// A second sweep finds nothing live to mark.
77+
expect((yield* store.sweepInterrupted()).interrupted).toBe(0);
78+
}),
79+
);
80+
81+
it.effect("re-putting a terminal record removes it from the live index (no re-sweep)", () =>
82+
Effect.gen(function* () {
83+
const blobs = makeInMemoryBlobStore();
84+
const store = makeExecutionRecordStore(blobs, partition);
85+
yield* store.put(record({ executionId: "exec_1", status: "running" }));
86+
// Execution completes before any restart.
87+
yield* store.put(record({ executionId: "exec_1", status: "completed", updatedAt: 2 }));
88+
89+
expect((yield* store.sweepInterrupted()).interrupted).toBe(0);
90+
expect((yield* store.get("exec_1"))?.status).toBe("completed");
91+
}),
92+
);
93+
});

0 commit comments

Comments
 (0)