From 565d49bcb611ea18484cf4f8beaff1b719be51f4 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 10:13:44 +0200 Subject: [PATCH 01/23] feat(runtime)!: report worker callback outcome instead of run state BREAKING CHANGE: POST /runs/:runId/effects/:effectId/:attempt/:action returns { status, cancelRequested } instead of the run state. createLooms().workerCallback and the HTTP client return the same shape. Heartbeats may carry details that the next attempt's task includes. failEffect and worker fail callbacks can mark an attempt non-retryable or set retryAfterMs. Co-authored-by: Cursor --- .changeset/effect-reliability.md | 9 + apps/docs/src/routes/docs.reliability.tsx | 8 +- docs/production.md | 19 +- packages/client/src/client.ts | 62 ++++++- packages/core/src/effects.ts | 65 ++++++- packages/core/src/fold.ts | 25 +++ packages/core/src/protocol.ts | 4 + packages/core/src/state.ts | 5 + packages/core/src/worker-callback.ts | 22 ++- packages/runtime/src/dispatch-effect.ts | 66 +++++-- packages/runtime/src/helpers.ts | 9 +- packages/runtime/src/index.ts | 1 + packages/runtime/src/local-effect-executor.ts | 3 + packages/runtime/src/looms.ts | 3 +- packages/runtime/src/planners.test.ts | 6 + packages/runtime/src/retry-policy.ts | 18 +- packages/runtime/src/runtime.ts | 2 +- packages/runtime/src/server/routes/effects.ts | 73 +++++++- packages/runtime/src/types.ts | 14 +- packages/runtime/src/worker-callbacks.ts | 170 +++++++++++------- .../runtime/src/worker-effect-dispatch.ts | 2 +- packages/runtime/src/worker-protocol.test.ts | 118 +++++++++++- 22 files changed, 593 insertions(+), 111 deletions(-) create mode 100644 .changeset/effect-reliability.md diff --git a/.changeset/effect-reliability.md b/.changeset/effect-reliability.md new file mode 100644 index 0000000..5e278e9 --- /dev/null +++ b/.changeset/effect-reliability.md @@ -0,0 +1,9 @@ +--- +'@swirls/looms': minor +--- + +Worker callbacks now answer `{ status: 'accepted' | 'stale' | 'cancelled', cancelRequested }` instead of the run state. `createLooms().workerCallback` and `createLoomsClient().workerCallback` return that result. `cancelRequested` tells an attempt that is already running to stop, because a dispatch that has left the host cannot be recalled. A stale report records nothing. + +`failEffect({ error, nonRetryable, retryAfterMs })` fails the current attempt. `nonRetryable` ends the effect even when the retry policy has attempts left. `retryAfterMs` waits at least that long before the next attempt and never shortens the policy backoff. Worker `fail` callbacks take the same two fields. + +A heartbeat may include `details`. The latest details are kept across retries and included on the next `EffectWorkerTask`, so a resumed attempt can continue from the recorded progress. diff --git a/apps/docs/src/routes/docs.reliability.tsx b/apps/docs/src/routes/docs.reliability.tsx index 243e287..4ccc211 100644 --- a/apps/docs/src/routes/docs.reliability.tsx +++ b/apps/docs/src/routes/docs.reliability.tsx @@ -67,7 +67,13 @@ function Page() { a restart preserves the retry budget and backoff. Configure start, heartbeat, and execution timeouts for work delivered to external workers. Worker callbacks include the effect ID and attempt number; stale callbacks cannot complete a newer attempt. Exhaustion records{' '} - runtime.effect.failed. + runtime.effect.failed. A handler can stop that budget early with{' '} + failEffect and nonRetryable: true, or ask the next attempt to wait + at least retryAfterMs without shortening the policy backoff. A worker heartbeat + can carry details; the next dispatch of that effect includes the latest details + so the attempt can resume. The callback response is{' '} + {'{ status, cancelRequested }'}, not the run state.{' '} + cancelRequested tells an attempt that is already running to stop.

Signals and webhook delivery diff --git a/docs/production.md b/docs/production.md index 866dee0..b84ea45 100644 --- a/docs/production.md +++ b/docs/production.md @@ -19,13 +19,24 @@ are `EventPayloadTooLargeError` and `SnapshotPayloadTooLargeError`. ## Worker effect protocol -1. The runtime dispatches `{ runId, effectId, attempt, type, input }`. +1. The runtime dispatches `{ runId, effectId, attempt, type, input, details? }`. + `details` is the progress carried by the previous attempt's last heartbeat. 2. The worker calls `started` before the schedule-to-start deadline. -3. Long work calls `heartbeat`; each accepted heartbeat extends the heartbeat - deadline without exceeding start-to-close. -4. The worker calls exactly one of `complete` or `fail`. +3. Long work calls `heartbeat`. Each accepted heartbeat extends the heartbeat + deadline without exceeding start-to-close. A heartbeat may include `details`; + the latest details are kept across retries and sent on the next dispatch. +4. The worker calls exactly one of `complete` or `fail`. `fail` may set + `nonRetryable` to stop the retry policy, or `retryAfterMs` to wait at least + that long before the next attempt (the policy backoff is never shortened). + A local handler does the same by failing with `failEffect`. 5. On cancellation, stop side effects and call `cancelled`. +Each callback response is `{ status: 'accepted' | 'stale' | 'cancelled', cancelRequested }`. +The host no longer returns the run state. `cancelRequested` is how a worker learns +that an attempt already in progress should stop: a queued dispatch cannot be recalled. +`stale` means the report was for another attempt, or the attempt had already settled, +and the host recorded nothing. + Callbacks include the same `effectId` and `attempt`. Stale callbacks are ignored. A dispatch transport failure is recorded as `ambiguous`: an operator must first verify the destination's idempotency record, then use `retryEffect` or diff --git a/packages/client/src/client.ts b/packages/client/src/client.ts index 413b98e..ae384f0 100644 --- a/packages/client/src/client.ts +++ b/packages/client/src/client.ts @@ -15,6 +15,7 @@ import { type JsonValue, type RunState, type WorkerCallbackInput, + type WorkerCallbackResult, } from '@looms/core' import { consumeSseStreamEffect, eventsFromSseData, sseFrames } from './sse' @@ -41,6 +42,53 @@ const StartResultSchema = Schema.Struct({ const RunResultSchema = Schema.Struct({ runId: Schema.String, state: RunStateSchema }) +interface WorkerFailRequestBody { + error: string + nonRetryable?: boolean + retryAfterMs?: number +} + +function workerCallbackBody(input: WorkerCallbackInput) { + switch (input.kind) { + case 'complete': + return { events: input.events } + + case 'fail': { + const body: WorkerFailRequestBody = { error: input.error } + + if (input.nonRetryable !== undefined) { + body.nonRetryable = input.nonRetryable + } + + if (input.retryAfterMs !== undefined) { + body.retryAfterMs = input.retryAfterMs + } + + return body + } + + case 'heartbeat': + return input.details === undefined ? {} : { details: input.details } + case 'started': + case 'cancelled': + return {} + + default: { + const exhaustive: never = input + return exhaustive + } + } +} + +const WorkerCallbackResultSchema = Schema.Struct({ + status: Schema.Union([ + Schema.Literal('accepted'), + Schema.Literal('stale'), + Schema.Literal('cancelled'), + ]), + cancelRequested: Schema.Boolean, +}) + /** `appended` counts the events the run gained while the wake ran. */ const WakeResultSchema = Schema.Struct({ runId: Schema.String, @@ -181,17 +229,15 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { runId?: string }) => request('/runs', StartResultSchema, { method: 'POST', body: stringifyJson(args) }) - const workerCallback = (runId: string, input: WorkerCallbackInput) => { - const body = - input.kind === 'complete' - ? { events: input.events } - : input.kind === 'fail' - ? { error: input.error } - : {} + const workerCallback = ( + runId: string, + input: WorkerCallbackInput, + ): Promise => { + const body = workerCallbackBody(input) return request( `/runs/${encodeURIComponent(runId)}/effects/${encodeURIComponent(input.effectId)}/${input.attempt}/${input.kind}`, - RunResultSchema, + WorkerCallbackResultSchema, { method: 'POST', headers: options.workerCallbackToken diff --git a/packages/core/src/effects.ts b/packages/core/src/effects.ts index 57035c6..40f3c88 100644 --- a/packages/core/src/effects.ts +++ b/packages/core/src/effects.ts @@ -215,6 +215,52 @@ export class EffectHandlerError extends Data.TaggedError('EffectHandlerError')<{ } } +export interface EffectFailureOptions { + readonly error: string + /** Fail the attempt even when the retry policy has attempts remaining. */ + readonly nonRetryable?: boolean + /** Wait at least this long before the next attempt. Does not shorten the policy backoff. */ + readonly retryAfterMs?: number +} + +/** A handler's deliberate attempt failure, distinct from an unexpected thrown error. */ +export class EffectFailure extends Data.TaggedError('EffectFailure')<{ + readonly error: string + readonly nonRetryable: boolean + readonly retryAfterMs?: number +}> { + constructor(options: EffectFailureOptions) { + const error = options.error + const nonRetryable = options.nonRetryable === true + + if (options.retryAfterMs === undefined) { + super({ error, nonRetryable }) + } else { + super({ error, nonRetryable, retryAfterMs: options.retryAfterMs }) + } + + this.name = 'EffectFailure' + } +} + +/** Fail the current attempt. The retry policy decides whether another attempt is scheduled. */ +export function failEffect(options: EffectFailureOptions): Effect.Effect { + return Effect.fail(new EffectFailure(options)) +} + +/** Unwrap a handler failure, including one wrapped by `EffectHandlerError`. */ +export function readEffectFailure(cause: unknown): EffectFailure | null { + if (cause instanceof EffectFailure) { + return cause + } + + if (cause instanceof EffectHandlerError) { + return readEffectFailure(cause.cause) + } + + return null +} + export type EffectHandlerResult< R = never, E extends EventInput = EventInput, @@ -266,6 +312,18 @@ function liftHandlerResult( return Effect.succeed(result) } +function invokeHandler( + handler: (input: JsonValue, ctx: EffectContext) => EffectHandlerResult, + input: JsonValue, + ctx: EffectContext, +): Effect.Effect, EffectHandlerError, R> { + try { + return liftHandlerResult(handler(input, ctx)) + } catch (cause) { + return Effect.fail(new EffectHandlerError(cause)) + } +} + /** * Define an effect. With `execute`, a run's actor cell runs the handler at * most once while it is alive; if the cell dies mid-execution, the replacement @@ -327,7 +385,12 @@ export function defineEffect(def: { Effect.gen(function* () { const input = schema ? yield* validateInputEffect(schema, raw) : raw - return handler ? yield* liftHandlerResult(handler(input, ctx)) : noOutput + if (!handler) { + return noOutput + } + + // invokeHandler turns a synchronous throw into EffectHandlerError. + return yield* invokeHandler(handler, input, ctx) }), } } diff --git a/packages/core/src/fold.ts b/packages/core/src/fold.ts index 535df71..9f5d9d9 100644 --- a/packages/core/src/fold.ts +++ b/packages/core/src/fold.ts @@ -35,6 +35,28 @@ function readString(obj: { [key: string]: JsonValue }, key: string): string | un return Predicate.isString(value) ? value : undefined } +/** Heartbeat progress already recorded for this effect, if any. */ +function priorHeartbeatDetails(state: RunState, effectId: string): JsonValue | null { + return state.effectExecutions[effectId]?.heartbeatDetails ?? null +} + +/** + * A heartbeat that carries `details` replaces the saved progress. Every other execution event + * keeps it, including the retry that starts the next attempt. + */ +function nextHeartbeatDetails( + state: RunState, + effectId: string, + event: EventEnvelope, + payload: { [key: string]: JsonValue }, +): JsonValue | null { + if (event.type === 'runtime.effect.heartbeat' && Object.hasOwn(payload, 'details')) { + return payload.details ?? null + } + + return priorHeartbeatDetails(state, effectId) +} + function cloneState(state: RunState): RunState { return { runId: state.runId, @@ -290,6 +312,7 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi deadlineAt: null, startedAt: event.ts, lastHeartbeatAt: null, + heartbeatDetails: priorHeartbeatDetails(state, effectId), lastError: null, }, }, @@ -352,6 +375,7 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi : sameAttempt ? previous.lastHeartbeatAt : null, + heartbeatDetails: nextHeartbeatDetails(state, effectId, event, payload), lastError: readString(payload, 'error') ?? null, }, }, @@ -384,6 +408,7 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi deadlineAt: nextAttemptAt, startedAt: null, lastHeartbeatAt: null, + heartbeatDetails: priorHeartbeatDetails(state, effectId), lastError: readString(payload, 'error') ?? 'effect failed', }, }, diff --git a/packages/core/src/protocol.ts b/packages/core/src/protocol.ts index 527ceaf..2ca4b74 100644 --- a/packages/core/src/protocol.ts +++ b/packages/core/src/protocol.ts @@ -94,6 +94,8 @@ export const protocolCatalog = defineEventCatalog('runtime', { 'effect.failed': Schema.Struct({ effectId: Schema.String, error: Schema.String, + nonRetryable: Schema.optional(Schema.Boolean), + retryAfterMs: Schema.optional(Schema.Finite), }), 'effect.attempt.started': Schema.Struct({ effectId: Schema.String, @@ -119,6 +121,8 @@ export const protocolCatalog = defineEventCatalog('runtime', { attempt: Schema.Finite, heartbeatAt: Schema.Finite, deadlineAt: Schema.NullOr(Schema.Finite), + /** Progress the next attempt should resume from. Absent on heartbeats that only extend the deadline. */ + details: Schema.optional(JsonValueSchema), }), 'effect.cancel.requested': Schema.Struct({ effectId: Schema.String, diff --git a/packages/core/src/state.ts b/packages/core/src/state.ts index 58d370b..94115ae 100644 --- a/packages/core/src/state.ts +++ b/packages/core/src/state.ts @@ -58,6 +58,8 @@ export interface EffectExecutionRecord { /** When the current attempt started executing, locally or on a worker. */ startedAt: number | null lastHeartbeatAt: number | null + /** Progress reported by the latest heartbeat. Survives retries so the next attempt can resume. */ + heartbeatDetails: JsonValue | null lastError: string | null } @@ -189,6 +191,9 @@ export const RunStateSchema = Schema.Struct({ lastHeartbeatAt: Schema.NullOr(Schema.Finite).pipe( Schema.withDecodingDefaultKey(Effect.succeed(null)), ), + heartbeatDetails: Schema.NullOr(JsonValueSchema).pipe( + Schema.withDecodingDefaultKey(Effect.succeed(null)), + ), lastError: Schema.NullOr(Schema.String), }), ).pipe(Schema.withDecodingDefaultKey(Effect.succeed({}))), diff --git a/packages/core/src/worker-callback.ts b/packages/core/src/worker-callback.ts index f93722c..2badf51 100644 --- a/packages/core/src/worker-callback.ts +++ b/packages/core/src/worker-callback.ts @@ -1,4 +1,5 @@ import type { EventInput } from './envelope' +import type { JsonValue } from './types' /** Identifies one attempt of an effect executed by an external worker. */ export interface WorkerCallback { @@ -8,8 +9,25 @@ export interface WorkerCallback { /** What an external worker reports about an effect attempt. */ export type WorkerCallbackInput = - | (WorkerCallback & { readonly kind: 'started' | 'heartbeat' | 'cancelled' }) + | (WorkerCallback & { readonly kind: 'started' | 'cancelled' }) + | (WorkerCallback & { readonly kind: 'heartbeat'; readonly details?: JsonValue }) | (WorkerCallback & { readonly kind: 'complete'; readonly events: ReadonlyArray }) - | (WorkerCallback & { readonly kind: 'fail'; readonly error: string }) + | (WorkerCallback & { + readonly kind: 'fail' + readonly error: string + /** When set, the attempt is terminal even if the retry policy has attempts left. */ + readonly nonRetryable?: boolean + /** Extra delay before the next attempt. The policy backoff is never shortened. */ + readonly retryAfterMs?: number + }) export type WorkerCallbackKind = WorkerCallbackInput['kind'] + +/** + * What the host tells the worker about the callback it just sent. + * A queued message cannot be recalled, so `cancelRequested` is how an in-flight attempt learns to stop. + */ +export interface WorkerCallbackResult { + readonly status: 'accepted' | 'stale' | 'cancelled' + readonly cancelRequested: boolean +} diff --git a/packages/runtime/src/dispatch-effect.ts b/packages/runtime/src/dispatch-effect.ts index 93a533d..db43c8f 100644 --- a/packages/runtime/src/dispatch-effect.ts +++ b/packages/runtime/src/dispatch-effect.ts @@ -1,10 +1,11 @@ import type { Context, ManagedRuntime } from 'effect' -import { Effect } from 'effect' +import { Effect, Predicate } from 'effect' import { asJson, isPrimitiveEffect, isWaitOnTimer, + readEffectFailure, type ComposedRegistry, type EffectContext, type EventInput, @@ -16,6 +17,50 @@ import { import { threadStartedEvents } from './helpers' +function causeMessage(cause: unknown): string { + if (Predicate.isString(cause)) { + return cause + } + + if (cause instanceof Error) { + return cause.message + } + + return String(cause) +} + +function failedAttempt(effectId: string, threadId: string, cause: unknown): EventInput { + const failure = readEffectFailure(cause) + const error = failure?.error ?? causeMessage(cause) + const retryAfterMs = failure?.retryAfterMs + + if (failure?.nonRetryable && retryAfterMs !== undefined) { + return { + type: 'runtime.effect.failed', + payload: { effectId, error, nonRetryable: true, retryAfterMs }, + threadId, + } + } + + if (failure?.nonRetryable) { + return { + type: 'runtime.effect.failed', + payload: { effectId, error, nonRetryable: true }, + threadId, + } + } + + if (retryAfterMs !== undefined) { + return { + type: 'runtime.effect.failed', + payload: { effectId, error, retryAfterMs }, + threadId, + } + } + + return { type: 'runtime.effect.failed', payload: { effectId, error }, threadId } +} + export function dispatchEffect( registry: ComposedRegistry, services: ManagedRuntime.ManagedRuntime, @@ -110,11 +155,7 @@ export function dispatchEffect( if (!handler) { return Effect.succeed([ - { - type: 'runtime.effect.failed', - payload: { effectId: effCtx.effectId, error: `No handler for ${eff.type}` }, - threadId: currentThreadId, - }, + failedAttempt(effCtx.effectId, currentThreadId, `No handler for ${eff.type}`), ]) } @@ -129,18 +170,7 @@ export function dispatchEffect( }) return execution.pipe( - Effect.catch((err) => - Effect.succeed([ - { - type: 'runtime.effect.failed', - payload: { - effectId: effCtx.effectId, - error: err instanceof Error ? err.message : String(err), - }, - threadId: currentThreadId, - }, - ]), - ), + Effect.catch((err) => Effect.succeed([failedAttempt(effCtx.effectId, currentThreadId, err)])), ) } diff --git a/packages/runtime/src/helpers.ts b/packages/runtime/src/helpers.ts index c3d4961..ae05828 100644 --- a/packages/runtime/src/helpers.ts +++ b/packages/runtime/src/helpers.ts @@ -35,14 +35,21 @@ export function workerTask( runId: string, item: OutstandingEffect, attempt: number, + details?: JsonValue | null, ): EffectWorkerTask { - return { + const task = { runId, effectId: item.effectId, attempt, type: item.effect.type, input: 'input' in item.effect ? item.effect.input : null, } + + if (details != null) { + return { ...task, details } + } + + return task } export class UnknownDefinitionError extends Data.TaggedError('UnknownDefinitionError')<{ diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index 1f041f8..aca9628 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -20,6 +20,7 @@ export { type WakeScheduler, type WorkerCallback, type WorkerCallbackInput, + type WorkerCallbackResult, } from './runtime' export { createLooms, type Looms, type CreateLoomsOptions, type StartResult } from './looms' export { diff --git a/packages/runtime/src/local-effect-executor.ts b/packages/runtime/src/local-effect-executor.ts index e611713..dedf602 100644 --- a/packages/runtime/src/local-effect-executor.ts +++ b/packages/runtime/src/local-effect-executor.ts @@ -273,6 +273,7 @@ export function executeLocalEffects( const attempt = input.attempts.get(item.effectId) ?? 1 const payload = Predicate.isObject(outcome.payload) ? outcome.payload : {} const error = Predicate.isString(payload.error) ? payload.error : 'effect failed' + const retryAfterMs = payload.retryAfterMs const failure = planAttemptFailure({ runId: input.runId, @@ -282,6 +283,8 @@ export function executeLocalEffects( now: input.now, retry, origin: { type: 'system' }, + nonRetryable: payload.nonRetryable === true, + retryAfterMs: Predicate.isNumber(retryAfterMs) ? retryAfterMs : undefined, }) if (failure.retrying) { diff --git a/packages/runtime/src/looms.ts b/packages/runtime/src/looms.ts index a1fc06d..74922d6 100644 --- a/packages/runtime/src/looms.ts +++ b/packages/runtime/src/looms.ts @@ -29,6 +29,7 @@ import { type RunOperationalStatus, type WakeScheduler, type WorkerCallbackInput, + type WorkerCallbackResult, } from './runtime' import { createFetchHandler, @@ -135,7 +136,7 @@ export interface Looms { cancel(runId: string, threadId?: string): Promise /** Start a new run that continues where a finished run ended. */ fork(sourceRunId: string, options?: { runId?: string }): Promise - workerCallback(runId: string, input: WorkerCallbackInput): Promise + workerCallback(runId: string, input: WorkerCallbackInput): Promise fetch(req: Request): Promise serve(options?: { port?: number; hostname?: string }): RunningServer stop(): Promise diff --git a/packages/runtime/src/planners.test.ts b/packages/runtime/src/planners.test.ts index 53eefd5..1d667aa 100644 --- a/packages/runtime/src/planners.test.ts +++ b/packages/runtime/src/planners.test.ts @@ -63,6 +63,7 @@ describe('deadline transition planner', () => { nextAttemptAt: null, deadlineAt: 1_001, lastHeartbeatAt: null, + heartbeatDetails: null, lastError: null, }, }, @@ -95,6 +96,7 @@ describe('deadline transition planner', () => { nextAttemptAt: null, deadlineAt: 900, lastHeartbeatAt: null, + heartbeatDetails: null, lastError: null, }, future: { @@ -104,6 +106,7 @@ describe('deadline transition planner', () => { nextAttemptAt: null, deadlineAt: 1_200, lastHeartbeatAt: null, + heartbeatDetails: null, lastError: null, }, }, @@ -143,6 +146,7 @@ describe('deadline transition planner', () => { deadlineAt: 9_000, startedAt: null, lastHeartbeatAt: null, + heartbeatDetails: null, lastError: 'failed', }, }, @@ -186,6 +190,7 @@ describe('worker effect planner', () => { nextAttemptAt: 1_000, deadlineAt: null, lastHeartbeatAt: null, + heartbeatDetails: null, lastError: 'temporary', }, }, @@ -249,6 +254,7 @@ describe('worker effect routing', () => { deadlineAt: null, startedAt: null, lastHeartbeatAt: null, + heartbeatDetails: null, lastError: null, } : undefined, diff --git a/packages/runtime/src/retry-policy.ts b/packages/runtime/src/retry-policy.ts index eadfe3e..2178dc3 100644 --- a/packages/runtime/src/retry-policy.ts +++ b/packages/runtime/src/retry-policy.ts @@ -18,6 +18,17 @@ export function retryBackoff(retry: RetryPolicy, attempt: number): number { ) } +/** Policy backoff, extended when the failed attempt asked to wait longer. */ +export function retryDelay(retry: RetryPolicy, attempt: number, retryAfterMs?: number): number { + const backoff = retryBackoff(retry, attempt) + + if (retryAfterMs === undefined || !Number.isFinite(retryAfterMs) || retryAfterMs < 0) { + return backoff + } + + return Math.max(backoff, retryAfterMs) +} + export interface AttemptFailurePlan { readonly event: EventEnvelope readonly retrying: boolean @@ -32,10 +43,13 @@ export function planAttemptFailure(input: { readonly now: number readonly retry: RetryPolicy | undefined readonly origin: EventOrigin + /** Customer/domain failures must not be retried even when attempts remain. */ + readonly nonRetryable?: boolean + readonly retryAfterMs?: number }): AttemptFailurePlan { const { runId, item, attempt, error, retry } = input - if (!retry || attempt >= retry.maxAttempts) { + if (input.nonRetryable || !retry || attempt >= retry.maxAttempts) { return { event: createEffectFailedEvent(runId, item, error), retrying: false } } @@ -46,7 +60,7 @@ export function planAttemptFailure(input: { payload: { effectId: item.effectId, attempt, - nextAttemptAt: input.now + retryBackoff(retry, attempt), + nextAttemptAt: input.now + retryDelay(retry, attempt, input.retryAfterMs), error, }, threadId: item.threadId, diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index ac0b33b..4fd29df 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -79,7 +79,7 @@ export type { StartRunArgs, WakeError, } from './types' -export type { WorkerCallback, WorkerCallbackInput } from '@looms/core' +export type { WorkerCallback, WorkerCallbackInput, WorkerCallbackResult } from '@looms/core' import type { CreateRuntimeOptions, LoomsRuntime, StartRunArgs } from './types' function currentTimeMillis(): number { diff --git a/packages/runtime/src/server/routes/effects.ts b/packages/runtime/src/server/routes/effects.ts index a63c6e5..402f65b 100644 --- a/packages/runtime/src/server/routes/effects.ts +++ b/packages/runtime/src/server/routes/effects.ts @@ -1,6 +1,11 @@ import { Effect, Schema } from 'effect' -import type { WorkerCallbackInput, WorkerCallbackKind } from '@looms/core' +import { + JsonValueSchema, + type JsonValue, + type WorkerCallbackInput, + type WorkerCallbackKind, +} from '@looms/core' import { EventInputBodySchema, provideStore, readJson, type RouteOptions } from '../http' import { integer, oneOf, route, type Route } from '../route-table' @@ -13,9 +18,28 @@ const WORKER_CALLBACK_KINDS = [ 'cancelled', ] as const satisfies readonly WorkerCallbackKind[] +interface FailWorkerCallback { + effectId: string + attempt: number + kind: 'fail' + error: string + nonRetryable?: boolean + retryAfterMs?: number +} + +interface HeartbeatWorkerCallback { + effectId: string + attempt: number + kind: 'heartbeat' + details?: JsonValue +} + const WorkerCallbackBodySchema = Schema.Struct({ events: Schema.optional(Schema.Array(EventInputBodySchema)), error: Schema.optional(Schema.String), + nonRetryable: Schema.optional(Schema.Boolean), + retryAfterMs: Schema.optional(Schema.Finite), + details: Schema.optional(JsonValueSchema), }) function workerCallbackInput( @@ -31,10 +55,47 @@ function workerCallbackInput( switch (params.action) { case 'complete': return { ...callback, kind: 'complete', events: body.events ?? [] } - case 'fail': - return body.error ? { ...callback, kind: 'fail', error: body.error } : null - default: + + case 'fail': { + if (body.error === undefined) { + return null + } + + const failure: FailWorkerCallback = { + ...callback, + kind: 'fail', + error: body.error, + } + + if (body.nonRetryable !== undefined) { + failure.nonRetryable = body.nonRetryable + } + + if (body.retryAfterMs !== undefined) { + failure.retryAfterMs = body.retryAfterMs + } + + return failure + } + + case 'heartbeat': { + const heartbeat: HeartbeatWorkerCallback = { ...callback, kind: 'heartbeat' } + + if (body.details !== undefined) { + heartbeat.details = body.details + } + + return heartbeat + } + + case 'started': + case 'cancelled': return { ...callback, kind: params.action } + + default: { + const exhaustive: never = params.action + return exhaustive + } } } @@ -80,8 +141,8 @@ export function effectRoutes({ runtime, store }: RouteOptions): readonly Route[] return Response.json({ error: 'error required' }, { status: 400 }) } - const state = yield* provideStore(runtime.workerCallback(params.runId, input), store) - return Response.json({ runId: params.runId, state }) + const result = yield* provideStore(runtime.workerCallback(params.runId, input), store) + return Response.json(result) }), }), ] diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 850756b..0654bb9 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -6,6 +6,7 @@ import type { EventEnvelope, EventInput, WorkerCallbackInput, + WorkerCallbackResult, EventStore, EventStoreAppendError, EventStoreError, @@ -92,6 +93,8 @@ export interface EffectWorkerTask { readonly attempt: number readonly type: string readonly input: JsonValue + /** Progress from the previous attempt's last heartbeat, when it reported any. */ + readonly details?: JsonValue } export interface EffectWorker { @@ -203,11 +206,18 @@ export interface LoomsRuntime< sourceRunId: string, options?: { runId?: string }, ): Effect.Effect - /** Applies a worker's report about one effect attempt; stale or foreign reports are ignored. */ + /** + * Applies a worker's report about one effect attempt. A report for another attempt, or one that + * arrives after the attempt settled, is `stale` and changes nothing. + */ workerCallback( runId: string, input: WorkerCallbackInput, - ): Effect.Effect + ): Effect.Effect< + WorkerCallbackResult, + Error | EventStoreAppendError | EventStoreError, + EventStoreTag + > readonly listRuns: Effect.Effect /** Indexed run rows. Folds a run only when its header has not been written yet. */ readonly listRunSummaries: Effect.Effect< diff --git a/packages/runtime/src/worker-callbacks.ts b/packages/runtime/src/worker-callbacks.ts index b63c9a0..882618e 100644 --- a/packages/runtime/src/worker-callbacks.ts +++ b/packages/runtime/src/worker-callbacks.ts @@ -10,6 +10,7 @@ import { type RetryPolicy, type RunState, type WorkerCallbackInput, + type WorkerCallbackResult, } from '@looms/core' import { materializeEffectOutcome } from './helpers' @@ -61,9 +62,28 @@ export interface WorkerCallbacksOptions { interface WorkerCallbackPlan { readonly events: ReadonlyArray readonly notices: ReadonlyArray + readonly result: WorkerCallbackResult } -const IGNORED: WorkerCallbackPlan = { events: [], notices: [] } +const STALE: WorkerCallbackPlan = { + events: [], + notices: [], + result: { status: 'stale', cancelRequested: false }, +} + +const CANCELLED: WorkerCallbackPlan = { + events: [], + notices: [], + result: { status: 'cancelled', cancelRequested: true }, +} + +function accepted( + events: ReadonlyArray, + notices: ReadonlyArray = [], + cancelRequested = false, +): WorkerCallbackPlan { + return { events, notices, result: { status: 'accepted', cancelRequested } } +} /** * Turn a worker callback into events against the current durable state. Callbacks for another @@ -88,6 +108,15 @@ function planWorkerCallback(input: { execution?.attempt === callback.attempt && execution.status === 'cancel_requested' + // A message already handed to a worker cannot be pulled back. Tell it to stop instead. + if ( + execution?.attempt === callback.attempt && + execution.status === 'cancel_requested' && + callback.kind !== 'cancelled' + ) { + return Effect.succeed(CANCELLED) + } + if ( !execution || execution.attempt !== callback.attempt || @@ -97,10 +126,9 @@ function planWorkerCallback(input: { execution.status === 'retry_wait' || execution.status === 'timed_out' || execution.status === 'ambiguous' || - (callback.kind === 'cancelled' && !isCancellationAcknowledgement) || - (execution.status === 'cancel_requested' && callback.kind !== 'cancelled') + (callback.kind === 'cancelled' && !isCancellationAcknowledgement) ) { - return Effect.succeed(IGNORED) + return Effect.succeed(STALE) } const retry = item ? registry.effects.get(item.effect.type)?.retry : undefined @@ -117,8 +145,8 @@ function planWorkerCallback(input: { case 'started': { return Effect.succeed( item && execution.status === 'dispatched' - ? { events: [workerStartedEvent(runId, item, callback.attempt, retry, now)], notices: [] } - : IGNORED, + ? accepted([workerStartedEvent(runId, item, callback.attempt, retry, now)]) + : STALE, ) } @@ -129,38 +157,47 @@ function planWorkerCallback(input: { !item || (!implicitStart && execution.status !== 'started' && execution.status !== 'heartbeat') ) { - return Effect.succeed(IGNORED) + return Effect.succeed(STALE) } - return Effect.succeed({ - events: [ - ...(implicitStart ? [workerStartedEvent(runId, item, callback.attempt, retry, now)] : []), - createEvent(runId, { - type: 'runtime.effect.heartbeat', - payload: { - effectId: item.effectId, - attempt: callback.attempt, - heartbeatAt: now, - deadlineAt: workerDeadline(retry, execution.startedAt ?? now, now), - }, - threadId: item.threadId, - origin: WORKER_ORIGIN, - }), - ], - notices: [notice('effect.heartbeat')], - }) + const recorded = { + effectId: item.effectId, + attempt: callback.attempt, + heartbeatAt: now, + deadlineAt: workerDeadline(retry, execution.startedAt ?? now, now), + } + + const payload = + callback.details === undefined ? recorded : { ...recorded, details: callback.details } + + return Effect.succeed( + accepted( + [ + ...(implicitStart + ? [workerStartedEvent(runId, item, callback.attempt, retry, now)] + : []), + createEvent(runId, { + type: 'runtime.effect.heartbeat', + payload, + threadId: item.threadId, + origin: WORKER_ORIGIN, + }), + ], + [notice('effect.heartbeat')], + ), + ) } case 'complete': { if (!item) { - return Effect.succeed(IGNORED) + return Effect.succeed(STALE) } return Effect.forEach(callback.events, (event) => materializeEffectOutcome(registry.catalogs, runId, item, event), ).pipe( - Effect.map((outcomes) => ({ - events: [ + Effect.map((outcomes) => + accepted([ ...outcomes, createEvent(runId, { type: 'runtime.effect.completed', @@ -170,15 +207,14 @@ function planWorkerCallback(input: { causationId: item.causingEventId, origin: WORKER_ORIGIN, }), - ], - notices: [], - })), + ]), + ), ) } case 'fail': { if (!item) { - return Effect.succeed(IGNORED) + return Effect.succeed(STALE) } const failure = planAttemptFailure({ @@ -189,38 +225,45 @@ function planWorkerCallback(input: { now, retry, origin: WORKER_ORIGIN, + nonRetryable: callback.nonRetryable, + retryAfterMs: callback.retryAfterMs, }) - return Effect.succeed({ - events: [failure.event], - notices: failure.retrying - ? [ - { - type: 'effect.retry', - runId, - at: now, - effectId: item.effectId, - attempt: callback.attempt, - detail: callback.error, - }, - ] - : [], - }) + return Effect.succeed( + accepted( + [failure.event], + failure.retrying + ? [ + { + type: 'effect.retry', + runId, + at: now, + effectId: item.effectId, + attempt: callback.attempt, + detail: callback.error, + }, + ] + : [], + ), + ) } case 'cancelled': { - return Effect.succeed({ - events: [ - createEvent(runId, { - type: 'runtime.effect.cancelled', - payload: { effectId: callback.effectId, attempt: callback.attempt }, - threadId: item?.threadId ?? null, - effectId: callback.effectId, - origin: WORKER_ORIGIN, - }), - ], - notices: [notice('effect.cancellation')], - }) + return Effect.succeed( + accepted( + [ + createEvent(runId, { + type: 'runtime.effect.cancelled', + payload: { effectId: callback.effectId, attempt: callback.attempt }, + threadId: item?.threadId ?? null, + effectId: callback.effectId, + origin: WORKER_ORIGIN, + }), + ], + [notice('effect.cancellation')], + true, + ), + ) } default: { @@ -242,19 +285,22 @@ export function createWorkerCallbacks(options: WorkerCallbacksOptions) { const committed = yield* repository.commit(store, runId, (cursor) => planWorkerCallback({ runId, state: cursor.state, callback, registry, now }).pipe( - Effect.map((plan) => ({ events: plan.events, value: plan.notices })), + Effect.map((plan) => ({ + events: plan.events, + value: { notices: plan.notices, result: plan.result }, + })), ), ) if (committed.append) { - for (const notice of committed.value) { + for (const notice of committed.value.notices) { notifyObserver(options.observer, notice) } } - return committed.cursor.state + return committed.value.result }), - ).pipe(Effect.andThen(options.wake(runId))) + ).pipe(Effect.tap((result) => (result.status === 'stale' ? Effect.void : options.wake(runId)))) return { workerCallback: applyWorkerCallback } } diff --git a/packages/runtime/src/worker-effect-dispatch.ts b/packages/runtime/src/worker-effect-dispatch.ts index 478a88d..6501d1d 100644 --- a/packages/runtime/src/worker-effect-dispatch.ts +++ b/packages/runtime/src/worker-effect-dispatch.ts @@ -147,7 +147,7 @@ export function planWorkerEffects(input: { causationId: item.causingEventId, origin: { type: 'system' }, }), - task: workerTask(input.runId, item, attempt), + task: workerTask(input.runId, item, attempt, existing?.heartbeatDetails), scheduleToStartDeadlineAt: retry?.scheduleToStartTimeoutMs ? input.now + retry.scheduleToStartTimeoutMs : null, diff --git a/packages/runtime/src/worker-protocol.test.ts b/packages/runtime/src/worker-protocol.test.ts index fe4f539..3f3f8dc 100644 --- a/packages/runtime/src/worker-protocol.test.ts +++ b/packages/runtime/src/worker-protocol.test.ts @@ -7,6 +7,7 @@ import { defineEffect, defineRuntimeModule, defineThread, + failEffect, invoke, makeMemoryEventStore, type EventEnvelope, @@ -22,7 +23,10 @@ const workerDefinition = { value: { kind: 'worker-test', name: 'worker-test' }, } -function workerModule(retry: RetryPolicy, execute?: () => ReadonlyArray) { +function workerModule( + retry: RetryPolicy, + execute?: () => ReadonlyArray | Effect.Effect, unknown>, +) { const effect = defineEffect({ type: 'worker.do', retry, @@ -418,7 +422,9 @@ describe('durable external effect workers', () => { expect(unauthorized?.status).toBe(401) expect(unknownAction?.status).toBe(404) expect(startedResponse?.status).toBe(200) + expect(await startedResponse?.json()).toEqual({ status: 'accepted', cancelRequested: false }) expect(completedResponse?.status).toBe(200) + expect(await completedResponse?.json()).toEqual({ status: 'accepted', cancelRequested: false }) expect( (await looms.getEvents(started.runId)).some((event) => event.type === 'worker.done'), @@ -703,4 +709,114 @@ describe('durable external effect workers', () => { expect((await looms.getRun(started.runId)).status).toBe('cancelled') await looms.stop() }) + + test('a non-retryable failure spends no further attempts', async () => { + const looms = createLooms({ + modules: [ + workerModule({ maxAttempts: 3, backoffMs: 1 }, () => + failEffect({ error: 'bad input', nonRetryable: true }), + ), + ], + }) + + const started = await looms.start(workerDefinition, {}) + const types = (await looms.getEvents(started.runId)).map((event) => event.type) + + expect(types).toContain('runtime.effect.failed') + expect(types).not.toContain('runtime.effect.retry.scheduled') + expect((await looms.getRun(started.runId)).status).toBe('failed') + await looms.stop() + }) + + test('retryAfterMs extends the policy backoff', async () => { + const looms = createLooms({ + modules: [ + workerModule({ maxAttempts: 3, backoffMs: 10 }, () => + failEffect({ error: 'later', retryAfterMs: 60_000 }), + ), + ], + }) + + const started = await looms.start(workerDefinition, {}) + + const scheduled = (await looms.getEvents(started.runId)).find( + (event) => event.type === 'runtime.effect.retry.scheduled', + ) + + const payload = scheduled?.payload + + expect(scheduled).toBeDefined() + + if (payload && !Array.isArray(payload) && Predicate.isReadonlyObject(payload)) { + expect(payload.nextAttemptAt).toBeGreaterThanOrEqual((scheduled?.ts ?? 0) + 60_000) + } + + await looms.stop() + }) + + test('heartbeat details resume on the next worker attempt', async () => { + const dispatched: EffectWorkerTask[] = [] + + const looms = createLooms({ + modules: [workerModule({ maxAttempts: 3, backoffMs: 5 })], + worker: { dispatch: (task) => dispatched.push(task) }, + }) + + const started = await looms.start(workerDefinition, {}) + const identity = effectIdentity(await looms.getEvents(started.runId)) + + const heartbeat = await looms.workerCallback(started.runId, { + ...identity, + kind: 'heartbeat', + details: { page: 2 }, + }) + + expect(heartbeat).toEqual({ status: 'accepted', cancelRequested: false }) + + const failed = await looms.workerCallback(started.runId, { + ...identity, + kind: 'fail', + error: 'transient', + }) + + expect(failed).toEqual({ status: 'accepted', cancelRequested: false }) + await Bun.sleep(40) + + expect(dispatched.at(-1)?.attempt).toBe(2) + expect(dispatched.at(-1)?.details).toEqual({ page: 2 }) + + const stale = await looms.workerCallback(started.runId, { + ...identity, + kind: 'heartbeat', + details: { page: 9 }, + }) + + expect(stale).toEqual({ status: 'stale', cancelRequested: false }) + await looms.stop() + }) + + test('a heartbeat during cancellation tells the worker to stop', async () => { + const looms = createLooms({ + modules: [workerModule({ maxAttempts: 1, cancellationTimeoutMs: 60_000 })], + worker: { dispatch: () => undefined, cancel: () => undefined }, + }) + + const started = await looms.start(workerDefinition, {}) + const identity = effectIdentity(await looms.getEvents(started.runId)) + await looms.workerCallback(started.runId, { ...identity, kind: 'started' }) + await looms.cancel(started.runId) + + const heartbeat = await looms.workerCallback(started.runId, { + ...identity, + kind: 'heartbeat', + details: { page: 1 }, + }) + + expect(heartbeat).toEqual({ status: 'cancelled', cancelRequested: true }) + + const events = await looms.getEvents(started.runId) + const heartbeats = events.filter((event) => event.type === 'runtime.effect.heartbeat') + expect(heartbeats).toHaveLength(0) + await looms.stop() + }) }) From 948d3f1d4bfc99b74cacbb3f000e405652311fdc Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 10:26:06 +0200 Subject: [PATCH 02/23] feat(runtime): keep run attributes public and secrets beside the log Attributes are immutable labels on the start event, run state, and summary, and authorization can see them. Secrets stay in a side store for handlers and in-process dispatch, and are kept out of events, snapshots, and HTTP responses. Co-authored-by: Cursor --- .changeset/run-context.md | 5 + docs/production.md | 19 ++ packages/client/src/client.ts | 4 + packages/core/src/effects.test.ts | 2 + packages/core/src/effects.ts | 7 + packages/core/src/fold.test.ts | 1 + packages/core/src/fold.ts | 4 + packages/core/src/index.ts | 1 + packages/core/src/launcher.ts | 3 + packages/core/src/projection.ts | 1 + packages/core/src/protocol.ts | 2 + packages/core/src/published.ts | 5 +- packages/core/src/run-context.ts | 54 ++++ packages/core/src/sqlite-store.ts | 82 ++++- packages/core/src/state.ts | 7 + packages/core/src/store.ts | 14 + packages/react/src/materialize.test.ts | 1 + packages/react/src/materialize.ts | 3 + packages/react/src/react.ts | 5 + packages/react/src/tables.ts | 3 +- packages/runtime/src/errors.ts | 17 ++ packages/runtime/src/helpers.ts | 11 +- packages/runtime/src/local-effect-executor.ts | 71 +++-- packages/runtime/src/looms.ts | 15 +- packages/runtime/src/planners.test.ts | 1 + packages/runtime/src/run-context.test.ts | 279 ++++++++++++++++++ packages/runtime/src/run-start.ts | 6 +- packages/runtime/src/runtime.ts | 91 +++++- packages/runtime/src/server.test.ts | 1 + packages/runtime/src/server/auth.ts | 4 + packages/runtime/src/server/fetch-handler.ts | 66 ++++- packages/runtime/src/server/routes/runs.ts | 6 + packages/runtime/src/types.ts | 14 + packages/runtime/src/wake-engine.ts | 4 + .../runtime/src/worker-effect-dispatch.ts | 4 +- 35 files changed, 764 insertions(+), 49 deletions(-) create mode 100644 .changeset/run-context.md create mode 100644 packages/core/src/run-context.ts create mode 100644 packages/runtime/src/run-context.test.ts diff --git a/.changeset/run-context.md b/.changeset/run-context.md new file mode 100644 index 0000000..b6bf62d --- /dev/null +++ b/.changeset/run-context.md @@ -0,0 +1,5 @@ +--- +'@swirls/looms': minor +--- + +Runs accept immutable `attributes` and a `secrets` context. Attributes are recorded on `runtime.run.started`, the run state, and the run summary, and are passed to `authorize` when present. Secrets stay in a side store: they are available to effect handlers and in-process worker dispatch, and they are not written to events, snapshots, or HTTP responses. diff --git a/docs/production.md b/docs/production.md index 866dee0..f21bcc8 100644 --- a/docs/production.md +++ b/docs/production.md @@ -17,6 +17,25 @@ All built-in stores reject an event over 1 MiB and a snapshot over 8 MiB, measured as serialized UTF-8, before storage is mutated. The stable error names are `EventPayloadTooLargeError` and `SnapshotPayloadTooLargeError`. +## Run attributes and secret context + +`attributes` on `start` are immutable labels (tenant, project, org). They are +written on `runtime.run.started`, folded into run state, and copied onto the +run summary. Authorization receives them on `route.attributes` when a run has +any. They must be safe for every reader of the run. + +`secrets` are bearer material for the run. The memory and SQLite stores keep +them in a side table, `looms_run_secrets`, which is not part of the event log +or a snapshot. Effect handlers read them on `ctx.secrets`. An in-process worker +receives them on the dispatch task. A transport that persists that task must +omit `secrets` and have the worker fetch them through an authenticated callback +for the current attempt. A handler must not copy secrets into an emitted event. + +A retry of the same start may omit `secrets` and keeps the stored context. Sending +a different context conflicts, and the error does not include the secret values. +Forks and effect-launched runs inherit the parent's attributes and secrets. +A store without `runSecrets` rejects a start that supplies secrets. + ## Worker effect protocol 1. The runtime dispatches `{ runId, effectId, attempt, type, input }`. diff --git a/packages/client/src/client.ts b/packages/client/src/client.ts index 413b98e..088ba16 100644 --- a/packages/client/src/client.ts +++ b/packages/client/src/client.ts @@ -13,7 +13,9 @@ import { type EventEnvelope, type EventInput, type JsonValue, + type RunAttributes, type RunState, + type SecretContext, type WorkerCallbackInput, } from '@looms/core' @@ -179,6 +181,8 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { definitionVersion?: string input?: JsonValue runId?: string + attributes?: RunAttributes + secrets?: SecretContext }) => request('/runs', StartResultSchema, { method: 'POST', body: stringifyJson(args) }) const workerCallback = (runId: string, input: WorkerCallbackInput) => { diff --git a/packages/core/src/effects.test.ts b/packages/core/src/effects.test.ts index a545e9d..8f075cf 100644 --- a/packages/core/src/effects.test.ts +++ b/packages/core/src/effects.test.ts @@ -10,6 +10,8 @@ const ctx: EffectContext = { runId: 'run_1', threadId: 'thr_1', causingEventId: 'evt_1', + attributes: {}, + secrets: {}, signal: new AbortController().signal, emit: async () => undefined, } diff --git a/packages/core/src/effects.ts b/packages/core/src/effects.ts index 57035c6..f9bc825 100644 --- a/packages/core/src/effects.ts +++ b/packages/core/src/effects.ts @@ -2,6 +2,7 @@ import { Data, Effect, Predicate, Schema, type Context } from 'effect' import { EventInputSchema, type EventInput } from './envelope' import { DEFAULT_DEFINITION_VERSION } from './module' +import type { RunAttributes, SecretContext } from './run-context' import { validateInputEffect, type InferDefinedSchema, @@ -187,6 +188,10 @@ export interface EffectContext { readonly threadId: string readonly causingEventId: string readonly signal: AbortSignal + /** Labels safe for every reader of the run. */ + readonly attributes: RunAttributes + /** Bearer material for this run. Never copy it into an emitted event. */ + readonly secrets: SecretContext emit(event: EventInput): Promise } @@ -196,6 +201,8 @@ export interface ScopedEffectContext { readonly threadId: string readonly causingEventId: string readonly signal: AbortSignal + readonly attributes: RunAttributes + readonly secrets: SecretContext emit(event: E): Promise } diff --git a/packages/core/src/fold.test.ts b/packages/core/src/fold.test.ts index 74b3654..697d952 100644 --- a/packages/core/src/fold.test.ts +++ b/packages/core/src/fold.test.ts @@ -97,6 +97,7 @@ describe('foldRun', () => { definitionVersion: 'v1', input: null, requestedThreadId: null, + attributes: {}, idempotencyKey: null, rootThreadId: threadId, launchedBy: null, diff --git a/packages/core/src/fold.ts b/packages/core/src/fold.ts index 535df71..09fa3f0 100644 --- a/packages/core/src/fold.ts +++ b/packages/core/src/fold.ts @@ -5,6 +5,7 @@ import type { EventEnvelope } from './envelope' import { createEffectId } from './ids' import { readRunLaunchSource } from './launcher' import { DEFAULT_DEFINITION_VERSION } from './module' +import { readAttributes } from './run-context' import type { ThreadRecord, OutstandingEffect, RunState, WaitRecord } from './state' import { completedRunStatus, emptyRunState } from './state' import { isTerminalStatus, type ThreadContext, type ThreadDefinition } from './thread' @@ -50,6 +51,7 @@ function cloneState(state: RunState): RunState { processedIdempotencyKeys: state.processedIdempotencyKeys ? [...state.processedIdempotencyKeys] : [], + attributes: state.attributes ?? {}, } } @@ -127,6 +129,7 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi rootThreadId, launchedBy: readRunLaunchSource(payload.launchedBy), correlationId: event.correlationId ?? null, + attributes: readAttributes(payload.attributes), } : null @@ -135,6 +138,7 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi runId: event.runId, rootThreadId: rootThreadId ?? null, startIdentity, + attributes: startIdentity?.attributes ?? {}, status: 'running', } } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c1ee11b..8107184 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -40,5 +40,6 @@ export * from './event-index' export * from './limits' export * from './definition-store' export * from './launcher' +export * from './run-context' export * from './published' export type { DebuggerHost, DebuggerRequest } from './debugger-host' diff --git a/packages/core/src/launcher.ts b/packages/core/src/launcher.ts index 70d7370..7667440 100644 --- a/packages/core/src/launcher.ts +++ b/packages/core/src/launcher.ts @@ -1,5 +1,6 @@ import { Context, Data, Effect, Option, Schema } from 'effect' +import type { RunAttributes, SecretContext } from './run-context' import type { JsonValue, RunStatus } from './types' /** The effect that launched a run, recorded on the launched run's `runtime.run.started`. */ @@ -36,6 +37,8 @@ export interface LaunchRunArgs { * correlation id, or to the launching run's id when that run has none. */ readonly correlationId?: string + readonly attributes?: RunAttributes + readonly secrets?: SecretContext } /** Enough of a run to decide whether a previous fire is still in progress. */ diff --git a/packages/core/src/projection.ts b/packages/core/src/projection.ts index e0fbf05..ed64487 100644 --- a/packages/core/src/projection.ts +++ b/packages/core/src/projection.ts @@ -296,6 +296,7 @@ export function toThreadTree(state: TreeBuildState): ThreadTree { rootThreadId: state.rootThreadId, startIdentity: null, forkedFrom: null, + attributes: {}, threads: state.records, waits: {}, outstandingEffects: [], diff --git a/packages/core/src/protocol.ts b/packages/core/src/protocol.ts index 527ceaf..5703874 100644 --- a/packages/core/src/protocol.ts +++ b/packages/core/src/protocol.ts @@ -5,6 +5,7 @@ import { WaitConditionSchema, type WaitCondition } from './effects' import { JsonValueSchema } from './envelope' import { RunLaunchSourceSchema } from './launcher' import { DEFAULT_DEFINITION_VERSION } from './module' +import { RunAttributesSchema } from './run-context' const DefinitionVersionSchema = Schema.String.pipe( Schema.withDecodingDefaultKey(Effect.succeed(DEFAULT_DEFINITION_VERSION)), @@ -19,6 +20,7 @@ export const protocolCatalog = defineEventCatalog('runtime', { input: JsonValueSchema, requestedThreadId: Schema.optional(Schema.NullOr(Schema.String)), launchedBy: Schema.optional(RunLaunchSourceSchema), + attributes: Schema.optional(RunAttributesSchema), }), 'run.completed': Schema.Struct({ output: Schema.NullOr(JsonValueSchema), diff --git a/packages/core/src/published.ts b/packages/core/src/published.ts index c9836ec..0e2e7f2 100644 --- a/packages/core/src/published.ts +++ b/packages/core/src/published.ts @@ -1,7 +1,8 @@ -import { Schema } from 'effect' +import { Effect, Schema } from 'effect' import { JsonValueSchema } from './envelope' import type { RegisteredDefinition } from './module' +import { RunAttributesSchema } from './run-context' import { jsonSchemaOfInput } from './schema' import { RunStateSchema, type RunState } from './state' @@ -40,6 +41,7 @@ export const RunSummarySchema = Schema.Struct({ status: RunStateSchema.fields.status, kind: Schema.NullOr(Schema.String), definitionName: Schema.NullOr(Schema.String), + attributes: RunAttributesSchema.pipe(Schema.withDecodingDefaultKey(Effect.succeed({}))), }) export type RunSummary = Schema.Schema.Type @@ -50,6 +52,7 @@ export function summarizeRun(state: RunState): RunSummary { status: state.status, kind: state.startIdentity?.kind ?? null, definitionName: state.startIdentity?.definitionName ?? null, + attributes: state.attributes ?? {}, } } diff --git a/packages/core/src/run-context.ts b/packages/core/src/run-context.ts new file mode 100644 index 0000000..6ab8fcd --- /dev/null +++ b/packages/core/src/run-context.ts @@ -0,0 +1,54 @@ +import { Option, Schema, type Effect } from 'effect' + +import type { JsonValue } from './types' + +/** + * Immutable labels for a run, such as tenant, project, or org. + * They are safe for every reader of the run: events, snapshots, summaries, and authorization. + */ +export type RunAttributes = { readonly [key: string]: string } + +/** + * Bearer material for a run, such as an attenuated macaroon. + * It is stored beside the log and is never written to an event, snapshot, or HTTP response. + */ +export type SecretContext = { readonly [key: string]: string } + +export const RunAttributesSchema = Schema.Record(Schema.String, Schema.String) + +export const SecretContextSchema = Schema.Record(Schema.String, Schema.String) + +const decodeAttributes = Schema.decodeUnknownOption(RunAttributesSchema) +const decodeSecretContext = Schema.decodeUnknownOption(SecretContextSchema) + +export interface RunSecretStore { + readonly get: (runId: string) => Effect.Effect + readonly put: (runId: string, secrets: SecretContext) => Effect.Effect +} + +export function hasEntries(record: { readonly [key: string]: string }): boolean { + return Object.keys(record).length > 0 +} + +export function sameStringRecord( + left: { readonly [key: string]: string }, + right: { readonly [key: string]: string }, +): boolean { + const leftKeys = Object.keys(left).toSorted() + const rightKeys = Object.keys(right).toSorted() + + if (leftKeys.length !== rightKeys.length) { + return false + } + + return leftKeys.every((key, index) => key === rightKeys[index] && left[key] === right[key]) +} + +/** Reads a string map from an event payload. Anything else becomes an empty map. */ +export function readAttributes(value: JsonValue | undefined): RunAttributes { + return value === undefined ? {} : Option.getOrElse(decodeAttributes(value), () => ({})) +} + +export function readSecretContext(value: JsonValue): SecretContext { + return Option.getOrElse(decodeSecretContext(value), () => ({})) +} diff --git a/packages/core/src/sqlite-store.ts b/packages/core/src/sqlite-store.ts index 2a6621b..f86b0e7 100644 --- a/packages/core/src/sqlite-store.ts +++ b/packages/core/src/sqlite-store.ts @@ -8,6 +8,7 @@ import { SnapshotPayloadTooLargeError, } from './limits' import type { RunSummary } from './published' +import { RunAttributesSchema, type RunSecretStore } from './run-context' import { SnapshotStoreError, withSnapshotStore, @@ -59,7 +60,13 @@ CREATE TABLE IF NOT EXISTS looms_run_headers ( run_id TEXT PRIMARY KEY, status TEXT NOT NULL, kind TEXT, - definition_name TEXT + definition_name TEXT, + attributes TEXT NOT NULL DEFAULT '{}' +); + +CREATE TABLE IF NOT EXISTS looms_run_secrets ( + run_id TEXT PRIMARY KEY, + body TEXT NOT NULL ); `.trim() @@ -178,6 +185,13 @@ function initSchema(exec: SqliteExec): void { exec.run(sql) } } + + // Databases created before attributes were indexed already have the headers table. + try { + exec.run(`ALTER TABLE looms_run_headers ADD COLUMN attributes TEXT NOT NULL DEFAULT '{}'`) + } catch { + // The column is already present. + } } function parseEvent(body: string): EventEnvelope { @@ -201,9 +215,23 @@ const HeaderRowSchema = Schema.Struct({ status: RunStateSchema.fields.status, kind: Schema.NullOr(Schema.String), definition_name: Schema.NullOr(Schema.String), + attributes: Schema.String, }) const decodeHeaderRow = Schema.decodeUnknownSync(HeaderRowSchema) +const encodeAttributes = Schema.encodeSync(Schema.fromJsonString(RunAttributesSchema)) + +const encodeSecrets = Schema.encodeSync( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)), +) + +const decodeAttributesColumn = Schema.decodeUnknownOption( + Schema.fromJsonString(RunAttributesSchema), +) + +const decodeSecretsColumn = Schema.decodeUnknownOption( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)), +) function headerFromRow(row: Schema.Schema.Type): RunSummary { return { @@ -211,6 +239,7 @@ function headerFromRow(row: Schema.Schema.Type): RunSumm status: row.status, kind: row.kind, definitionName: row.definition_name, + attributes: Option.getOrElse(decodeAttributesColumn(row.attributes), () => ({})), } } @@ -289,13 +318,20 @@ function createSqliteSnapshotStore(exec: SqliteExec): SnapshotStore { Effect.try({ try: () => { exec.run( - `INSERT INTO looms_run_headers (run_id, status, kind, definition_name) - VALUES (?, ?, ?, ?) + `INSERT INTO looms_run_headers (run_id, status, kind, definition_name, attributes) + VALUES (?, ?, ?, ?, ?) ON CONFLICT (run_id) DO UPDATE SET status = excluded.status, kind = excluded.kind, - definition_name = excluded.definition_name`, - [header.runId, header.status, header.kind, header.definitionName], + definition_name = excluded.definition_name, + attributes = excluded.attributes`, + [ + header.runId, + header.status, + header.kind, + header.definitionName, + encodeAttributes(header.attributes), + ], ) }, catch: (cause) => @@ -305,13 +341,46 @@ function createSqliteSnapshotStore(exec: SqliteExec): SnapshotStore { listHeaders: Effect.try({ try: () => exec - .rows('SELECT run_id, status, kind, definition_name FROM looms_run_headers') + .rows('SELECT run_id, status, kind, definition_name, attributes FROM looms_run_headers') .map((row) => headerFromRow(decodeHeaderRow(row))), catch: (cause) => new SnapshotStoreError('Failed to list run headers', cause), }), } } +const SecretRowSchema = Schema.Struct({ body: Schema.String }) +const decodeSecretRow = Schema.decodeUnknownSync(SecretRowSchema) + +function createSqliteSecrets(exec: SqliteExec): RunSecretStore { + return { + get: (runId) => + Effect.try({ + try: () => { + const [row] = exec.rows('SELECT body FROM looms_run_secrets WHERE run_id = ?', [runId]) + + if (row === undefined) { + return {} + } + + return Option.getOrElse(decodeSecretsColumn(decodeSecretRow(row).body), () => ({})) + }, + catch: (cause) => new EventStoreError(`Failed to read secret context for ${runId}`, cause), + }), + put: (runId, secrets) => + Effect.try({ + try: () => { + exec.run( + `INSERT INTO looms_run_secrets (run_id, body) + VALUES (?, ?) + ON CONFLICT (run_id) DO UPDATE SET body = excluded.body`, + [runId, encodeSecrets(secrets)], + ) + }, + catch: (cause) => new EventStoreError(`Failed to store secret context for ${runId}`, cause), + }), + } +} + function createSqliteEventStore(exec: SqliteExec): EventStore { const waiters = new Map void>>() @@ -580,6 +649,7 @@ function createSqliteEventStore(exec: SqliteExec): EventStore { .map((row) => decodeRunIdRow(row).run_id), catch: (cause) => new EventStoreError('listRuns failed', cause), }), + runSecrets: createSqliteSecrets(exec), } return service diff --git a/packages/core/src/state.ts b/packages/core/src/state.ts index 58d370b..e5e7cd6 100644 --- a/packages/core/src/state.ts +++ b/packages/core/src/state.ts @@ -9,6 +9,7 @@ import { import { JsonValueSchema } from './envelope' import { RunLaunchSourceSchema, type RunLaunchSource } from './launcher' import { DEFAULT_DEFINITION_VERSION } from './module' +import { RunAttributesSchema, type RunAttributes } from './run-context' import { isTerminalStatus } from './thread' import type { JsonValue, ThreadStatus, RunStatus } from './types' @@ -71,6 +72,7 @@ export interface RunStartIdentity { rootThreadId: string launchedBy: RunLaunchSource | null correlationId: string | null + attributes: RunAttributes } /** The run and last sequence a fork copied its history from. */ @@ -85,6 +87,8 @@ export interface RunState { rootThreadId: string | null startIdentity: RunStartIdentity | null forkedFrom: RunForkSource | null + /** Immutable labels copied from `runtime.run.started`. Safe for every reader of the run. */ + attributes: RunAttributes threads: { [threadId: string]: ThreadRecord } waits: { [waitId: string]: WaitRecord } outstandingEffects: readonly OutstandingEffect[] @@ -117,8 +121,10 @@ export const RunStateSchema = Schema.Struct({ correlationId: Schema.NullOr(Schema.String).pipe( Schema.withDecodingDefaultKey(Effect.succeed(null)), ), + attributes: RunAttributesSchema.pipe(Schema.withDecodingDefaultKey(Effect.succeed({}))), }), ).pipe(Schema.withDecodingDefaultKey(Effect.succeed(null))), + attributes: RunAttributesSchema.pipe(Schema.withDecodingDefaultKey(Effect.succeed({}))), forkedFrom: Schema.NullOr(Schema.Struct({ runId: Schema.String, seq: Schema.Finite })).pipe( Schema.withDecodingDefaultKey(Effect.succeed(null)), ), @@ -207,6 +213,7 @@ export function emptyRunState(runId: string): RunState { rootThreadId: null, startIdentity: null, forkedFrom: null, + attributes: {}, threads: {}, waits: {}, outstandingEffects: [], diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index 7651bdd..cf46794 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -2,6 +2,7 @@ import { Context, Data, Effect, Layer, Option, Queue, Ref, Stream } from 'effect import { withAssignedSeq, type AppendableEvent, type EventEnvelope } from './envelope' import { assertEventByteLimits, EventPayloadTooLargeError } from './limits' +import { readSecretContext, type RunSecretStore, type SecretContext } from './run-context' export class EventStoreError extends Data.TaggedError('EventStoreError')<{ readonly message: string @@ -288,6 +289,11 @@ export interface EventStore { ) => Stream.Stream readonly listRuns: Effect.Effect + /** + * Run-scoped bearer material. Absent on stores that cannot keep it out of the event log. + * A start that supplies secrets fails closed when this is missing. + */ + readonly runSecrets?: RunSecretStore } /** @@ -321,6 +327,13 @@ interface RunLog { export const makeMemoryEventStore = Effect.gen(function* () { const logs = yield* Ref.make(new Map()) + const secrets = yield* Ref.make(new Map()) + + const runSecrets: RunSecretStore = { + get: (runId) => + Ref.get(secrets).pipe(Effect.map((map) => readSecretContext(map.get(runId) ?? {}))), + put: (runId, value) => Ref.update(secrets, (map) => new Map(map).set(runId, value)), + } const getOrCreate = (map: Map, runId: string): RunLog => { const existing = map.get(runId) @@ -552,6 +565,7 @@ export const makeMemoryEventStore = Effect.gen(function* () { ), listRuns: Ref.get(logs).pipe(Effect.map((map) => [...map.keys()])), + runSecrets, } return service diff --git a/packages/react/src/materialize.test.ts b/packages/react/src/materialize.test.ts index 0cc08ab..58e9107 100644 --- a/packages/react/src/materialize.test.ts +++ b/packages/react/src/materialize.test.ts @@ -82,6 +82,7 @@ describe('materializeEvents', () => { kind: 'agent', definitionName: 'assistant', launchedBy, + attributes: {}, }) }) diff --git a/packages/react/src/materialize.ts b/packages/react/src/materialize.ts index 545242c..c92a3f3 100644 --- a/packages/react/src/materialize.ts +++ b/packages/react/src/materialize.ts @@ -2,6 +2,7 @@ import { completedRunStatus, isJsonObject, isJsonString, + readAttributes, readRunLaunchSource, type EventEnvelope, type JsonValue, @@ -29,6 +30,7 @@ function setRunStatus(tables: MaterializedTables, runId: string, status: RunStat kind: existing?.kind ?? null, definitionName: existing?.definitionName ?? null, launchedBy: existing?.launchedBy ?? null, + attributes: existing?.attributes ?? {}, }) } @@ -56,6 +58,7 @@ export function materializeEvents( kind: readString(payload, 'kind') ?? existing?.kind ?? null, definitionName: readString(payload, 'definitionName') ?? existing?.definitionName ?? null, launchedBy: readRunLaunchSource(payload.launchedBy) ?? existing?.launchedBy ?? null, + attributes: readAttributes(payload.attributes), } tables.runs.set(event.runId, next) diff --git a/packages/react/src/react.ts b/packages/react/src/react.ts index 89f2ae1..7b17773 100644 --- a/packages/react/src/react.ts +++ b/packages/react/src/react.ts @@ -13,11 +13,13 @@ import { import { isTerminalStatus, + sameStringRecord, threadTree, toThreadTree, type AnyEventEnvelope, type ProjectionDefinition, type RegisteredEvent, + type RunAttributes, type RunLaunchSource, type RunStatus, type ThreadTree, @@ -232,6 +234,7 @@ export interface RunSummary { readonly startedAt: number | undefined /** The effect in another run that launched this one. */ readonly launchedBy: RunLaunchSource | undefined + readonly attributes: RunAttributes readonly connection: LoomsConnectionStatus } @@ -243,6 +246,7 @@ function shallowEqualSummary(a: RunSummary, b: RunSummary): boolean { a.rootThreadId === b.rootThreadId && a.startedAt === b.startedAt && a.launchedBy === b.launchedBy && + sameStringRecord(a.attributes, b.attributes) && a.connection === b.connection ) } @@ -264,6 +268,7 @@ export function useRunSummary rootThreadId: run?.rootThreadId ?? undefined, startedAt: s.query.startedAt(), launchedBy: run?.launchedBy ?? undefined, + attributes: run?.attributes ?? {}, connection: s.status(), } }, []) diff --git a/packages/react/src/tables.ts b/packages/react/src/tables.ts index b78ff95..9fef0d4 100644 --- a/packages/react/src/tables.ts +++ b/packages/react/src/tables.ts @@ -1,4 +1,4 @@ -import type { RunLaunchSource, RunStatus, ThreadStatus } from '@looms/core' +import type { RunAttributes, RunLaunchSource, RunStatus, ThreadStatus } from '@looms/core' export interface RunRow { runId: string @@ -8,6 +8,7 @@ export interface RunRow { definitionName: string | null /** The effect in another run that launched this one. */ launchedBy: RunLaunchSource | null + attributes: RunAttributes } export interface ThreadRow { diff --git a/packages/runtime/src/errors.ts b/packages/runtime/src/errors.ts index 8d3611a..2e5b1a9 100644 --- a/packages/runtime/src/errors.ts +++ b/packages/runtime/src/errors.ts @@ -2,6 +2,7 @@ import { Data } from 'effect' import type { JsonValue, + RunAttributes, RunLaunchSource, RunStartIdentity as DurableRunStartIdentity, } from '@looms/core' @@ -15,6 +16,22 @@ export interface RunStartIdentity { readonly idempotencyKey: string | null readonly launchedBy: RunLaunchSource | null readonly correlationId: string | null + readonly attributes: RunAttributes +} + +/** The stored secret context differs. The message never includes the secret values. */ +export class SecretContextConflictError extends Data.TaggedError('SecretContextConflictError')<{ + readonly runId: string + readonly message: string +}> { + constructor(runId: string) { + super({ + runId, + message: `Run "${runId}" already has a different secret context`, + }) + + this.name = 'SecretContextConflictError' + } } export class StartRunConflictError extends Data.TaggedError('StartRunConflictError')<{ diff --git a/packages/runtime/src/helpers.ts b/packages/runtime/src/helpers.ts index c3d4961..dc0340d 100644 --- a/packages/runtime/src/helpers.ts +++ b/packages/runtime/src/helpers.ts @@ -13,6 +13,7 @@ import { validateEventsEffect, validateInputEffect, withSnapshotStore, + hasEntries, type AnyRuntimeModule, type AppendableEvent, type EventCatalog, @@ -23,6 +24,7 @@ import { InvalidEventError, type JsonValue, type OutstandingEffect, + type SecretContext, type RegisteredDefinition, type RunCursor, type RunState, @@ -35,14 +37,21 @@ export function workerTask( runId: string, item: OutstandingEffect, attempt: number, + secrets?: SecretContext, ): EffectWorkerTask { - return { + const task = { runId, effectId: item.effectId, attempt, type: item.effect.type, input: 'input' in item.effect ? item.effect.input : null, } + + if (secrets && hasEntries(secrets)) { + return { ...task, secrets } + } + + return task } export class UnknownDefinitionError extends Data.TaggedError('UnknownDefinitionError')<{ diff --git a/packages/runtime/src/local-effect-executor.ts b/packages/runtime/src/local-effect-executor.ts index e611713..bc1272d 100644 --- a/packages/runtime/src/local-effect-executor.ts +++ b/packages/runtime/src/local-effect-executor.ts @@ -11,8 +11,10 @@ import { type EventInput, type OutstandingEffect, type RegisteredDefinition, + type RunAttributes, type RunLauncher, type RunLaunchSource, + type SecretContext, } from '@looms/core' import { dispatchEffect } from './dispatch-effect' @@ -43,6 +45,8 @@ export interface ExecuteLocalEffectsInput { readonly runId: string /** Inherited by runs the effects launch. */ readonly correlationId: string + readonly attributes: RunAttributes + readonly secrets: SecretContext readonly outstanding: ReadonlyArray readonly attempts: ReadonlyMap readonly now: number @@ -56,6 +60,8 @@ function launcherFor( launcher: RunLauncher, runId: string, correlationId: string, + attributes: RunAttributes, + secrets: SecretContext, item: OutstandingEffect, launched: EventEnvelope[], ): RunLauncher { @@ -71,28 +77,37 @@ function launcherFor( launch: (args) => { const targetRunId = args.runId ?? createRunId() - return launcher.launch({ ...args, runId: targetRunId, launchedBy, correlationId }).pipe( - Effect.tap(() => - Effect.sync(() => { - launched.push( - createEvent(runId, { - type: 'runtime.run.launched', - payload: { - runId: targetRunId, - kind: args.kind, - definitionName: args.definitionName, - definitionVersion: args.definitionVersion ?? DEFAULT_DEFINITION_VERSION, - effectId: item.effectId, - }, - threadId: item.threadId, - causationId: item.causingEventId, - correlationId, - origin: { type: 'thread', threadId: item.threadId }, - }), - ) - }), - ), - ) + return launcher + .launch({ + ...args, + runId: targetRunId, + launchedBy, + correlationId, + attributes: args.attributes ?? attributes, + secrets: args.secrets ?? secrets, + }) + .pipe( + Effect.tap(() => + Effect.sync(() => { + launched.push( + createEvent(runId, { + type: 'runtime.run.launched', + payload: { + runId: targetRunId, + kind: args.kind, + definitionName: args.definitionName, + definitionVersion: args.definitionVersion ?? DEFAULT_DEFINITION_VERSION, + effectId: item.effectId, + }, + threadId: item.threadId, + causationId: item.causingEventId, + correlationId, + origin: { type: 'thread', threadId: item.threadId }, + }), + ) + }), + ), + ) }, status: launcher.status, } @@ -183,13 +198,23 @@ export function executeLocalEffects( registry, services, definitions, - launcherFor(launcher, input.runId, input.correlationId, item, launched), + launcherFor( + launcher, + input.runId, + input.correlationId, + input.attributes, + input.secrets, + item, + launched, + ), item.effect, { effectId: item.effectId, runId: input.runId, threadId: item.threadId, causingEventId: item.causingEventId, + attributes: input.attributes, + secrets: input.secrets, signal: controller.signal, emit: appendLive, }, diff --git a/packages/runtime/src/looms.ts b/packages/runtime/src/looms.ts index a1fc06d..636091e 100644 --- a/packages/runtime/src/looms.ts +++ b/packages/runtime/src/looms.ts @@ -27,6 +27,7 @@ import { type EffectWorker, type LoomsRuntime, type RunOperationalStatus, + type StartRunArgs, type WakeScheduler, type WorkerCallbackInput, } from './runtime' @@ -95,7 +96,13 @@ export interface Looms { start( definition: TDef, input?: DefinitionInput, - options?: { runId?: string; threadId?: string; idempotencyKey?: string }, + options?: { + runId?: string + threadId?: string + idempotencyKey?: string + attributes?: StartRunArgs['attributes'] + secrets?: StartRunArgs['secrets'] + }, ): Promise /** Start a run by `{ kind, definitionName }` when you only have names (HTTP bodies, CLIs). */ startRun(args: { @@ -106,6 +113,8 @@ export interface Looms { runId?: string threadId?: string idempotencyKey?: string + attributes?: StartRunArgs['attributes'] + secrets?: StartRunArgs['secrets'] }): Promise /** Commit a run's start and schedule a wake without waiting for it. */ launchRun(args: { @@ -116,6 +125,8 @@ export interface Looms { runId?: string threadId?: string idempotencyKey?: string + attributes?: StartRunArgs['attributes'] + secrets?: StartRunArgs['secrets'] }): Promise getRun(runId: string): Promise getEvents(runId: string, options?: { fromSeq?: number; limit?: number }): Promise @@ -262,6 +273,8 @@ export function createLooms(options: CreateLoomsOptions = {}): Looms { runId: startOpts?.runId, threadId: startOpts?.threadId, idempotencyKey: startOpts?.idempotencyKey, + attributes: startOpts?.attributes, + secrets: startOpts?.secrets, }), ), getRun: (runId) => runEffect((i) => i.runtime.getRun(runId)), diff --git a/packages/runtime/src/planners.test.ts b/packages/runtime/src/planners.test.ts index 53eefd5..6b7a2bd 100644 --- a/packages/runtime/src/planners.test.ts +++ b/packages/runtime/src/planners.test.ts @@ -24,6 +24,7 @@ function runningState(overrides: Partial = {}): RunState { rootThreadId: null, startIdentity: null, forkedFrom: null, + attributes: {}, threads: {}, waits: {}, outstandingEffects: [], diff --git a/packages/runtime/src/run-context.test.ts b/packages/runtime/src/run-context.test.ts new file mode 100644 index 0000000..2fb87db --- /dev/null +++ b/packages/runtime/src/run-context.test.ts @@ -0,0 +1,279 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { + RunLauncherTag, + complete, + defineEffect, + defineRuntimeModule, + defineThread, + hasEntries, + invoke, +} from '@looms/core' + +import { SecretContextConflictError, StartRunConflictError } from './errors' +import { createLooms } from './looms' +import { authDenied, authAllowed } from './server' + +const SECRET = 'super-secret-token' + +const seen = defineEffect({ + type: 'context.seen', + execute: (_input, ctx) => [ + { + type: 'context.observed', + payload: { + tenant: ctx.attributes.tenant ?? '', + hasSecret: ctx.secrets.token === SECRET, + }, + threadId: ctx.threadId, + }, + ], +}) + +const contextThread = defineThread({ + kind: 'context', + initialState: () => ({ done: false }), + step: (state, event) => (event.type === 'context.observed' ? { ...state, done: true } : state), + effects: (state) => (state.done ? [complete({ ok: true })] : [invoke(seen, {}, 'see')]), +}) + +const context = { kind: 'context', name: 'context', version: 'v1' } + +const launchChild = defineEffect({ + type: 'context.launch', + execute: (_input, ctx) => + Effect.gen(function* () { + const launcher = yield* RunLauncherTag + + yield* launcher.launch({ + runId: `${ctx.runId}-child`, + kind: 'context', + definitionName: 'context', + input: null, + }) + + return [ + { + type: 'context.observed', + payload: { tenant: '', hasSecret: false }, + threadId: ctx.threadId, + }, + ] + }), +}) + +const parentThread = defineThread({ + kind: 'parent', + initialState: () => ({ launched: false }), + step: (state, event) => + event.type === 'context.observed' ? { ...state, launched: true } : state, + effects: (state) => + state.launched ? [complete({ ok: true })] : [invoke(launchChild, {}, 'launch')], +}) + +const contextModule = defineRuntimeModule({ + namespace: 'context', + protocolVersion: '1.0.0', + threads: { context: contextThread, parent: parentThread }, + effects: { seen, launch: launchChild }, + definitions: [ + { kind: 'context', name: 'context', version: 'v1' }, + { kind: 'parent', name: 'parent', version: 'v1' }, + ], +}) + +function containsSecret(encoded: string): boolean { + return encoded.includes(SECRET) +} + +describe('run attributes and secret context', () => { + test('records attributes and keeps secrets out of the log, state, and HTTP responses', async () => { + const dispatched: Array<{ secrets?: { token?: string } }> = [] + + const looms = createLooms({ + modules: [contextModule], + worker: { handles: ['context.seen'], dispatch: (task) => dispatched.push(task) }, + }) + + const started = await looms.start(context, null, { + runId: 'run_context', + attributes: { tenant: 'acme', project: 'billing' }, + secrets: { token: SECRET }, + }) + + expect(started.state.attributes).toEqual({ tenant: 'acme', project: 'billing' }) + expect(containsSecret(JSON.stringify(started))).toBe(false) + + const events = await looms.getEvents('run_context') + const state = await looms.getRun('run_context') + const summaries = await looms.fetch(new Request('http://looms.test/runs/summaries')) + const listed = await summaries?.json() + + expect(containsSecret(JSON.stringify(events))).toBe(false) + expect(containsSecret(JSON.stringify(state))).toBe(false) + expect(containsSecret(JSON.stringify(listed))).toBe(false) + expect(state.attributes).toEqual({ tenant: 'acme', project: 'billing' }) + + expect(events.find((event) => event.type === 'runtime.run.started')?.payload).toMatchObject({ + attributes: { tenant: 'acme', project: 'billing' }, + }) + + const store = await looms.store + expect(store.runSecrets).toBeDefined() + + if (store.runSecrets) { + expect(await Effect.runPromise(store.runSecrets.get('run_context'))).toEqual({ + token: SECRET, + }) + } + + expect(dispatched[0]?.secrets).toEqual({ token: SECRET }) + await looms.stop() + }) + + test('a different attribute or secret context conflicts without revealing the secret', async () => { + const looms = createLooms({ modules: [contextModule] }) + + await looms.start(context, null, { + runId: 'run_once', + idempotencyKey: 'same', + attributes: { tenant: 'acme' }, + secrets: { token: SECRET }, + }) + + const attributeConflict = await looms + .start(context, null, { + runId: 'run_once', + idempotencyKey: 'same', + attributes: { tenant: 'other' }, + secrets: { token: SECRET }, + }) + .then( + () => null, + (error) => error, + ) + + expect(attributeConflict).toBeInstanceOf(StartRunConflictError) + + const secretConflict = await looms + .start(context, null, { + runId: 'run_once', + idempotencyKey: 'same', + attributes: { tenant: 'acme' }, + secrets: { token: 'rotated' }, + }) + .then( + () => null, + (error) => error, + ) + + expect(secretConflict).toBeInstanceOf(SecretContextConflictError) + expect(containsSecret(JSON.stringify(secretConflict))).toBe(false) + expect(JSON.stringify(secretConflict)).not.toContain('rotated') + + const retried = await looms.start(context, null, { + runId: 'run_once', + idempotencyKey: 'same', + attributes: { tenant: 'acme' }, + }) + + expect(retried.runId).toBe('run_once') + + const store = await looms.store + + if (store.runSecrets) { + expect(await Effect.runPromise(store.runSecrets.get('run_once'))).toEqual({ token: SECRET }) + } + + await looms.stop() + }) + + test('authorization sees attributes from the stored run and the start body', async () => { + const decisions: Array<{ attributes?: { tenant?: string } }> = [] + + const looms = createLooms({ + modules: [contextModule], + authorize: (_request, route) => + Effect.sync(() => { + decisions.push(route) + return route.attributes?.tenant === 'acme' ? authAllowed : authDenied(403, 'Forbidden') + }), + }) + + const started = await looms.fetch( + new Request('http://looms.test/runs', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + kind: 'context', + definitionName: 'context', + runId: 'run_auth', + attributes: { tenant: 'acme' }, + }), + }), + ) + + expect(started?.status).toBe(200) + + const read = await looms.fetch(new Request('http://looms.test/runs/run_auth')) + expect(read?.status).toBe(200) + expect(decisions.some((route) => route.attributes?.tenant === 'acme')).toBe(true) + await looms.stop() + }) + + test('a launched run and a fork inherit attributes and secrets without logging them', async () => { + const looms = createLooms({ modules: [contextModule] }) + + await looms.start({ kind: 'parent', name: 'parent', version: 'v1' }, null, { + runId: 'run_parent', + attributes: { tenant: 'acme' }, + secrets: { token: SECRET }, + }) + + const deadline = Date.now() + 5_000 + + while ((await looms.getRun('run_parent-child')).rootThreadId === null) { + if (Date.now() > deadline) { + throw new Error('the child run did not start') + } + + await Bun.sleep(5) + } + + const child = await looms.getRun('run_parent-child') + const childEvents = await looms.getEvents('run_parent-child') + expect(child.attributes).toEqual({ tenant: 'acme' }) + expect(containsSecret(JSON.stringify(childEvents))).toBe(false) + + const store = await looms.store + + if (store.runSecrets) { + expect(await Effect.runPromise(store.runSecrets.get('run_parent-child'))).toEqual({ + token: SECRET, + }) + } + + const finished = Date.now() + 5_000 + + while ((await looms.getRun('run_parent')).status !== 'completed') { + if (Date.now() > finished) { + throw new Error('the parent run did not complete') + } + + await Bun.sleep(5) + } + + const forked = await looms.fork('run_parent', { runId: 'run_fork' }) + expect(forked.state.attributes).toEqual({ tenant: 'acme' }) + expect(containsSecret(JSON.stringify(await looms.getEvents('run_fork')))).toBe(false) + + if (store.runSecrets) { + expect(hasEntries(await Effect.runPromise(store.runSecrets.get('run_fork')))).toBe(true) + expect(await Effect.runPromise(store.runSecrets.get('run_fork'))).toEqual({ token: SECRET }) + } + + await looms.stop() + }) +}) diff --git a/packages/runtime/src/run-start.ts b/packages/runtime/src/run-start.ts index 817ac25..5eb16c7 100644 --- a/packages/runtime/src/run-start.ts +++ b/packages/runtime/src/run-start.ts @@ -1,6 +1,6 @@ import { Effect, Predicate } from 'effect' -import type { JsonValue, RunState } from '@looms/core' +import { sameStringRecord, type JsonValue, type RunState } from '@looms/core' import { RuntimeExecutionError, StartRunConflictError, type RunStartIdentity } from './errors' @@ -69,6 +69,10 @@ export function assertSameDurableStart( mismatches.push('correlationId') } + if (!sameStringRecord(existing.attributes ?? {}, requested.attributes)) { + mismatches.push('attributes') + } + return mismatches.length > 0 ? Effect.fail(new StartRunConflictError(runId, requested, existing, mismatches)) : Effect.succeed(existing.rootThreadId) diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index ac0b33b..bf1eb22 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -11,12 +11,16 @@ import { descendantThreadIds, EventStoreTag, foldRun, + hasEntries, isRunTerminal, unavailableRunLauncher, planFork, project, replayTo, + sameStringRecord, summarizeRun, + type EventStore, + type SecretContext, type AnyRuntimeModule, type EventInput, type EventOrigin, @@ -34,6 +38,7 @@ import { RunNotFoundError, RunTerminalError, RuntimeExecutionError, + SecretContextConflictError, StartRunConflictError, } from './errors' import { @@ -111,6 +116,65 @@ const RESCAN_CONCURRENCY = 16 const DEFAULT_KEEP_SNAPSHOTS = 1 const DEFAULT_MAX_PENDING_RUN_OPERATIONS = 256 +interface StartedPayloadInput { + readonly threadId: string + readonly kind: string + readonly definitionName: string + readonly definitionVersion: string + readonly input: StartRunArgs['input'] + readonly requestedThreadId: string | null + readonly launchedBy: StartRunArgs['launchedBy'] + readonly attributes: StartRunArgs['attributes'] +} + +function startedPayload(input: StartedPayloadInput) { + const payload = { + rootThreadId: input.threadId, + kind: input.kind, + definitionName: input.definitionName, + definitionVersion: input.definitionVersion, + input: input.input ?? null, + requestedThreadId: input.requestedThreadId, + } + + const launched = input.launchedBy ? { ...payload, launchedBy: input.launchedBy } : payload + const attributes = input.attributes ?? {} + + return hasEntries(attributes) ? { ...launched, attributes } : launched +} + +function reconcileRunSecrets( + store: EventStore, + runId: string, + requested: SecretContext | undefined, +) { + const next = requested ?? {} + + return Effect.gen(function* () { + const stored = store.runSecrets + + if (!stored) { + if (hasEntries(next)) { + return yield* new RuntimeExecutionError('This event store cannot persist a secret context') + } + + return yield* Effect.void + } + + const current = yield* stored.get(runId) + + if (sameStringRecord(current, next) || requested === undefined) { + return yield* Effect.void + } + + if (!hasEntries(current)) { + return yield* stored.put(runId, next) + } + + return yield* new SecretContextConflictError(runId) + }) +} + export function createRuntime( options: CreateRuntimeOptions, ): LoomsRuntime { @@ -257,24 +321,27 @@ export function createRuntime { status: 'completed', kind: 'workflow', definitionName: 'summarized', + attributes: {}, }, ]) diff --git a/packages/runtime/src/server/auth.ts b/packages/runtime/src/server/auth.ts index 83008f3..4dba778 100644 --- a/packages/runtime/src/server/auth.ts +++ b/packages/runtime/src/server/auth.ts @@ -1,5 +1,7 @@ import { Effect } from 'effect' +import type { RunAttributes } from '@looms/core' + /** * What a route can do. `public` routes (health) are never passed to `authorize`; `read` and * `write` cover run state, `worker` covers external worker callbacks, and `operations` covers @@ -14,6 +16,8 @@ export interface RouteInfo { /** Stable route name, such as `runs.signal` or `effects.retry`. */ readonly name: string readonly runId?: string + /** Present when the run has labels. Safe for the authorizer to branch on. */ + readonly attributes?: RunAttributes } export type AuthDecision = diff --git a/packages/runtime/src/server/fetch-handler.ts b/packages/runtime/src/server/fetch-handler.ts index 831604d..c24a65b 100644 --- a/packages/runtime/src/server/fetch-handler.ts +++ b/packages/runtime/src/server/fetch-handler.ts @@ -1,6 +1,12 @@ -import { Effect, Predicate } from 'effect' +import { Data, Effect, Option, Predicate, Schema } from 'effect' -import type { EventStore } from '@looms/core' +import { + EventStoreTag, + hasEntries, + RunAttributesSchema, + type EventStore, + type RunAttributes, +} from '@looms/core' import type { LoomsRuntime } from '../runtime' import { authResponse, defaultAuthorize, type Authorize } from './auth' @@ -33,10 +39,33 @@ function createRoutes(options: RouteOptions): readonly Route[] { ] } +const StartAttributesSchema = Schema.Struct({ + attributes: Schema.optional(RunAttributesSchema), +}) + +class UnreadableStartBody extends Data.TaggedError('UnreadableStartBody')<{ + readonly cause: unknown +}> {} + +function readRequestAttributes(req: Request): Effect.Effect { + return Effect.tryPromise({ + try: () => req.clone().json(), + catch: (cause) => new UnreadableStartBody({ cause }), + }).pipe( + Effect.map((body) => { + const decoded = Schema.decodeUnknownOption(StartAttributesSchema)(body) + return Option.getOrElse(decoded, () => ({})).attributes ?? {} + }), + Effect.catchTag('UnreadableStartBody', () => Effect.succeed({})), + ) +} + function authorizeRoute( req: Request, matched: MatchedRoute, authorize: Authorize, + runtime: LoomsRuntime, + store: EventStore, ): Effect.Effect { const { access, name } = matched.route @@ -46,10 +75,27 @@ function authorizeRoute( const runId = matched.params.runId - return authorize( - req, - Predicate.isString(runId) ? { access, name, runId } : { access, name }, - ).pipe(Effect.map(authResponse)) + return Effect.gen(function* () { + const stored = Predicate.isString(runId) + ? yield* runtime.getRun(runId).pipe( + Effect.provideService(EventStoreTag, store), + Effect.orElseSucceed(() => undefined), + ) + : undefined + + const requested = matched.route.name === 'runs.start' ? yield* readRequestAttributes(req) : {} + const attributes = stored && hasEntries(stored.attributes) ? stored.attributes : requested + + const route = Predicate.isString(runId) + ? hasEntries(attributes) + ? { access, name, runId, attributes } + : { access, name, runId } + : hasEntries(attributes) + ? { access, name, attributes } + : { access, name } + + return yield* authorize(req, route).pipe(Effect.map(authResponse)) + }) } /** Whether the fetch handler answers `pathname`: the HTTP API or the debugger mount. */ @@ -90,7 +136,13 @@ export function createFetchHandler( return new Response('Not Found', { status: 404 }) } - const denied = yield* authorizeRoute(req, matched, authorize) + const denied = yield* authorizeRoute( + req, + matched, + authorize, + options.runtime, + options.store, + ) if (denied) { return denied diff --git a/packages/runtime/src/server/routes/runs.ts b/packages/runtime/src/server/routes/runs.ts index 1dab2e5..908a7c4 100644 --- a/packages/runtime/src/server/routes/runs.ts +++ b/packages/runtime/src/server/routes/runs.ts @@ -4,6 +4,8 @@ import { createRunId, EventStoreTag, JsonValueSchema, + RunAttributesSchema, + SecretContextSchema, stringifyJson, type EventStore, type JsonValue, @@ -32,6 +34,8 @@ const StartRunBodySchema = Schema.Struct({ input: Schema.optional(JsonValueSchema), runId: Schema.optional(Schema.String), idempotencyKey: Schema.optional(Schema.String), + attributes: Schema.optional(RunAttributesSchema), + secrets: Schema.optional(SecretContextSchema), }) const SignalBodySchema = Schema.Struct({ @@ -185,6 +189,8 @@ export function runRoutes({ runtime, store }: RouteOptions): readonly Route[] { input: body.input ?? null, runId, idempotencyKey, + attributes: body.attributes, + secrets: body.secrets, } if (wantsRunStream(req, url)) { diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 850756b..87b74b9 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -14,6 +14,8 @@ import type { EventStoreTrimCoverage, RunLauncher, RunLaunchSource, + RunAttributes, + SecretContext, EventStoreTruncationError, JsonValue, ProjectionDefinition, @@ -92,6 +94,11 @@ export interface EffectWorkerTask { readonly attempt: number readonly type: string readonly input: JsonValue + /** + * Present only on the in-memory dispatch. A transport that persists the task must leave this out + * and let the worker read the secret context through an authenticated callback. + */ + readonly secrets?: SecretContext } export interface EffectWorker { @@ -115,6 +122,13 @@ export interface StartRunArgs { launchedBy?: RunLaunchSource /** Recorded on `runtime.run.started` to group related runs. */ correlationId?: string + /** Immutable labels stored on the start event and the run summary. */ + attributes?: RunAttributes + /** + * Bearer material stored beside the log. A retry that sends a different context conflicts. + * Omitting it on a retry keeps the context already stored. + */ + secrets?: SecretContext } export interface StartResult { diff --git a/packages/runtime/src/wake-engine.ts b/packages/runtime/src/wake-engine.ts index 341c9d6..120f3b3 100644 --- a/packages/runtime/src/wake-engine.ts +++ b/packages/runtime/src/wake-engine.ts @@ -139,6 +139,7 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake return yield* Effect.gen(function* () { const store = yield* EventStoreTag + const secrets = store.runSecrets ? yield* store.runSecrets.get(runId) : {} const runInContext = Effect.runPromiseWith(yield* Effect.context()) executionContext.liveStore = store @@ -247,6 +248,7 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake registry, worker: options.worker, now, + secrets, }) if (workerPlans.length > 0) { @@ -388,6 +390,8 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake { runId, correlationId: cursor.state.startIdentity?.correlationId ?? runId, + attributes: cursor.state.attributes ?? {}, + secrets, outstanding, attempts, now, diff --git a/packages/runtime/src/worker-effect-dispatch.ts b/packages/runtime/src/worker-effect-dispatch.ts index 478a88d..a305e79 100644 --- a/packages/runtime/src/worker-effect-dispatch.ts +++ b/packages/runtime/src/worker-effect-dispatch.ts @@ -9,6 +9,7 @@ import { type EventEnvelope, type OutstandingEffect, type RunState, + type SecretContext, } from '@looms/core' import { RuntimeExecutionError } from './errors' @@ -103,6 +104,7 @@ export function planWorkerEffects(input: { readonly registry: ComposedRegistry readonly worker?: EffectWorker readonly now: number + readonly secrets?: SecretContext }): ReadonlyArray { return input.state.outstandingEffects .filter((item) => { @@ -147,7 +149,7 @@ export function planWorkerEffects(input: { causationId: item.causingEventId, origin: { type: 'system' }, }), - task: workerTask(input.runId, item, attempt), + task: workerTask(input.runId, item, attempt, input.secrets), scheduleToStartDeadlineAt: retry?.scheduleToStartTimeoutMs ? input.now + retry.scheduleToStartTimeoutMs : null, From 529bf6c3ce049b8e6bb430b77edf26f4494be379 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 10:45:53 +0200 Subject: [PATCH 03/23] feat(runtime): offload oversized JSON through a content-addressed blob store Strings over 64 KiB, and any event payload that would still exceed 1 MiB, are stored by SHA-256. Reads and snapshots resolve the references, and trim deletes a body once nothing retained points at it. Co-authored-by: Cursor --- .changeset/blob-offload.md | 5 + .oxlintrc.json | 3 +- docs/production.md | 18 ++ packages/cloudflare/src/durable-object.ts | 4 + packages/cloudflare/src/event-store.ts | 11 +- packages/cloudflare/src/index.ts | 7 + packages/cloudflare/src/r2-blob-store.test.ts | 83 +++++ packages/cloudflare/src/r2-blob-store.ts | 103 +++++++ packages/core/src/blob-offload.test.ts | 257 ++++++++++++++++ packages/core/src/blob-offload.ts | 282 +++++++++++++++++ packages/core/src/blob-store.ts | 267 +++++++++++++++++ packages/core/src/blobs.ts | 283 ++++++++++++++++++ packages/core/src/index.ts | 3 + packages/runtime/src/blob-offload.test.ts | 82 +++++ .../runtime/src/filesystem-blob-store.test.ts | 28 ++ packages/runtime/src/filesystem-blob-store.ts | 84 ++++++ packages/runtime/src/index.ts | 1 + packages/runtime/src/looms.ts | 21 +- 18 files changed, 1537 insertions(+), 5 deletions(-) create mode 100644 .changeset/blob-offload.md create mode 100644 packages/cloudflare/src/r2-blob-store.test.ts create mode 100644 packages/cloudflare/src/r2-blob-store.ts create mode 100644 packages/core/src/blob-offload.test.ts create mode 100644 packages/core/src/blob-offload.ts create mode 100644 packages/core/src/blob-store.ts create mode 100644 packages/core/src/blobs.ts create mode 100644 packages/runtime/src/blob-offload.test.ts create mode 100644 packages/runtime/src/filesystem-blob-store.test.ts create mode 100644 packages/runtime/src/filesystem-blob-store.ts diff --git a/.changeset/blob-offload.md b/.changeset/blob-offload.md new file mode 100644 index 0000000..8a46da6 --- /dev/null +++ b/.changeset/blob-offload.md @@ -0,0 +1,5 @@ +--- +'@swirls/looms': minor +--- + +`createLooms({ blobStore })` and `LoomsDurableObject` `blobs` store strings over 64 KiB by SHA-256 and keep a reference in the event or snapshot. A payload that is still over the 1 MiB event limit is stored as one blob. Reads, subscribers, folds, projections, and snapshot loads resolve the reference, so a run can complete with a payload larger than that limit. Trim removes a blob body once nothing retained still points at it. `memoryBlobStore`, `filesystemBlobStore`, and `r2BlobStore` are the included stores. Without a blob store, events over 1 MiB and snapshots over 8 MiB are still rejected. diff --git a/.oxlintrc.json b/.oxlintrc.json index e56ea0a..4c413cb 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -85,7 +85,8 @@ "files": [ "apps/demo/src/server.ts", "packages/debugger/src/server/**/*.ts", - "packages/s2/src/lite.ts" + "packages/s2/src/lite.ts", + "packages/runtime/src/filesystem-blob-store.ts" ], "rules": { "effecttsgo/global-fetch": "off", diff --git a/docs/production.md b/docs/production.md index 866dee0..a873644 100644 --- a/docs/production.md +++ b/docs/production.md @@ -17,6 +17,24 @@ All built-in stores reject an event over 1 MiB and a snapshot over 8 MiB, measured as serialized UTF-8, before storage is mutated. The stable error names are `EventPayloadTooLargeError` and `SnapshotPayloadTooLargeError`. +Pass `blobStore` to `createLooms`, or `blobs` to `LoomsDurableObject` / +`durableObjectEventStore`, to keep larger JSON. Strings over 64 KiB are replaced +with `{ "$looms": "blob", "sha256", "bytes", "mediaType": "application/json" }` +before those limits run. That object is reserved in stored JSON. If an event is +still over 1 MiB, its payload is stored as one blob. The body is SHA-256 +content, so identical JSON is stored once. `read`, live subscribers, folds, +projections, and snapshot loads resolve references, including references nested +inside another blob. Objects and arrays stay inline so snapshot state still +decodes; a snapshot that is over 8 MiB after large strings are removed is still +rejected. Trim deletes a body when no retained event and no latest snapshot in +any run still needs it. Without a blob store, oversized events and snapshots +are still rejected. + +`memoryBlobStore` is the in-memory implementation. `filesystemBlobStore(directory)` +is the Bun and Node implementation. `r2BlobStore(bucket)` writes an R2 bucket +under `looms/blobs/`. One blob store should have one writer. Two Durable Objects +sharing a bucket can delete a hash the other still references. + ## Worker effect protocol 1. The runtime dispatches `{ runId, effectId, attempt, type, input }`. diff --git a/packages/cloudflare/src/durable-object.ts b/packages/cloudflare/src/durable-object.ts index d5746df..60047b7 100644 --- a/packages/cloudflare/src/durable-object.ts +++ b/packages/cloudflare/src/durable-object.ts @@ -6,6 +6,7 @@ import { createRemoteRunLauncher, type AnyRuntimeModule, type EventInput, + type BlobStore, type EventStoreTrimCoverage, type LaunchRunArgs, type RunLauncher, @@ -22,6 +23,8 @@ import type { DurableObjectNamespaceLike } from './router' export interface LoomsDurableObjectConfig { readonly modules: readonly AnyRuntimeModule[] readonly projectors?: readonly Projector[] + /** Content-addressed store for event and snapshot JSON over the inline byte limits. */ + readonly blobs?: BlobStore readonly snapshotEvery?: number readonly maxWakeIterations?: number readonly maxPendingRunOperations?: number @@ -87,6 +90,7 @@ export abstract class LoomsDurableObject extends DurableObject Effect.Effect + /** Offloads oversized event and snapshot JSON. See `withBlobOffload`. */ + readonly blobs?: BlobStore } export interface DurableObjectEventStore extends EventStore { @@ -32,7 +34,12 @@ export function durableObjectEventStore( transaction: (fn) => ctx.storage.transactionSync(fn), }) - const store = sqliteEventStore({ exec }) + const sqlite = sqliteEventStore({ exec }) + const snapshots = snapshotStoreOf(sqlite) + + const store = options.blobs + ? withBlobOffload(sqlite, options.blobs, snapshots ? { snapshots } : undefined) + : sqlite if (options.projectors && options.projectors.length > 0) { const delivery = createProjectorDelivery( diff --git a/packages/cloudflare/src/index.ts b/packages/cloudflare/src/index.ts index 47e4c3c..397f22f 100644 --- a/packages/cloudflare/src/index.ts +++ b/packages/cloudflare/src/index.ts @@ -11,6 +11,13 @@ export { type DurableObjectRunStub, type LoomsDurableObjectConfig, } from './durable-object' +export { + r2BlobStore, + type R2BucketLike, + type R2ListResult, + type R2ObjectBodyLike, + type R2StoredObject, +} from './r2-blob-store' export { routeToDurableObject, type DurableObjectNamespaceLike, diff --git a/packages/cloudflare/src/r2-blob-store.test.ts b/packages/cloudflare/src/r2-blob-store.test.ts new file mode 100644 index 0000000..ef916dc --- /dev/null +++ b/packages/cloudflare/src/r2-blob-store.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { isBlobRef, isJsonObject, offloadJson, resolveJson } from '@looms/core' + +import { r2BlobStore, type R2BucketLike, type R2ListResult } from './r2-blob-store' + +function fakeBucket(): R2BucketLike & { readonly keys: () => string[] } { + const objects = new Map() + + return { + keys: () => [...objects.keys()], + + put: (key, value) => { + objects.set(key, value.slice()) + return Promise.resolve({ key }) + }, + + get: (key) => { + const body = objects.get(key) + + if (!body) { + return Promise.resolve(null) + } + + const copy = body.slice() + + return Promise.resolve({ + arrayBuffer: () => + Promise.resolve(copy.buffer.slice(copy.byteOffset, copy.byteOffset + copy.byteLength)), + }) + }, + + delete: (key) => { + objects.delete(key) + return Promise.resolve(undefined) + }, + + list: (options) => { + const prefix = options?.prefix ?? '' + const keys = [...objects.keys()].filter((key) => key.startsWith(prefix)).toSorted() + const start = options?.cursor === undefined ? 0 : Number(options.cursor) + const page = keys.slice(start, start + 1) + const next = start + page.length + const truncated = next < keys.length + + const result: R2ListResult = { + objects: page.map((key) => ({ key })), + truncated, + } + + if (truncated) { + return Promise.resolve({ ...result, cursor: String(next) }) + } + + return Promise.resolve(result) + }, + } +} + +describe('r2BlobStore', () => { + test('pages the bucket and resolves an offloaded value', async () => { + const bucket = fakeBucket() + const blobs = r2BlobStore(bucket) + const value = { text: 'q'.repeat(80) } + const stored = await Effect.runPromise(offloadJson(value, blobs, { thresholdBytes: 16 })) + const text = isJsonObject(stored) ? stored.text : undefined + + if (text === undefined || !isBlobRef(text)) { + throw new Error('expected the string to be stored by hash') + } + + await Effect.runPromise(blobs.retain('run-a', [text.sha256])) + await Effect.runPromise(blobs.retain('run-b', [text.sha256])) + await Effect.runPromise(blobs.replaceRefs('run-a', new Set())) + + const resolved = await Effect.runPromise(resolveJson(stored, blobs)) + + expect(resolved).toEqual(value) + expect(bucket.keys().length).toBeGreaterThan(1) + }) +}) diff --git a/packages/cloudflare/src/r2-blob-store.ts b/packages/cloudflare/src/r2-blob-store.ts new file mode 100644 index 0000000..5adc31e --- /dev/null +++ b/packages/cloudflare/src/r2-blob-store.ts @@ -0,0 +1,103 @@ +import { Effect } from 'effect' + +import { blobStoreFromBytes, BlobStoreError, type BlobBytes, type BlobStore } from '@looms/core' + +const PREFIX = 'looms/blobs/' + +export interface R2ObjectBodyLike { + arrayBuffer(): Promise +} + +export interface R2ListResult { + readonly objects: ReadonlyArray<{ readonly key: string }> + readonly truncated: boolean + readonly cursor?: string +} + +/** + * Structural subset of a Cloudflare `R2Bucket` binding. An `R2Bucket` satisfies it. + */ +export interface R2StoredObject { + readonly key: string +} + +export interface R2BucketLike { + put(key: string, value: Uint8Array): Promise + get(key: string): Promise + delete(key: string): Promise + list(options?: { prefix?: string; cursor?: string }): Promise +} + +/** + * Content-addressed blob store in R2 under `looms/blobs/`. + * Reference counting is local to this bucket. Give one store one writer: two Durable + * Objects deleting the same hash at the same time can remove a body the other still needs. + */ +export function r2BlobStore(bucket: R2BucketLike): BlobStore { + return blobStoreFromBytes(r2BlobBytes(bucket)) +} + +function r2BlobBytes(bucket: R2BucketLike): BlobBytes { + const keyFor = (key: string) => `${PREFIX}${key}` + + return { + read: (key) => + Effect.tryPromise({ + try: () => bucket.get(keyFor(key)), + catch: (cause) => new BlobStoreError(`Unable to read blob key ${key}`, cause), + }).pipe( + Effect.flatMap((object) => { + if (!object) { + return Effect.succeed(null) + } + + return Effect.tryPromise({ + try: () => object.arrayBuffer(), + catch: (cause) => new BlobStoreError(`Unable to read blob key ${key}`, cause), + }).pipe(Effect.map((buffer) => new Uint8Array(buffer))) + }), + ), + + write: (key, body) => + Effect.tryPromise({ + try: () => bucket.put(keyFor(key), body), + catch: (cause) => new BlobStoreError(`Unable to write blob key ${key}`, cause), + }).pipe(Effect.asVoid), + + remove: (key) => + Effect.tryPromise({ + try: () => bucket.delete(keyFor(key)), + catch: (cause) => new BlobStoreError(`Unable to delete blob key ${key}`, cause), + }).pipe(Effect.asVoid), + + list: (prefix) => listPrefixed(bucket, `${PREFIX}${prefix}`), + } +} + +function listPrefixed( + bucket: R2BucketLike, + prefix: string, +): Effect.Effect, BlobStoreError> { + return Effect.gen(function* () { + const keys: string[] = [] + let cursor: string | undefined + + for (;;) { + const page = yield* Effect.tryPromise({ + try: () => + cursor === undefined ? bucket.list({ prefix }) : bucket.list({ prefix, cursor }), + catch: (cause) => new BlobStoreError(`Unable to list blob keys ${prefix}`, cause), + }) + + for (const object of page.objects) { + keys.push(object.key.slice(PREFIX.length)) + } + + if (!page.truncated || page.cursor === undefined || page.cursor === cursor) { + return keys + } + + cursor = page.cursor + } + }) +} diff --git a/packages/core/src/blob-offload.test.ts b/packages/core/src/blob-offload.test.ts new file mode 100644 index 0000000..8a2090a --- /dev/null +++ b/packages/core/src/blob-offload.test.ts @@ -0,0 +1,257 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect, Option, Stream } from 'effect' + +import { withBlobOffload } from './blob-offload' +import { BlobNotFoundError, memoryBlobStore } from './blob-store' +import { collectBlobDigests, expandBlobDigests, isBlobRef } from './blobs' +import { createEvent } from './envelope' +import { makeMemorySnapshotStore, snapshotStoreOf } from './snapshot-store' +import { emptyRunState } from './state' +import { makeMemoryEventStore } from './store' +import { isJsonObject, type JsonValue } from './types' + +function started(runId: string, payload: JsonValue) { + return createEvent(runId, { + type: 'runtime.run.started', + payload, + threadId: null, + origin: { type: 'system' }, + }) +} + +function firstDigest(value: JsonValue): string | undefined { + if (isBlobRef(value)) { + return value.sha256 + } + + if (Array.isArray(value)) { + for (const item of value) { + const found = firstDigest(item) + + if (found) { + return found + } + } + + return undefined + } + + if (isJsonObject(value)) { + for (const child of Object.values(value)) { + const found = firstDigest(child) + + if (found) { + return found + } + } + } + + return undefined +} + +describe('blob offload', () => { + test('keeps small JSON inline and resolves values over the threshold', async () => { + const inner = await Effect.runPromise(makeMemoryEventStore) + const blobs = memoryBlobStore() + const store = withBlobOffload(inner, blobs, { thresholdBytes: 32 }) + const large = 'x'.repeat(80) + + await Effect.runPromise(store.append('run_small', [started('run_small', { n: 1 })])) + + await Effect.runPromise(store.append('run_large', [started('run_large', { blob: large })])) + + const small = await Effect.runPromise(inner.read('run_small')) + expect(small[0]?.payload).toEqual({ n: 1 }) + expect(JSON.stringify(small)).not.toContain('"$looms"') + + const raw = await Effect.runPromise(inner.read('run_large')) + expect(JSON.stringify(raw)).toContain('"$looms":"blob"') + expect(JSON.stringify(raw)).not.toContain(large) + + const resolved = await Effect.runPromise(store.read('run_large')) + expect(resolved[0]?.payload).toEqual({ blob: large }) + + const live = await Effect.runPromise( + store.subscribe('run_large', { fromSeq: 1 }).pipe(Stream.take(1), Stream.runCollect), + ) + + expect(live[0]?.payload).toEqual({ blob: large }) + }) + + test('stores one body for the same bytes and deletes it only after every run releases it', async () => { + const inner = await Effect.runPromise(makeMemoryEventStore) + const blobs = memoryBlobStore() + const store = withBlobOffload(inner, blobs, { thresholdBytes: 32 }) + const payload = { blob: 'x'.repeat(80) } + + await Effect.runPromise(store.append('run_a', [started('run_a', payload)])) + await Effect.runPromise(store.append('run_b', [started('run_b', payload)])) + + const raw = await Effect.runPromise(inner.read('run_a')) + const digest = firstDigest(raw[0]?.payload ?? null) + + expect(digest).toBeDefined() + + await Effect.runPromise(store.trim!('run_a', 10)) + + const surviving = await Effect.runPromise(store.read('run_b')) + expect(surviving[0]?.payload).toEqual(payload) + + await Effect.runPromise(store.trim!('run_b', 10)) + + const missing = await Effect.runPromise( + blobs.get(digest!).pipe( + Effect.match({ + onFailure: (error) => error, + onSuccess: () => null, + }), + ), + ) + + expect(missing).toBeInstanceOf(BlobNotFoundError) + }) + + test('deletes nested bodies when the parent reference is trimmed', async () => { + const inner = await Effect.runPromise(makeMemoryEventStore) + const blobs = memoryBlobStore() + const store = withBlobOffload(inner, blobs, { thresholdBytes: 32, maxEventBytes: 200 }) + const text = 'y'.repeat(80) + const payload = [text, text, text, text] + + await Effect.runPromise(store.append('run_nested', [started('run_nested', payload)])) + + const raw = await Effect.runPromise(inner.read('run_nested')) + const seeds = new Set() + collectBlobDigests(raw[0]?.payload ?? null, seeds) + + const expanded = await Effect.runPromise(expandBlobDigests(blobs, seeds)) + expect(expanded.size).toBeGreaterThan(1) + expect(isBlobRef(raw[0]?.payload ?? null)).toBe(true) + + const resolved = await Effect.runPromise(store.read('run_nested')) + expect(resolved[0]?.payload).toEqual(payload) + + await Effect.runPromise(store.trim!('run_nested', 10)) + + for (const digest of expanded) { + const missing = await Effect.runPromise( + blobs.get(digest).pipe( + Effect.match({ + onFailure: (error) => error, + onSuccess: () => null, + }), + ), + ) + + expect(missing).toBeInstanceOf(BlobNotFoundError) + } + }) + + test('fails a read that names a missing blob and ignores a lookalike object', async () => { + const inner = await Effect.runPromise(makeMemoryEventStore) + const blobs = memoryBlobStore() + const store = withBlobOffload(inner, blobs, { thresholdBytes: 32 }) + + const lookalike = { + $looms: 'blob', + sha256: 'b'.repeat(64), + bytes: 2, + mediaType: 'application/json', + extra: true, + } + + await Effect.runPromise( + inner.append('run_missing', [ + started('run_missing', { + $looms: 'blob', + sha256: 'a'.repeat(64), + bytes: 2, + mediaType: 'application/json', + }), + ]), + ) + + await Effect.runPromise(store.append('run_lookalike', [started('run_lookalike', lookalike)])) + + const missing = await Effect.runPromise( + store.read('run_missing').pipe( + Effect.match({ + onFailure: (error) => error, + onSuccess: () => null, + }), + ), + ) + + expect(missing).toBeInstanceOf(Error) + expect(String(missing)).toContain('was not found') + + const kept = await Effect.runPromise(store.read('run_lookalike')) + expect(kept[0]?.payload).toEqual(lookalike) + }) + + test('offloads snapshot state and keeps those blobs when the log is trimmed', async () => { + const inner = await Effect.runPromise(makeMemoryEventStore) + const snapshots = await Effect.runPromise(makeMemorySnapshotStore) + const blobs = memoryBlobStore() + const store = withBlobOffload(inner, blobs, { snapshots, thresholdBytes: 32 }) + const saved = snapshotStoreOf(store) + const large = 'z'.repeat(200) + + const state = { + ...emptyRunState('run_snap'), + startIdentity: { + kind: 'workflow', + definitionName: 'blob', + definitionVersion: 'v1', + input: { blob: large }, + requestedThreadId: null, + idempotencyKey: null, + rootThreadId: 'thr_root', + launchedBy: null, + correlationId: null, + }, + } + + expect(saved).toBeDefined() + + await Effect.runPromise( + saved!.save({ + runId: 'run_snap', + cursor: 1, + stateHash: 'hash', + takenAt: 1, + state, + }), + ) + + const raw = await Effect.runPromise(snapshots.loadLatest('run_snap')) + expect(Option.isSome(raw)).toBe(true) + + if (Option.isSome(raw)) { + const encoded = JSON.stringify(raw.value.state) + expect(encoded).toContain('"$looms":"blob"') + expect(encoded).not.toContain(large) + } + + const loaded = await Effect.runPromise(saved!.loadLatest('run_snap')) + expect(Option.isSome(loaded)).toBe(true) + + if (Option.isSome(loaded)) { + expect(loaded.value.state.startIdentity?.input).toEqual({ blob: large }) + } + + await Effect.runPromise( + store.append('run_snap', [started('run_snap', { other: 'q'.repeat(80) })]), + ) + + await Effect.runPromise(store.trim!('run_snap', 10)) + + const afterTrim = await Effect.runPromise(saved!.loadLatest('run_snap')) + expect(Option.isSome(afterTrim)).toBe(true) + + if (Option.isSome(afterTrim)) { + expect(afterTrim.value.state.startIdentity?.input).toEqual({ blob: large }) + } + }) +}) diff --git a/packages/core/src/blob-offload.ts b/packages/core/src/blob-offload.ts new file mode 100644 index 0000000..680da6b --- /dev/null +++ b/packages/core/src/blob-offload.ts @@ -0,0 +1,282 @@ +import { Effect, Option, Schema, Stream } from 'effect' + +import { BlobStoreError, type BlobStore, type BlobStoreFailure } from './blob-store' +import { + DEFAULT_BLOB_THRESHOLD_BYTES, + claimJson, + collectBlobDigests, + expandBlobDigests, + offloadJson, + resolveJson, +} from './blobs' +import type { AppendableEvent, EventEnvelope } from './envelope' +import { DEFAULT_MAX_EVENT_BYTES, utf8JsonBytes } from './limits' +import { SnapshotStoreError, withSnapshotStore, type SnapshotStore } from './snapshot-store' +import { RunStateSchema, type RunState } from './state' +import { EventStoreError, type EventStore } from './store' +import type { JsonValue } from './types' + +const decodeJsonText = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Json)) +const decodeRunState = Schema.decodeUnknownEffect(RunStateSchema) + +export interface BlobOffloadOptions { + /** Snapshot store to offload and resolve. Trim keeps digests referenced by its latest snapshot. */ + readonly snapshots?: SnapshotStore + /** UTF-8 JSON size above which a string is stored by hash. Default 64 KiB. */ + readonly thresholdBytes?: number + /** Serialized event size that claims the whole payload. Default 1 MiB. */ + readonly maxEventBytes?: number +} + +/** + * Store oversized JSON outside the event log. + * + * Append replaces values over the threshold with a content-addressed reference before the + * event byte limit runs. `read`, `readStream`, and `subscribe` resolve those references, so + * folds, projections, and callers see the original values. The stored log keeps the references. + * When `snapshots` is set, snapshot save and load do the same, and trim deletes a body only + * once no retained event and no latest snapshot in any run still needs it. + */ +export function withBlobOffload( + store: EventStore, + blobs: BlobStore, + options?: BlobOffloadOptions, +): EventStore { + const thresholdBytes = options?.thresholdBytes ?? DEFAULT_BLOB_THRESHOLD_BYTES + const maxEventBytes = options?.maxEventBytes ?? DEFAULT_MAX_EVENT_BYTES + const snapshots = options?.snapshots + + const wrappedSnapshots = snapshots + ? wrapSnapshots(snapshots, store, blobs, thresholdBytes) + : undefined + + const release = (runId: string): Effect.Effect => + keepDigests(store, snapshots, blobs, runId).pipe( + Effect.flatMap((keep) => blobs.replaceRefs(runId, keep)), + Effect.mapError(asEventError), + ) + + const wrapped: EventStore = { + append: (runId, events, appendOptions) => + Effect.gen(function* () { + const stored = yield* Effect.forEach(events, (event) => + offloadAppendable(runId, event, blobs, thresholdBytes, maxEventBytes), + ).pipe(Effect.mapError(asEventError)) + + const seeds = digestsOf(stored.map((event) => event.payload)) + + const expanded = yield* expandBlobDigests(blobs, seeds).pipe(Effect.mapError(asEventError)) + + yield* blobs.retain(runId, [...expanded]).pipe(Effect.mapError(asEventError)) + + return yield* store.append(runId, stored, appendOptions) + }), + + read: (runId, readOptions) => + store + .read(runId, readOptions) + .pipe( + Effect.flatMap((events) => + Effect.forEach(events, (event) => + resolveEnvelope(event, blobs).pipe(Effect.mapError(asEventError)), + ), + ), + ), + + readStream: store.readStream + ? (runId, readOptions) => + store.readStream!(runId, readOptions).pipe( + Stream.mapEffect((event) => + resolveEnvelope(event, blobs).pipe(Effect.mapError(asEventError)), + ), + ) + : undefined, + + tail: (runId) => store.tail(runId), + + bounds: store.bounds ? (runId) => store.bounds!(runId) : undefined, + + trim: store.trim + ? (runId, beforeSeq) => store.trim!(runId, beforeSeq).pipe(Effect.andThen(release(runId))) + : undefined, + + subscribe: (runId, subscribeOptions) => + store + .subscribe(runId, subscribeOptions) + .pipe( + Stream.mapEffect((event) => + resolveEnvelope(event, blobs).pipe(Effect.mapError(asEventError)), + ), + ), + + listRuns: store.listRuns, + } + + if (!wrappedSnapshots) { + return wrapped + } + + return withSnapshotStore(wrapped, wrappedSnapshots) +} + +function offloadAppendable( + runId: string, + event: AppendableEvent, + blobs: BlobStore, + thresholdBytes: number, + maxEventBytes: number, +): Effect.Effect { + return Effect.gen(function* () { + const payload = yield* offloadJson(event.payload, blobs, { thresholdBytes }) + const offloaded = { ...event, payload } + + if (eventBytes(runId, offloaded) <= maxEventBytes) { + return offloaded + } + + const ref = yield* claimJson(payload, blobs) + + return { ...event, payload: ref } + }) +} + +function eventBytes(runId: string, event: AppendableEvent): number { + return utf8JsonBytes({ ...event, runId, seq: event.seq ?? 0 }) +} + +function resolveEnvelope( + event: EventEnvelope, + blobs: BlobStore, +): Effect.Effect { + return resolveJson(event.payload, blobs).pipe(Effect.map((payload) => ({ ...event, payload }))) +} + +function digestsOf(values: ReadonlyArray): Set { + const seeds = new Set() + + for (const value of values) { + collectBlobDigests(value, seeds) + } + + return seeds +} + +function keepDigests( + store: EventStore, + snapshots: SnapshotStore | undefined, + blobs: BlobStore, + runId: string, +): Effect.Effect, EventStoreError | BlobStoreFailure> { + return Effect.gen(function* () { + const events = yield* store.read(runId) + const seeds = digestsOf(events.map((event) => event.payload)) + + if (snapshots) { + const latest = yield* snapshots + .loadLatest(runId) + .pipe(Effect.mapError((cause) => new EventStoreError(cause.message, cause))) + + if (Option.isSome(latest)) { + const encoded = yield* jsonValue(latest.value.state).pipe( + Effect.mapError((cause) => new EventStoreError(cause.message, cause)), + ) + + collectBlobDigests(encoded, seeds) + } + } + + return yield* expandBlobDigests(blobs, seeds) + }) +} + +function wrapSnapshots( + snapshots: SnapshotStore, + store: EventStore, + blobs: BlobStore, + thresholdBytes: number, +): SnapshotStore { + const release = (runId: string): Effect.Effect => + keepDigests(store, snapshots, blobs, runId).pipe( + Effect.flatMap((keep) => blobs.replaceRefs(runId, keep)), + Effect.mapError((cause) => new SnapshotStoreError(cause.message, cause)), + ) + + return { + loadLatest: (runId) => + Effect.gen(function* () { + const loaded = yield* snapshots.loadLatest(runId) + + if (Option.isNone(loaded)) { + return loaded + } + + const encoded = yield* jsonValue(loaded.value.state).pipe(Effect.mapError(asSnapshotError)) + const resolved = yield* resolveJson(encoded, blobs).pipe(Effect.mapError(asSnapshotError)) + + const state = yield* decodeRunState(resolved).pipe( + Effect.mapError( + (cause) => new SnapshotStoreError('snapshot state did not match RunState', cause), + ), + ) + + return Option.some({ ...loaded.value, state }) + }), + + save: (snapshot) => + Effect.gen(function* () { + const encoded = yield* jsonValue(snapshot.state).pipe(Effect.mapError(asSnapshotError)) + + const offloaded = yield* offloadJson(encoded, blobs, { + thresholdBytes, + claimRoot: false, + }).pipe(Effect.mapError(asSnapshotError)) + + const state = yield* decodeRunState(offloaded).pipe( + Effect.mapError( + (cause) => new SnapshotStoreError('snapshot state did not match RunState', cause), + ), + ) + + const seeds = digestsOf([offloaded]) + + const expanded = yield* expandBlobDigests(blobs, seeds).pipe( + Effect.mapError(asSnapshotError), + ) + + yield* blobs.retain(snapshot.runId, [...expanded]).pipe(Effect.mapError(asSnapshotError)) + + return yield* snapshots.save({ + runId: snapshot.runId, + cursor: snapshot.cursor, + stateHash: snapshot.stateHash, + takenAt: snapshot.takenAt, + state, + }) + }), + + listCursors: snapshots.listCursors ? (runId) => snapshots.listCursors!(runId) : undefined, + + prune: snapshots.prune + ? (runId, keepLatest) => + snapshots.prune!(runId, keepLatest).pipe(Effect.andThen(release(runId))) + : undefined, + + saveHeader: (header) => snapshots.saveHeader(header), + + listHeaders: snapshots.listHeaders, + } +} + +function jsonValue(state: RunState): Effect.Effect { + return decodeJsonText(JSON.stringify(state)).pipe( + Effect.mapError((cause) => new BlobStoreError('snapshot state is not JSON', cause)), + ) +} + +function asEventError(cause: BlobStoreFailure | EventStoreError): EventStoreError { + return new EventStoreError(cause.message, cause) +} + +function asSnapshotError(cause: BlobStoreFailure | EventStoreError): SnapshotStoreError { + return new SnapshotStoreError(cause.message, cause) +} diff --git a/packages/core/src/blob-store.ts b/packages/core/src/blob-store.ts new file mode 100644 index 0000000..0289a66 --- /dev/null +++ b/packages/core/src/blob-store.ts @@ -0,0 +1,267 @@ +import { Data, Effect, Schema } from 'effect' + +const SHA256 = /^[a-f0-9]{64}$/ + +const decodeDigestList = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Array(Schema.String)), +) + +export class BlobStoreError extends Data.TaggedError('BlobStoreError')<{ + readonly message: string + readonly cause?: unknown +}> { + override readonly cause?: unknown + + constructor(message: string, cause?: unknown) { + super({ message, cause }) + this.name = 'BlobStoreError' + this.cause = cause + } +} + +export class BlobNotFoundError extends Data.TaggedError('BlobNotFoundError')<{ + readonly digest: string + readonly message: string +}> { + constructor(digest: string) { + super({ digest, message: `blob ${digest} was not found` }) + this.name = 'BlobNotFoundError' + } +} + +export type BlobStoreFailure = BlobStoreError | BlobNotFoundError + +/** + * Content-addressed bytes. `put` is idempotent: the same digest stores one body. + * `retain` and `replaceRefs` record which digests a run still needs. A body is + * deleted only when no run's reference set mentions it. One store should have one writer. + */ +export interface BlobStore { + readonly put: (digest: string, body: Uint8Array) => Effect.Effect + readonly get: (digest: string) => Effect.Effect + readonly retain: ( + runId: string, + digests: ReadonlyArray, + ) => Effect.Effect + readonly replaceRefs: ( + runId: string, + keep: ReadonlySet, + ) => Effect.Effect +} + +export function isSha256Digest(value: string): boolean { + return SHA256.test(value) +} + +/** Byte operations behind {@link blobStoreFromBytes}. Keys are store-relative, not run ids. */ +export interface BlobBytes { + readonly read: (key: string) => Effect.Effect + readonly write: (key: string, body: Uint8Array) => Effect.Effect + readonly remove: (key: string) => Effect.Effect + readonly list: (prefix: string) => Effect.Effect, BlobStoreError> +} + +export function memoryBlobStore(): BlobStore { + return blobStoreFromBytes(memoryBlobBytes()) +} + +export function blobStoreFromBytes(bytes: BlobBytes): BlobStore { + return { + put: (digest, body) => + Effect.gen(function* () { + yield* assertDigest(digest) + + const actual = yield* sha256Hex(body) + + if (actual !== digest) { + return yield* new BlobStoreError(`blob body does not match digest ${digest}`) + } + + const existing = yield* bytes.read(objectKey(digest)) + + if (existing) { + return yield* Effect.void + } + + return yield* bytes.write(objectKey(digest), body) + }), + + get: (digest) => + Effect.gen(function* () { + yield* assertDigest(digest) + + const body = yield* bytes.read(objectKey(digest)) + + if (!body) { + return yield* new BlobNotFoundError(digest) + } + + const actual = yield* sha256Hex(body) + + if (actual !== digest) { + return yield* new BlobStoreError(`blob ${digest} failed its integrity check`) + } + + return body + }), + + retain: (runId, digests) => + Effect.gen(function* () { + const current = yield* readRunDigests(bytes, runId) + const next = new Set(current) + + for (const digest of digests) { + next.add(digest) + } + + if (next.size === current.length) { + return + } + + yield* writeRunDigests(bytes, runId, [...next]) + }), + + replaceRefs: (runId, keep) => + Effect.gen(function* () { + const previous = yield* readRunDigests(bytes, runId) + + yield* writeRunDigests(bytes, runId, [...keep]) + + const dropped = previous.filter((digest) => !keep.has(digest)) + + if (dropped.length === 0) { + return + } + + const referenced = yield* referencedDigests(bytes) + + for (const digest of dropped) { + if (!referenced.has(digest)) { + yield* bytes.remove(objectKey(digest)) + } + } + }), + } +} + +function memoryBlobBytes(): BlobBytes { + const objects = new Map() + + return { + read: (key) => + Effect.sync(() => { + const body = objects.get(key) + + return body ? body.slice() : null + }), + + write: (key, body) => + Effect.sync(() => { + objects.set(key, body.slice()) + }), + + remove: (key) => + Effect.sync(() => { + objects.delete(key) + }), + + list: (prefix) => + Effect.sync(() => [...objects.keys()].filter((key) => key.startsWith(prefix))), + } +} + +function objectKey(digest: string): string { + return `objects/${digest}` +} + +function runKey(runId: string): string { + return `runs/${encodeURIComponent(runId)}` +} + +function assertDigest(digest: string): Effect.Effect { + if (isSha256Digest(digest)) { + return Effect.void + } + + return Effect.fail(new BlobStoreError(`invalid blob digest ${digest}`)) +} + +export function sha256Hex(body: Uint8Array): Effect.Effect { + const copy = new Uint8Array(body.byteLength) + copy.set(body) + + return Effect.tryPromise({ + try: () => crypto.subtle.digest('SHA-256', copy), + catch: (cause) => new BlobStoreError('SHA-256 failed', cause), + }).pipe(Effect.map((buffer) => hex(new Uint8Array(buffer)))) +} + +function hex(bytes: Uint8Array): string { + let encoded = '' + + for (const byte of bytes) { + encoded += byte.toString(16).padStart(2, '0') + } + + return encoded +} + +function readRunDigests( + bytes: BlobBytes, + runId: string, +): Effect.Effect, BlobStoreError> { + return bytes.read(runKey(runId)).pipe( + Effect.flatMap((body) => { + if (!body) { + return Effect.succeed([]) + } + + return decodeDigestList(new TextDecoder().decode(body)).pipe( + Effect.mapError( + (cause) => new BlobStoreError(`blob references for ${runId} are unreadable`, cause), + ), + ) + }), + ) +} + +function writeRunDigests( + bytes: BlobBytes, + runId: string, + digests: ReadonlyArray, +): Effect.Effect { + return bytes.write(runKey(runId), new TextEncoder().encode(JSON.stringify(digests))) +} + +function referencedDigests(bytes: BlobBytes): Effect.Effect, BlobStoreError> { + return bytes.list('runs/').pipe( + Effect.flatMap((keys) => + Effect.forEach(keys, (key) => + bytes.read(key).pipe( + Effect.flatMap((body) => { + if (!body) { + return Effect.succeed([]) + } + + return decodeDigestList(new TextDecoder().decode(body)).pipe( + Effect.mapError( + (cause) => new BlobStoreError('blob references are unreadable', cause), + ), + ) + }), + ), + ), + ), + Effect.map((lists) => { + const digests = new Set() + + for (const list of lists) { + for (const digest of list) { + digests.add(digest) + } + } + + return digests + }), + ) +} diff --git a/packages/core/src/blobs.ts b/packages/core/src/blobs.ts new file mode 100644 index 0000000..1558950 --- /dev/null +++ b/packages/core/src/blobs.ts @@ -0,0 +1,283 @@ +import { Effect, Predicate, Schema } from 'effect' + +import { + BlobStoreError, + isSha256Digest, + sha256Hex, + type BlobStore, + type BlobStoreFailure, +} from './blob-store' +import { utf8JsonBytes } from './limits' +import { isJsonObject, type JsonValue } from './types' + +export const DEFAULT_BLOB_THRESHOLD_BYTES = 64 * 1024 + +const BLOB_KIND = 'blob' +const BLOB_MEDIA_TYPE = 'application/json' +const decodeJsonText = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Json)) + +/** + * Reserved stored shape. A JSON object with exactly these fields is a claim check, + * not caller data. `bytes` is the UTF-8 length of the JSON body named by `sha256`. + */ +export interface BlobRef { + readonly $looms: typeof BLOB_KIND + readonly sha256: string + readonly bytes: number + readonly mediaType: typeof BLOB_MEDIA_TYPE + readonly [key: string]: JsonValue +} + +export function isBlobRef(value: JsonValue): value is BlobRef { + if (!isJsonObject(value) || Object.keys(value).length !== 4) { + return false + } + + const kind = value.$looms + const sha256 = value.sha256 + const bytes = value.bytes + const mediaType = value.mediaType + + return ( + kind === BLOB_KIND && + Predicate.isString(sha256) && + isSha256Digest(sha256) && + Predicate.isNumber(bytes) && + Number.isInteger(bytes) && + bytes >= 0 && + mediaType === BLOB_MEDIA_TYPE + ) +} + +export interface OffloadJsonOptions { + readonly thresholdBytes?: number + /** When false, the root value stays inline even if it is over the threshold. Children are still claimed. */ + readonly claimRoot?: boolean +} + +/** Store `value` and return a reference, even when it is under the offload threshold. */ +export function claimJson( + value: JsonValue, + blobs: BlobStore, +): Effect.Effect { + return putJson(value, blobs) +} + +/** Replace values larger than `thresholdBytes` with {@link BlobRef}s. Bodies are stored by hash. */ +export function offloadJson( + value: JsonValue, + blobs: BlobStore, + options?: OffloadJsonOptions, +): Effect.Effect { + const thresholdBytes = options?.thresholdBytes ?? DEFAULT_BLOB_THRESHOLD_BYTES + + return offloadNode(value, blobs, thresholdBytes, options?.claimRoot ?? true) +} + +/** Replace {@link BlobRef}s with the JSON they name, including references nested inside a body. */ +export function resolveJson( + value: JsonValue, + blobs: BlobStore, +): Effect.Effect { + return resolveNode(value, blobs, new Set()) +} + +/** + * Digests named by `value`, plus digests nested inside those bodies. + * A parent blob that itself contains references still keeps those bodies alive. + */ +export function expandBlobDigests( + blobs: BlobStore, + seeds: ReadonlySet, +): Effect.Effect, BlobStoreFailure> { + if (seeds.size === 0) { + return Effect.succeed(new Set()) + } + + return Effect.gen(function* () { + const all = new Set() + const pending = [...seeds] + + while (pending.length > 0) { + const digest = pending.pop() + + if (digest === undefined || all.has(digest)) { + continue + } + + all.add(digest) + + const body = yield* blobs.get(digest) + const json = yield* decodeBody(body) + const nested = new Set() + + collectBlobDigests(json, nested) + + for (const child of nested) { + if (!all.has(child)) { + pending.push(child) + } + } + } + + return all + }) +} + +/** Record every blob digest embedded in `value`. Does not read blob bodies. */ +export function collectBlobDigests(value: JsonValue, into: Set): void { + if (isBlobRef(value)) { + into.add(value.sha256) + return + } + + if (Array.isArray(value)) { + for (const item of value) { + collectBlobDigests(item, into) + } + + return + } + + if (isJsonObject(value)) { + for (const child of Object.values(value)) { + collectBlobDigests(child, into) + } + } +} + +function offloadNode( + value: JsonValue, + blobs: BlobStore, + thresholdBytes: number, + claim: boolean, +): Effect.Effect { + if (isBlobRef(value)) { + return Effect.succeed(value) + } + + if (Array.isArray(value)) { + return Effect.forEach(value, (item) => offloadNode(item, blobs, thresholdBytes, true)).pipe( + Effect.flatMap((items) => storeIfOver(items, blobs, thresholdBytes, claim)), + ) + } + + if (isJsonObject(value)) { + return offloadObject(value, blobs, thresholdBytes, claim) + } + + return storeIfOver(value, blobs, thresholdBytes, claim) +} + +function offloadObject( + value: { readonly [key: string]: JsonValue }, + blobs: BlobStore, + thresholdBytes: number, + claim: boolean, +): Effect.Effect { + return Effect.gen(function* () { + const next: { [key: string]: JsonValue } = {} + + for (const key of Object.keys(value)) { + const child = value[key] + + if (child === undefined) { + continue + } + + next[key] = yield* offloadNode(child, blobs, thresholdBytes, true) + } + + return yield* storeIfOver(next, blobs, thresholdBytes, claim) + }) +} + +function storeIfOver( + value: JsonValue, + blobs: BlobStore, + thresholdBytes: number, + claim: boolean, +): Effect.Effect { + // Strings only. Objects and arrays stay in place so schema-shaped snapshot state still decodes. + // An event that is still over the byte limit claims its whole payload afterward. + if (!claim || !Predicate.isString(value) || utf8JsonBytes(value) <= thresholdBytes) { + return Effect.succeed(value) + } + + return putJson(value, blobs) +} + +function putJson(value: JsonValue, blobs: BlobStore): Effect.Effect { + const body = new TextEncoder().encode(JSON.stringify(value)) + + return sha256Hex(body).pipe( + Effect.flatMap((sha256) => + blobs.put(sha256, body).pipe( + Effect.map((): BlobRef => ({ + $looms: BLOB_KIND, + sha256, + bytes: body.byteLength, + mediaType: BLOB_MEDIA_TYPE, + })), + ), + ), + ) +} + +function resolveNode( + value: JsonValue, + blobs: BlobStore, + seen: ReadonlySet, +): Effect.Effect { + if (isBlobRef(value)) { + if (seen.has(value.sha256)) { + return Effect.fail(new BlobStoreError(`blob ${value.sha256} refers to itself`)) + } + + const next = new Set(seen) + next.add(value.sha256) + + return blobs.get(value.sha256).pipe( + Effect.flatMap((body) => decodeBody(body)), + Effect.flatMap((decoded) => resolveNode(decoded, blobs, next)), + ) + } + + if (Array.isArray(value)) { + return Effect.forEach(value, (item) => resolveNode(item, blobs, seen)) + } + + if (isJsonObject(value)) { + return resolveObject(value, blobs, seen) + } + + return Effect.succeed(value) +} + +function resolveObject( + value: { readonly [key: string]: JsonValue }, + blobs: BlobStore, + seen: ReadonlySet, +): Effect.Effect { + return Effect.gen(function* () { + const next: { [key: string]: JsonValue } = {} + + for (const key of Object.keys(value)) { + const child = value[key] + + if (child === undefined) { + continue + } + + next[key] = yield* resolveNode(child, blobs, seen) + } + + return next + }) +} + +function decodeBody(body: Uint8Array): Effect.Effect { + return decodeJsonText(new TextDecoder().decode(body)).pipe( + Effect.mapError((cause) => new BlobStoreError('blob body is not JSON', cause)), + ) +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c1ee11b..d92a0e4 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -38,6 +38,9 @@ export * from './serialize' export * from './coalesce' export * from './event-index' export * from './limits' +export * from './blob-store' +export * from './blobs' +export * from './blob-offload' export * from './definition-store' export * from './launcher' export * from './published' diff --git a/packages/runtime/src/blob-offload.test.ts b/packages/runtime/src/blob-offload.test.ts new file mode 100644 index 0000000..a5a9a9e --- /dev/null +++ b/packages/runtime/src/blob-offload.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { + EventPayloadTooLargeError, + makeMemoryEventStore, + makeMemorySnapshotStore, + memoryBlobStore, +} from '@looms/core' +import { defineWorkflow, workflow } from '@looms/workflow' + +import { createLooms } from './looms' + +const echo = defineWorkflow({ + name: 'blob-echo', + nodes: [{ id: 'done', run: () => null }], +}) + +describe('createLooms blob offload', () => { + test('rejects an event over 1 MiB when no blob store is configured', async () => { + const looms = createLooms({ modules: [workflow({ definitions: [echo] })] }) + const input = { blob: 'x'.repeat(1_500_000) } + + const error = await looms.start(echo, input, { runId: 'too-big' }).then( + () => null, + (cause) => cause, + ) + + expect(error).toBeInstanceOf(EventPayloadTooLargeError) + await looms.stop() + }) + + test('runs a payload over the event limit and reloads it from the offloaded snapshot', async () => { + const inner = await Effect.runPromise(makeMemoryEventStore) + const snapshots = await Effect.runPromise(makeMemorySnapshotStore) + const blobs = memoryBlobStore() + const large = 'x'.repeat(1_500_000) + const input = { blob: large } + + const looms = createLooms({ + modules: [workflow({ definitions: [echo] })], + store: inner, + snapshotStore: snapshots, + blobStore: blobs, + }) + + try { + const started = await looms.start(echo, input, { runId: 'big' }) + expect(started.state.status).toBe('completed') + expect(started.state.startIdentity?.input).toEqual(input) + + const rawEvents = JSON.stringify(await Effect.runPromise(inner.read('big'))) + expect(rawEvents).toContain('"$looms":"blob"') + expect(rawEvents).not.toContain(large) + + const rawSnapshot = await Effect.runPromise(snapshots.loadLatest('big')) + expect(JSON.stringify(rawSnapshot)).toContain('"$looms":"blob"') + expect(JSON.stringify(rawSnapshot)).not.toContain(large) + + const events = await looms.getEvents('big') + expect(JSON.stringify(events)).toContain(large) + } finally { + await looms.stop() + } + + const reloaded = createLooms({ + modules: [workflow({ definitions: [echo] })], + store: inner, + snapshotStore: snapshots, + blobStore: blobs, + rescanTimers: false, + }) + + try { + const state = await reloaded.getRun('big') + expect(state.startIdentity?.input).toEqual(input) + } finally { + await reloaded.stop() + } + }) +}) diff --git a/packages/runtime/src/filesystem-blob-store.test.ts b/packages/runtime/src/filesystem-blob-store.test.ts new file mode 100644 index 0000000..fbb8cf4 --- /dev/null +++ b/packages/runtime/src/filesystem-blob-store.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, test } from 'bun:test' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { Effect } from 'effect' + +import { offloadJson, resolveJson } from '@looms/core' + +import { filesystemBlobStore } from './filesystem-blob-store' + +describe('filesystemBlobStore', () => { + test('round-trips an offloaded value', async () => { + const directory = await mkdtemp(join(tmpdir(), 'looms-blobs-')) + + try { + const blobs = filesystemBlobStore(directory) + const value = { text: 'z'.repeat(100) } + const stored = await Effect.runPromise(offloadJson(value, blobs, { thresholdBytes: 16 })) + const resolved = await Effect.runPromise(resolveJson(stored, blobs)) + + expect(resolved).toEqual(value) + expect(JSON.stringify(stored)).toContain('"$looms":"blob"') + } finally { + await rm(directory, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/runtime/src/filesystem-blob-store.ts b/packages/runtime/src/filesystem-blob-store.ts new file mode 100644 index 0000000..f971768 --- /dev/null +++ b/packages/runtime/src/filesystem-blob-store.ts @@ -0,0 +1,84 @@ +import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises' + +import { Data, Effect, Predicate } from 'effect' + +import { blobStoreFromBytes, BlobStoreError, type BlobBytes, type BlobStore } from '@looms/core' + +class MissingBlobFile extends Data.TaggedError('MissingBlobFile') {} + +/** + * Content-addressed blob store rooted at `directory`. For Bun and Node hosts. + * A Durable Object should use `r2BlobStore` instead. One directory should have one writer. + */ +export function filesystemBlobStore(directory: string): BlobStore { + return blobStoreFromBytes(filesystemBlobBytes(directory)) +} + +function filesystemBlobBytes(directory: string): BlobBytes { + return { + read: (key) => readOptional(`${directory}/${key}`, key), + + write: (key, body) => + Effect.tryPromise({ + try: () => mkdir(`${directory}/${folderOf(key)}`, { recursive: true }), + catch: (cause) => + new BlobStoreError(`Unable to create a directory for blob key ${key}`, cause), + }).pipe( + Effect.andThen( + Effect.tryPromise({ + try: () => writeFile(`${directory}/${key}`, body), + catch: (cause) => new BlobStoreError(`Unable to write blob key ${key}`, cause), + }), + ), + ), + + remove: (key) => + Effect.tryPromise({ + try: () => rm(`${directory}/${key}`, { force: true }), + catch: (cause) => new BlobStoreError(`Unable to delete blob key ${key}`, cause), + }), + + list: (prefix) => listFolder(directory, prefix), + } +} + +function readOptional(path: string, key: string): Effect.Effect { + return Effect.tryPromise({ + try: () => readFile(path), + catch: (cause): BlobStoreError | MissingBlobFile => + isMissingFile(cause) + ? new MissingBlobFile() + : new BlobStoreError(`Unable to read blob key ${key}`, cause), + }).pipe( + Effect.catchTag('MissingBlobFile', () => Effect.succeed(null)), + Effect.map((body) => (body === null ? null : new Uint8Array(body))), + ) +} + +function listFolder( + directory: string, + prefix: string, +): Effect.Effect, BlobStoreError> { + const folder = prefix.endsWith('/') ? prefix.slice(0, -1) : prefix + + return Effect.tryPromise({ + try: () => readdir(`${directory}/${folder}`), + catch: (cause): BlobStoreError | MissingBlobFile => + isMissingFile(cause) + ? new MissingBlobFile() + : new BlobStoreError(`Unable to list blob keys ${prefix}`, cause), + }).pipe( + Effect.catchTag('MissingBlobFile', () => Effect.succeed([])), + Effect.map((names) => names.map((name) => `${folder}/${name}`)), + ) +} + +function folderOf(key: string): string { + const slash = key.lastIndexOf('/') + + return slash < 0 ? '' : key.slice(0, slash) +} + +function isMissingFile(cause: unknown): boolean { + return Predicate.isObject(cause) && 'code' in cause && cause.code === 'ENOENT' +} diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index 1f041f8..d9da3e1 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -40,5 +40,6 @@ export { type RunningServer, type LoomsFetchResult, } from './server' +export { filesystemBlobStore } from './filesystem-blob-store' export { createEventStreamResponse, type EventStreamOptions } from './sse' export { notifyObserver, type RuntimeObservation, type RuntimeObserver } from './observer' diff --git a/packages/runtime/src/looms.ts b/packages/runtime/src/looms.ts index a1fc06d..c605a71 100644 --- a/packages/runtime/src/looms.ts +++ b/packages/runtime/src/looms.ts @@ -3,7 +3,9 @@ import { Data, Effect, Predicate } from 'effect' import { EventStoreTag, makeMemoryEventStore, + makeMemorySnapshotStore, snapshotStoreOf, + withBlobOffload, SnapshotStoreTag, type AnyRuntimeModule, type DebuggerHost, @@ -11,6 +13,7 @@ import { type DefinitionRef, type EventEnvelope, type EventInput, + type BlobStore, type EventStore, type EventStoreTrimCoverage, type JsonValue, @@ -51,6 +54,11 @@ export interface CreateLoomsOptions { readonly snapshotEvery?: number /** Defaults to the EventStore's attached store (S2) or an in-memory SnapshotStore. */ readonly snapshotStore?: SnapshotStore + /** + * Content-addressed store for JSON values that would exceed the 1 MiB event limit or the + * 8 MiB snapshot limit. Reads resolve references before folds, projections, and callers see them. + */ + readonly blobStore?: BlobStore readonly trimAfterSnapshot?: { keepSnapshots: number coverage?: (runId: string) => EventStoreTrimCoverage | Promise @@ -175,7 +183,7 @@ export function createLooms(options: CreateLoomsOptions = {}): Looms { Effect.gen(function* () { const configuredStore = options.store - const store = !configuredStore + let store = !configuredStore ? yield* makeMemoryEventStore : Predicate.isFunction(configuredStore) ? yield* Effect.tryPromise({ @@ -189,13 +197,22 @@ export function createLooms(options: CreateLoomsOptions = {}): Looms { }) : configuredStore + if (options.blobStore) { + const snapshots = + options.snapshotStore ?? snapshotStoreOf(store) ?? (yield* makeMemorySnapshotStore) + + store = withBlobOffload(store, options.blobStore, { snapshots }) + } + const runtime = createRuntime({ modules, store, maxWakeIterations: options.maxWakeIterations, maxPendingRunOperations: options.maxPendingRunOperations, snapshotEvery: options.snapshotEvery, - snapshotStore: options.snapshotStore ?? snapshotStoreOf(store), + snapshotStore: options.blobStore + ? snapshotStoreOf(store) + : (options.snapshotStore ?? snapshotStoreOf(store)), trimAfterSnapshot: options.trimAfterSnapshot, runCacheSize: options.runCacheSize, scheduler: options.scheduler, From 2b78886724e4c61157bd4599ed0276eee15844f9 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 11:02:49 +0200 Subject: [PATCH 04/23] feat(runtime)!: resolve definitions through a cached host resolver Start, spawn, and recovery can load a definition that was not registered on the module, pin that version on the run, and fail a missing start before any event is written. BREAKING CHANGE: Evaluator definitions require version, and lookup takes the name and version. defineEvaluator still defaults the version to v1. Co-authored-by: Cursor --- .changeset/definition-resolver.md | 7 + docs/production.md | 15 ++ packages/actor/src/cell.ts | 4 + packages/actor/src/host.ts | 4 + packages/agent/src/definitions-store.ts | 4 +- packages/agent/src/effects.ts | 10 +- packages/agent/src/module.ts | 9 +- packages/cloudflare/src/durable-object.ts | 4 + packages/core/src/definition-resolver.test.ts | 141 ++++++++++++ packages/core/src/definition-resolver.ts | 207 ++++++++++++++++++ packages/core/src/definition-store.ts | 62 +++++- packages/core/src/index.ts | 1 + packages/core/src/module-scope.ts | 6 + packages/core/src/module.ts | 11 + packages/evaluator/src/definitions-store.ts | 13 +- packages/evaluator/src/definitions.ts | 12 +- packages/evaluator/src/effects.ts | 15 +- packages/evaluator/src/module.test.ts | 1 + packages/evaluator/src/module.ts | 10 +- packages/evaluator/src/threads.ts | 4 + .../runtime/src/definition-resolver.test.ts | 129 +++++++++++ packages/runtime/src/dispatch-effect.ts | 4 +- packages/runtime/src/helpers.test.ts | 20 +- packages/runtime/src/helpers.ts | 34 ++- packages/runtime/src/local-effect-executor.ts | 8 +- packages/runtime/src/looms.ts | 7 + packages/runtime/src/repository.ts | 30 ++- packages/runtime/src/runtime.ts | 39 +++- packages/runtime/src/types.ts | 34 ++- packages/runtime/src/wake-engine.ts | 8 +- .../schedule/src/definition-resolver.test.ts | 93 ++++++++ packages/schedule/src/definitions-store.ts | 4 +- packages/schedule/src/effects.ts | 2 +- packages/schedule/src/module.ts | 10 +- packages/schedule/src/threads.ts | 21 +- packages/workflow/src/definitions-store.ts | 4 +- packages/workflow/src/effects.ts | 11 +- packages/workflow/src/module.ts | 11 +- 38 files changed, 903 insertions(+), 106 deletions(-) create mode 100644 .changeset/definition-resolver.md create mode 100644 packages/core/src/definition-resolver.test.ts create mode 100644 packages/core/src/definition-resolver.ts create mode 100644 packages/runtime/src/definition-resolver.test.ts create mode 100644 packages/schedule/src/definition-resolver.test.ts diff --git a/.changeset/definition-resolver.md b/.changeset/definition-resolver.md new file mode 100644 index 0000000..9314e40 --- /dev/null +++ b/.changeset/definition-resolver.md @@ -0,0 +1,7 @@ +--- +'@swirls/looms': minor +--- + +Definitions can be loaded at start, spawn, and recovery through `definitionResolver`, with registered definitions checked first and an LRU cache in front. A missing definition fails the start before any event is written, and the accepted version stays pinned on the run. Evaluator definitions now include `version`, defaulting to `v1`, and lookup takes the name and version. + +BREAKING CHANGE: `EvaluatorDefinition` requires `version`, and evaluator lookup is `(name, version)` instead of `(name)`. `defineEvaluator` still defaults the version to `v1`. diff --git a/docs/production.md b/docs/production.md index 866dee0..0571152 100644 --- a/docs/production.md +++ b/docs/production.md @@ -97,6 +97,21 @@ and schemas needed by active old-version runs. If a new projector is bad, stop delivery, deploy a corrected new version, and rebuild from archive; do not move a checkpoint backward after source events were trimmed. +## Resolving definitions + +Pass `definitionResolver` to `createLooms`, `createRuntime`, an actor host, or +`LoomsDurableObjectConfig` when a definition is not in the module's static list. +`resolve(kind, name, version)` returns the full definition object, or `undefined` +when that version does not exist. Registered module definitions are checked +first. Results, including misses, stay in an LRU cache. + +Start, spawn, and launch resolve before they write `runtime.thread.started`. +A miss fails the start with `UnknownDefinitionError` and writes no events. The +accepted version is pinned on `runtime.run.started` and `runtime.thread.started`. +Before a wake replays a log, those pinned identities are resolved again so fold +can read them synchronously. An unknown workflow version fails that thread. +Evaluator definitions include `version`, which defaults to `v1`. + ## Production configuration Configure durable storage, actor routing, worker and operations tokens, bounded diff --git a/packages/actor/src/cell.ts b/packages/actor/src/cell.ts index b5540b3..4b51324 100644 --- a/packages/actor/src/cell.ts +++ b/packages/actor/src/cell.ts @@ -3,6 +3,7 @@ import { Effect } from 'effect' import { EventStoreTag, type AnyRuntimeModule, + type DefinitionResolver, type EventInput, type EventStore, type EventStoreTrimCoverage, @@ -32,6 +33,8 @@ export interface ActorCellOptions< readonly store: EventStore readonly scheduler?: WakeScheduler readonly modules: TModules + /** Loads definitions that were not registered on a module. See `CreateRuntimeOptions`. */ + readonly definitionResolver?: DefinitionResolver readonly snapshotEvery?: number readonly maxWakeIterations?: number readonly maxPendingRunOperations?: number @@ -78,6 +81,7 @@ export function createActorCell< const runtime = createRuntime({ modules: options.modules, + definitionResolver: options.definitionResolver, store, scheduler, launcher: options.launcher, diff --git a/packages/actor/src/host.ts b/packages/actor/src/host.ts index 7976f53..ed9957d 100644 --- a/packages/actor/src/host.ts +++ b/packages/actor/src/host.ts @@ -2,6 +2,7 @@ import { Data, Effect } from 'effect' import type { AnyRuntimeModule, + DefinitionResolver, EventInput, EventStore, EventStoreTrimCoverage, @@ -54,6 +55,8 @@ export interface LocalActorHostOptions< > { readonly createStore: (runId: string) => EventStore | Promise readonly modules: TModules + /** Loads definitions that were not registered on a module. See `CreateRuntimeOptions`. */ + readonly definitionResolver?: DefinitionResolver readonly snapshotStore?: SnapshotStore readonly snapshotEvery?: number readonly maxWakeIterations?: number @@ -134,6 +137,7 @@ export function createLocalActorHost< store, scheduler, modules: options.modules, + definitionResolver: options.definitionResolver, snapshotStore: options.snapshotStore, snapshotEvery: options.snapshotEvery, maxWakeIterations: options.maxWakeIterations, diff --git a/packages/agent/src/definitions-store.ts b/packages/agent/src/definitions-store.ts index dd1c8f7..aeaca61 100644 --- a/packages/agent/src/definitions-store.ts +++ b/packages/agent/src/definitions-store.ts @@ -17,5 +17,5 @@ export function makeAgentDefinitionStore( return makeDefinitionStore('agent', definitions) } -export const AgentDefinitionsLive = (definitions: ReadonlyArray) => - Layer.succeed(AgentDefinitionsTag, makeAgentDefinitionStore(definitions)) +export const AgentDefinitionsLive = (definitions: AgentDefinitionStore) => + Layer.succeed(AgentDefinitionsTag, definitions) diff --git a/packages/agent/src/effects.ts b/packages/agent/src/effects.ts index 10f7d88..ebd9cf7 100644 --- a/packages/agent/src/effects.ts +++ b/packages/agent/src/effects.ts @@ -4,6 +4,7 @@ import { createThreadId, DEFAULT_DEFINITION_VERSION, validateInputEffect, + type DefinitionResolveError, type EventInputOf, type JsonValue, } from '@looms/core' @@ -86,7 +87,7 @@ function runCallLlm(args: { emit: (event: EventInputOf) => Promise }): Effect.Effect< ReadonlyArray>, - AgentExecutionError | LlmError, + AgentExecutionError | LlmError | DefinitionResolveError, LlmTag | AgentDefinitionsTag > { return Effect.gen(function* () { @@ -104,7 +105,10 @@ function runCallLlm(args: { const agents = yield* AgentDefinitionsTag const llm = yield* LlmTag - const definition = definitionName ? agents.get(definitionName, definitionVersion) : undefined + + const definition = definitionName + ? yield* agents.resolve(definitionName, definitionVersion) + : undefined if (!definition) { return [ @@ -240,7 +244,7 @@ export const executeToolEffect = agentModule.effect({ const agents = yield* AgentDefinitionsTag const definition = input.definitionName - ? agents.get(input.definitionName, input.definitionVersion) + ? yield* agents.resolve(input.definitionName, input.definitionVersion) : undefined if (!definition) { diff --git a/packages/agent/src/module.ts b/packages/agent/src/module.ts index fe7d98d..95854d1 100644 --- a/packages/agent/src/module.ts +++ b/packages/agent/src/module.ts @@ -21,8 +21,9 @@ export interface AgentModuleOptions { export function agent(options: AgentModuleOptions = {}) { const definitions = options.definitions ?? [] - const agentDefinitions = definitions.filter( - (definition): definition is AgentDefinition => definition.kind === 'agent', + const agentDefinitions = makeDefinitionStore( + 'agent', + definitions.filter((definition): definition is AgentDefinition => definition.kind === 'agent'), ) const sessionDefinitions = makeDefinitionStore( @@ -53,6 +54,10 @@ export function agent(options: AgentModuleOptions = {}) { }, services: () => Layer.merge(llmLayer, AgentDefinitionsLive(agentDefinitions)), interruptsOnSignal, + bindHost(host) { + agentDefinitions.useResolver(host.definitionResolver) + sessionDefinitions.useResolver(host.definitionResolver) + }, })) } diff --git a/packages/cloudflare/src/durable-object.ts b/packages/cloudflare/src/durable-object.ts index d5746df..d60a287 100644 --- a/packages/cloudflare/src/durable-object.ts +++ b/packages/cloudflare/src/durable-object.ts @@ -5,6 +5,7 @@ import { createActorCell, type ActorCell } from '@looms/actor' import { createRemoteRunLauncher, type AnyRuntimeModule, + type DefinitionResolver, type EventInput, type EventStoreTrimCoverage, type LaunchRunArgs, @@ -21,6 +22,8 @@ import type { DurableObjectNamespaceLike } from './router' export interface LoomsDurableObjectConfig { readonly modules: readonly AnyRuntimeModule[] + /** Loads definitions that were not registered on a module. See `CreateRuntimeOptions`. */ + readonly definitionResolver?: DefinitionResolver readonly projectors?: readonly Projector[] readonly snapshotEvery?: number readonly maxWakeIterations?: number @@ -96,6 +99,7 @@ export abstract class LoomsDurableObject extends DurableObject) { + return { + ...createEvent('run_1', { + type, + payload, + origin: { type: 'system' as const }, + }), + seq: 1, + } +} + +describe('definition resolver', () => { + test('static lookup returns the registered object and misses as undefined', async () => { + const resolver = staticDefinitionResolver([checkout]) + const found = await Effect.runPromise(resolver.resolve('workflow', 'checkout', 'v1')) + + expect(found).toBe(checkout) + expect(resolver.peek('workflow', 'refund', 'v2')).toBeUndefined() + expect(await Effect.runPromise(resolver.resolve('workflow', 'refund', 'v2'))).toBeUndefined() + }) + + test('chain stops at the first definition', async () => { + let calls = 0 + + const second = { + peek: () => undefined, + resolve: () => + Effect.sync(() => { + calls += 1 + return refund + }), + } + + const resolver = chainDefinitionResolvers([staticDefinitionResolver([checkout]), second]) + const found = await Effect.runPromise(resolver.resolve('workflow', 'checkout', 'v1')) + + expect(found).toBe(checkout) + expect(calls).toBe(0) + expect(await Effect.runPromise(resolver.resolve('workflow', 'refund', 'v2'))).toBe(refund) + expect(calls).toBe(1) + }) + + test('cache returns a miss without fetching again and evicts the oldest entry', async () => { + const fetches: string[] = [] + + const inner = { + peek: () => undefined, + resolve: (kind: string, name: string, version: string) => + Effect.sync(() => { + fetches.push(`${kind}:${name}@${version}`) + + if (name === 'missing') { + return undefined + } + + return { kind, name, version } + }), + } + + const resolver = cachedDefinitionResolver(inner, { capacity: 1 }) + + expect(await Effect.runPromise(resolver.resolve('workflow', 'missing', 'v1'))).toBeUndefined() + expect(await Effect.runPromise(resolver.resolve('workflow', 'missing', 'v1'))).toBeUndefined() + expect(fetches).toEqual(['workflow:missing@v1']) + expect(resolver.peek('workflow', 'missing', 'v1')).toBeUndefined() + + await Effect.runPromise(resolver.resolve('workflow', 'checkout', 'v1')) + expect(resolver.peek('workflow', 'missing', 'v1')).toBeUndefined() + expect(resolver.peek('workflow', 'checkout', 'v1')?.name).toBe('checkout') + + await Effect.runPromise(resolver.resolve('workflow', 'missing', 'v1')) + expect(fetches).toEqual(['workflow:missing@v1', 'workflow:checkout@v1', 'workflow:missing@v1']) + }) + + test('a store peeks a primed resolver and keeps its own definitions first', async () => { + const dynamic = { kind: 'workflow', name: 'dynamic', version: 'v9', nodes: ['a'] } + + const resolver = cachedDefinitionResolver({ + peek: () => undefined, + resolve: (kind: string, name: string, version: string) => + Effect.succeed( + kind === 'workflow' && name === 'dynamic' && version === 'v9' ? dynamic : undefined, + ), + }) + + const store = makeDefinitionStore('workflow', [checkout]) + store.useResolver(resolver) + + expect(store.get('checkout', 'v1')).toBe(checkout) + expect(store.get('dynamic', 'v9')).toBeUndefined() + expect(await Effect.runPromise(store.resolve('dynamic', 'v9'))).toBe(dynamic) + expect(store.get('dynamic', 'v9')).toMatchObject({ nodes: ['a'] }) + }) + + test('pinned refs follow the start events and default a missing version', () => { + const refs = pinnedDefinitionRefs([ + started('runtime.run.started', { + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v2', + }), + started('runtime.thread.started', { + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v2', + }), + started('runtime.thread.started', { kind: 'agent', definitionName: 'helper' }), + started('workflow.node.started', { nodeId: 'a' }), + ]) + + expect(refs).toEqual([ + { kind: 'workflow', name: 'checkout', version: 'v2' }, + { kind: 'agent', name: 'helper', version: 'v1' }, + ]) + }) +}) diff --git a/packages/core/src/definition-resolver.ts b/packages/core/src/definition-resolver.ts new file mode 100644 index 0000000..c10e9d9 --- /dev/null +++ b/packages/core/src/definition-resolver.ts @@ -0,0 +1,207 @@ +import { Data, Effect, Predicate } from 'effect' + +import type { VersionedDefinition } from './definition-store' +import type { EventEnvelope } from './envelope' +import type { JsonValue } from './types' +import { DEFAULT_DEFINITION_VERSION, definitionKey } from './module' + +export class DefinitionResolveError extends Data.TaggedError('DefinitionResolveError')<{ + readonly kind: string + readonly name: string + readonly version: string + readonly message: string +}> { + constructor(kind: string, name: string, version: string, message: string) { + super({ kind, name, version, message }) + this.name = 'DefinitionResolveError' + } +} + +/** + * Loads one definition by the identity pinned on a run. + * `peek` is synchronous and only returns a definition this resolver already holds. + * `resolve` may fetch, and it caches through {@link cachedDefinitionResolver}. + */ +export interface DefinitionResolver { + readonly peek: (kind: string, name: string, version: string) => VersionedDefinition | undefined + readonly resolve: ( + kind: string, + name: string, + version: string, + ) => Effect.Effect +} + +export const DEFAULT_DEFINITION_CACHE_CAPACITY = 256 + +/** Definitions registered on a module. A miss is `undefined`, not an error. */ +export function staticDefinitionResolver( + definitions: ReadonlyArray, +): DefinitionResolver { + const byKey = new Map( + definitions.map( + (definition) => + [definitionKey(definition.kind, definition.name, definition.version), definition] as const, + ), + ) + + const lookup = (kind: string, name: string, version: string) => + byKey.get(definitionKey(kind, name, version)) + + return { + peek: lookup, + resolve: (kind, name, version) => Effect.succeed(lookup(kind, name, version)), + } +} + +/** First resolver that returns a definition wins. Later resolvers are not called. */ +export function chainDefinitionResolvers( + resolvers: ReadonlyArray, +): DefinitionResolver { + return { + peek(kind, name, version) { + for (const resolver of resolvers) { + const found = resolver.peek(kind, name, version) + + if (found) { + return found + } + } + + return undefined + }, + resolve(kind, name, version) { + return Effect.gen(function* () { + for (const resolver of resolvers) { + const found = yield* resolver.resolve(kind, name, version) + + if (found) { + return found + } + } + + return undefined + }) + }, + } +} + +/** + * Remembers successful lookups and misses. A miss stays cached so recovery does not + * fetch a deleted definition on every wake. Failures are not cached. + */ +export function cachedDefinitionResolver( + inner: DefinitionResolver, + options?: { readonly capacity?: number }, +): DefinitionResolver { + const capacity = options?.capacity ?? DEFAULT_DEFINITION_CACHE_CAPACITY + const entries = new Map() + + const remember = (key: string, value: VersionedDefinition | undefined): void => { + entries.delete(key) + entries.set(key, value) + + while (entries.size > capacity) { + const oldest = entries.keys().next().value + + if (oldest === undefined) { + return + } + + entries.delete(oldest) + } + } + + return { + peek(kind, name, version) { + const key = definitionKey(kind, name, version) + + if (!entries.has(key)) { + return inner.peek(kind, name, version) + } + + const value = entries.get(key) + remember(key, value) + return value + }, + resolve(kind, name, version) { + const key = definitionKey(kind, name, version) + + if (entries.has(key)) { + const value = entries.get(key) + remember(key, value) + return Effect.succeed(value) + } + + return inner.resolve(kind, name, version).pipe( + Effect.tap((value) => + Effect.sync(() => { + remember(key, value) + }), + ), + ) + }, + } +} + +/** Identities pinned by `runtime.run.started` and `runtime.thread.started`. */ +export function pinnedDefinitionRefs( + events: readonly EventEnvelope[], +): readonly VersionedDefinition[] { + const refs: VersionedDefinition[] = [] + const seen = new Set() + + for (const event of events) { + if (event.type !== 'runtime.run.started' && event.type !== 'runtime.thread.started') { + continue + } + + const pinned = readPinned(event.payload) + + if (!pinned) { + continue + } + + const key = definitionKey(pinned.kind, pinned.name, pinned.version) + + if (seen.has(key)) { + continue + } + + seen.add(key) + refs.push(pinned) + } + + return refs +} + +/** Resolves every pinned identity so a later synchronous lookup can see it. */ +export function primePinnedDefinitions( + resolver: DefinitionResolver, + events: readonly EventEnvelope[], +): Effect.Effect { + return Effect.gen(function* () { + for (const ref of pinnedDefinitionRefs(events)) { + yield* resolver.resolve(ref.kind, ref.name, ref.version) + } + }) +} + +function readPinned(payload: JsonValue): VersionedDefinition | undefined { + if (!Predicate.isObject(payload) || !('kind' in payload) || !('definitionName' in payload)) { + return undefined + } + + const kind = payload.kind + const name = payload.definitionName + const version = 'definitionVersion' in payload ? payload.definitionVersion : undefined + + if (!Predicate.isString(kind) || !Predicate.isString(name)) { + return undefined + } + + return { + kind, + name, + version: Predicate.isString(version) ? version : DEFAULT_DEFINITION_VERSION, + } +} diff --git a/packages/core/src/definition-store.ts b/packages/core/src/definition-store.ts index c79cfe6..dae99ee 100644 --- a/packages/core/src/definition-store.ts +++ b/packages/core/src/definition-store.ts @@ -1,3 +1,6 @@ +import { Effect } from 'effect' + +import type { DefinitionResolveError, DefinitionResolver } from './definition-resolver' import { definitionKey } from './module' export interface VersionedDefinition { @@ -9,6 +12,13 @@ export interface VersionedDefinition { /** Lookup of one definition kind by name and version. */ export interface DefinitionStore { get(name: string, version: string): TDefinition | undefined + /** Static definition, or a definition the bound resolver has already loaded. */ + resolve( + name: string, + version: string, + ): Effect.Effect + /** Called by the runtime before the module's services are built. */ + useResolver(resolver: DefinitionResolver): void } /** Indexes definitions of one kind by `definitionKey`. */ @@ -23,5 +33,55 @@ export function makeDefinitionStore( ), ) - return { get: (name, version) => byKey.get(definitionKey(kind, name, version)) } + let resolver: DefinitionResolver | undefined + + const get = (name: string, version: string): TDefinition | undefined => { + const stored = byKey.get(definitionKey(kind, name, version)) + + if (stored) { + return stored + } + + const peeked = resolver?.peek(kind, name, version) + + if (peeked && isStoredDefinition(kind, peeked)) { + return peeked + } + + return undefined + } + + return { + get, + useResolver(next) { + resolver = next + }, + resolve(name, version) { + const stored = get(name, version) + + if (stored) { + return Effect.succeed(stored) + } + + if (!resolver) { + // Effect.void is Effect, which is not a missing definition. + // oxlint-disable-next-line effecttsgo/effect-succeed-with-void + return Effect.succeed(undefined) + } + + return resolver.resolve(kind, name, version).pipe( + Effect.map((found) => + found && isStoredDefinition(kind, found) ? found : undefined, + ), + ) + }, + } +} + +// oxlint-disable-next-line typescript/no-unnecessary-type-parameters -- the predicate returns the store's definition type +function isStoredDefinition( + kind: string, + value: VersionedDefinition, +): value is TDefinition { + return value.kind === kind } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c1ee11b..6117a30 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -39,6 +39,7 @@ export * from './coalesce' export * from './event-index' export * from './limits' export * from './definition-store' +export * from './definition-resolver' export * from './launcher' export * from './published' export type { DebuggerHost, DebuggerRequest } from './debugger-host' diff --git a/packages/core/src/module-scope.ts b/packages/core/src/module-scope.ts index 4ac8898..02c0660 100644 --- a/packages/core/src/module-scope.ts +++ b/packages/core/src/module-scope.ts @@ -24,6 +24,7 @@ import type { EventEnvelope, EventInput } from './envelope' import type { RunLauncherTag } from './launcher' import { type AnyRuntimeModule, + type DefinitionHost, type EffectMiddleware, type RuntimeModule, type DefinitionRef, @@ -255,6 +256,7 @@ export function defineModule< readonly definitions?: readonly DefinitionRef[] readonly middleware?: readonly EffectMiddleware[] readonly interruptsOnSignal?: SignalInterruptRule + readonly bindHost?: (host: DefinitionHost) => void } & ModuleServicesConfig>, ): RuntimeModule, EventCatalog>, FX, TH, PR> & { readonly effects: FX @@ -308,5 +310,9 @@ export function defineModule(options: any, setup: (scope: any) => any): any { module.interruptsOnSignal = members.interruptsOnSignal } + if (members.bindHost) { + module.bindHost = members.bindHost + } + return module } diff --git a/packages/core/src/module.ts b/packages/core/src/module.ts index 752010f..429d7da 100644 --- a/packages/core/src/module.ts +++ b/packages/core/src/module.ts @@ -1,6 +1,7 @@ import type { Context, Effect, Layer } from 'effect' import type { EventCatalog } from './catalog' +import type { DefinitionResolver } from './definition-resolver' import type { EffectContext, EffectDefinition, RuntimeEffect } from './effects' import type { EventInput } from './envelope' import type { ProjectionDefinition } from './projection' @@ -26,6 +27,11 @@ export type EffectMiddleware = ( */ export type SignalInterruptRule = (event: EventInput) => readonly string[] +/** Passed to a module once, before its services layer is constructed. */ +export interface DefinitionHost { + readonly definitionResolver: DefinitionResolver +} + export interface RuntimeModuleDependency { readonly namespace: string readonly protocolVersion?: string @@ -91,6 +97,11 @@ export interface RuntimeModule< readonly interruptsOnSignal?: SignalInterruptRule /** Host-side services this module's effect handlers need (an LLM, a definition lookup, a DB pool). */ readonly services?: () => Layer.Layer + /** + * Receives the host's definition resolver before `services()` runs. + * Modules that look definitions up during fold share one store with this resolver. + */ + readonly bindHost?: (host: DefinitionHost) => void } export function defineRuntimeModule< diff --git a/packages/evaluator/src/definitions-store.ts b/packages/evaluator/src/definitions-store.ts index 2aa44f6..13e86f9 100644 --- a/packages/evaluator/src/definitions-store.ts +++ b/packages/evaluator/src/definitions-store.ts @@ -1,10 +1,10 @@ import { Context, Layer } from 'effect' +import { makeDefinitionStore, type DefinitionStore } from '@looms/core' + import type { EvaluatorDefinition } from './definitions' -export interface EvaluatorDefinitionStore { - get(name: string): EvaluatorDefinition | undefined -} +export type EvaluatorDefinitionStore = DefinitionStore export class EvaluatorDefinitionsTag extends Context.Service< EvaluatorDefinitionsTag, @@ -14,9 +14,8 @@ export class EvaluatorDefinitionsTag extends Context.Service< export function makeEvaluatorDefinitionStore( definitions: ReadonlyArray, ): EvaluatorDefinitionStore { - const map = new Map(definitions.map((def) => [def.name, def])) - return { get: (name) => map.get(name) } + return makeDefinitionStore('evaluator', definitions) } -export const EvaluatorDefinitionsLive = (definitions: ReadonlyArray) => - Layer.succeed(EvaluatorDefinitionsTag, makeEvaluatorDefinitionStore(definitions)) +export const EvaluatorDefinitionsLive = (definitions: EvaluatorDefinitionStore) => + Layer.succeed(EvaluatorDefinitionsTag, definitions) diff --git a/packages/evaluator/src/definitions.ts b/packages/evaluator/src/definitions.ts index 233773c..83da692 100644 --- a/packages/evaluator/src/definitions.ts +++ b/packages/evaluator/src/definitions.ts @@ -1,7 +1,7 @@ import type { StandardSchemaV1 } from '@standard-schema/spec' import type { Schema } from 'effect' -import { defineKind, type JsonValue } from '@looms/core' +import { DEFAULT_DEFINITION_VERSION, defineKind, type JsonValue } from '@looms/core' import type { EvaluatorQuestions, @@ -13,6 +13,7 @@ import type { export interface EvaluatorDefinition { readonly kind: 'evaluator' readonly name: TName + readonly version: string readonly description?: string /** Provider model id. Omit it to use the adapter default. */ readonly model?: string @@ -26,7 +27,12 @@ export interface EvaluatorDefinition( - def: Omit, 'kind'>, + def: Omit, 'kind' | 'version'> & { + readonly version?: string + }, ): EvaluatorDefinition { - return defineKind('evaluator', def) + return defineKind('evaluator', { + ...def, + version: def.version ?? DEFAULT_DEFINITION_VERSION, + }) } diff --git a/packages/evaluator/src/effects.ts b/packages/evaluator/src/effects.ts index f3c4bd6..83311f8 100644 --- a/packages/evaluator/src/effects.ts +++ b/packages/evaluator/src/effects.ts @@ -1,6 +1,12 @@ import { Effect, Schema } from 'effect' -import { stringifyJson, validateInputEffect, type EventInputOf, type JsonValue } from '@looms/core' +import { + DEFAULT_DEFINITION_VERSION, + stringifyJson, + validateInputEffect, + type EventInputOf, + type JsonValue, +} from '@looms/core' import { EvaluatorDefinitionsTag } from './definitions-store' import { EvaluatorTag } from './evaluator' @@ -10,6 +16,7 @@ import type { EvaluatorAnswers, EvaluatorEvaluateResult, ResolvedEvaluatorDecisi const EvaluateInput = Schema.Struct({ evaluationId: Schema.String, definitionName: Schema.optional(Schema.String), + definitionVersion: Schema.optional(Schema.String), input: Schema.optional(Schema.Json), }) @@ -47,7 +54,11 @@ export const evaluateEffect = evaluatorModule.effect({ const threadId = ctx.threadId const definitions = yield* EvaluatorDefinitionsTag const evaluator = yield* EvaluatorTag - const definition = input.definitionName ? definitions.get(input.definitionName) : undefined + const version = input.definitionVersion ?? DEFAULT_DEFINITION_VERSION + + const definition = input.definitionName + ? yield* definitions.resolve(input.definitionName, version) + : undefined if (!definition) { const name = input.definitionName ?? 'unknown' diff --git a/packages/evaluator/src/module.test.ts b/packages/evaluator/src/module.test.ts index 5f322b1..89088a0 100644 --- a/packages/evaluator/src/module.test.ts +++ b/packages/evaluator/src/module.test.ts @@ -76,6 +76,7 @@ describe('@looms/evaluator module', () => { test('defineEvaluator sets kind', () => { expect(scoreRefund.kind).toBe('evaluator') expect(scoreRefund.name).toBe('score-refund') + expect(scoreRefund.version).toBe('v1') }) test('composes and exposes evaluate handler', () => { diff --git a/packages/evaluator/src/module.ts b/packages/evaluator/src/module.ts index 96cdcbc..3004df0 100644 --- a/packages/evaluator/src/module.ts +++ b/packages/evaluator/src/module.ts @@ -1,6 +1,6 @@ import { Layer } from 'effect' -import { defineModule } from '@looms/core' +import { defineModule, makeDefinitionStore } from '@looms/core' import type { EvaluatorDefinition } from './definitions' import { EvaluatorDefinitionsLive } from './definitions-store' @@ -24,14 +24,15 @@ export interface EvaluatorModuleOptions { } export function evaluator(options: EvaluatorModuleOptions = {}) { - const definitions = options.definitions ?? [] + const listed = options.definitions ?? [] + const definitions = makeDefinitionStore('evaluator', listed) const evaluatorLayer = options.evaluator ? Layer.succeed(EvaluatorTag, evaluatorFromAdapter(options.evaluator)) : StubEvaluatorLive(options.evaluatorPolicy) return defineModule(evaluatorModule, () => ({ - definitions, + definitions: listed, threads: { evaluator: evaluatorThread }, effects: { evaluate: evaluateEffect, @@ -40,5 +41,8 @@ export function evaluator(options: EvaluatorModuleOptions = {}) { evaluations, }, services: () => Layer.merge(evaluatorLayer, EvaluatorDefinitionsLive(definitions)), + bindHost(host) { + definitions.useResolver(host.definitionResolver) + }, })) } diff --git a/packages/evaluator/src/threads.ts b/packages/evaluator/src/threads.ts index 737d8a4..8905e54 100644 --- a/packages/evaluator/src/threads.ts +++ b/packages/evaluator/src/threads.ts @@ -17,6 +17,7 @@ import { evaluatorModule } from './scope' export const EvaluatorThreadStateSchema = Schema.Struct({ definitionName: Schema.optional(Schema.String), + definitionVersion: Schema.optional(Schema.String), input: Schema.Json, evaluationId: Schema.String, dispatched: Schema.Boolean, @@ -42,6 +43,7 @@ export const evaluatorThread = evaluatorModule.thread({ shape: EvaluatorThreadStateSchema, initialState: (ctx) => ({ definitionName: ctx.definitionName, + definitionVersion: ctx.definitionVersion, input: ctx.input, evaluationId: `eval_${ctx.threadId}`, dispatched: false, @@ -59,6 +61,7 @@ export const evaluatorThread = evaluatorModule.thread({ return { ...state, definitionName: event.payload.definitionName || state.definitionName, + definitionVersion: event.payload.definitionVersion || state.definitionVersion, input: event.payload.input ?? state.input, evaluationId: `eval_${ctx.threadId}`, } @@ -139,6 +142,7 @@ export const evaluatorThread = evaluatorModule.thread({ { evaluationId: state.evaluationId, definitionName: state.definitionName, + definitionVersion: state.definitionVersion, input: state.input, }, `evaluate_${state.evaluationId}`, diff --git a/packages/runtime/src/definition-resolver.test.ts b/packages/runtime/src/definition-resolver.test.ts new file mode 100644 index 0000000..2922f0c --- /dev/null +++ b/packages/runtime/src/definition-resolver.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { + makeMemoryEventStore, + type DefinitionResolver, + type RunSummary, + type SnapshotStore, +} from '@looms/core' +import { defineWorkflow, workflow } from '@looms/workflow' + +import { UnknownDefinitionError } from './helpers' +import { createLooms } from './looms' + +function discardingSnapshots(): SnapshotStore { + const headers: RunSummary[] = [] + + return { + loadLatest: () => Effect.succeedNone, + save: () => Effect.void, + saveHeader: (header) => + Effect.sync(() => { + const index = headers.findIndex((item) => item.runId === header.runId) + + if (index >= 0) { + headers[index] = header + return + } + + headers.push(header) + }), + get listHeaders() { + return Effect.succeed(headers) + }, + } +} + +describe('definition resolver', () => { + test('runs a workflow supplied by the host and resolves it again on replay', async () => { + const dynamic = defineWorkflow({ + name: 'dynamic', + version: 'v2', + nodes: [{ id: 'n', run: () => 'ok' }], + }) + + let calls = 0 + + const resolver: DefinitionResolver = { + peek: () => undefined, + resolve: (kind, name, version) => + Effect.sync(() => { + calls += 1 + + if (kind === 'workflow' && name === 'dynamic' && version === 'v2') { + return dynamic + } + + return undefined + }), + } + + const store = await Effect.runPromise(makeMemoryEventStore) + const snapshots = discardingSnapshots() + + const looms = createLooms({ + modules: [workflow()], + store, + snapshotStore: snapshots, + definitionResolver: resolver, + rescanTimers: false, + }) + + try { + const started = await looms.start(dynamic, null, { runId: 'dyn' }) + expect(started.state.status).toBe('completed') + expect(started.state.startIdentity?.definitionVersion).toBe('v2') + const afterStart = calls + + const reloaded = createLooms({ + modules: [workflow()], + store, + snapshotStore: snapshots, + definitionResolver: resolver, + rescanTimers: false, + }) + + try { + const state = await reloaded.getRun('dyn') + expect(state.status).toBe('completed') + expect(state.startIdentity?.definitionVersion).toBe('v2') + expect(calls).toBeGreaterThan(afterStart) + } finally { + await reloaded.stop() + } + } finally { + await looms.stop() + } + }) + + test('rejects an unknown definition before writing events', async () => { + const missing = defineWorkflow({ + name: 'missing', + nodes: [{ id: 'n', run: () => null }], + }) + + const store = await Effect.runPromise(makeMemoryEventStore) + + const looms = createLooms({ + modules: [workflow()], + store, + rescanTimers: false, + }) + + try { + const outcome = await looms.start(missing, null, { runId: 'missing' }).then( + () => undefined, + (cause: unknown) => cause, + ) + + expect(outcome).toBeInstanceOf(UnknownDefinitionError) + + const events = await Effect.runPromise(store.read('missing')) + expect(events).toHaveLength(0) + } finally { + await looms.stop() + } + }) +}) diff --git a/packages/runtime/src/dispatch-effect.ts b/packages/runtime/src/dispatch-effect.ts index 93a533d..18b4da4 100644 --- a/packages/runtime/src/dispatch-effect.ts +++ b/packages/runtime/src/dispatch-effect.ts @@ -9,7 +9,7 @@ import { type EffectContext, type EventInput, RunLauncherTag, - type RegisteredDefinition, + type DefinitionResolver, type RunLauncher, type RuntimeEffect, } from '@looms/core' @@ -19,7 +19,7 @@ import { threadStartedEvents } from './helpers' export function dispatchEffect( registry: ComposedRegistry, services: ManagedRuntime.ManagedRuntime, - definitions: ReadonlyMap, + definitions: DefinitionResolver, launcher: RunLauncher, effect: RuntimeEffect, ctx: EffectContext, diff --git a/packages/runtime/src/helpers.test.ts b/packages/runtime/src/helpers.test.ts index 81b5913..9d0b432 100644 --- a/packages/runtime/src/helpers.test.ts +++ b/packages/runtime/src/helpers.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from 'bun:test' import { Effect, Schema } from 'effect' -import { DEFAULT_DEFINITION_VERSION, definitionKey } from '@looms/core' +import { DEFAULT_DEFINITION_VERSION, staticDefinitionResolver } from '@looms/core' import { threadStartedEvents } from './helpers' @@ -15,12 +15,8 @@ describe('threadStartedEvents', () => { value: { kind: 'workflow', name: 'checkout', version: DEFAULT_DEFINITION_VERSION }, } - const definitions = new Map([ - [definitionKey('workflow', 'checkout', DEFAULT_DEFINITION_VERSION), def], - ]) - const events = await Effect.runPromise( - threadStartedEvents(definitions, { + threadStartedEvents(staticDefinitionResolver([def]), { kind: 'workflow', definitionName: 'checkout', definitionVersion: DEFAULT_DEFINITION_VERSION, @@ -43,12 +39,10 @@ describe('threadStartedEvents', () => { }) }) - test('rejects unknown definitions with explicit error', async () => { - const definitions = new Map() - + test('rejects unknown definitions with explicit error', () => { expect( Effect.runPromise( - threadStartedEvents(definitions, { + threadStartedEvents(staticDefinitionResolver([]), { kind: 'workflow', definitionName: 'checkout', definitionVersion: DEFAULT_DEFINITION_VERSION, @@ -69,12 +63,8 @@ describe('threadStartedEvents', () => { value: { kind: 'workflow', name: 'checkout', version: DEFAULT_DEFINITION_VERSION }, } - const definitions = new Map([ - [definitionKey('workflow', 'checkout', DEFAULT_DEFINITION_VERSION), def], - ]) - const events = await Effect.runPromise( - threadStartedEvents(definitions, { + threadStartedEvents(staticDefinitionResolver([def]), { kind: 'workflow', definitionName: 'checkout', definitionVersion: DEFAULT_DEFINITION_VERSION, diff --git a/packages/runtime/src/helpers.ts b/packages/runtime/src/helpers.ts index c3d4961..d7b6e34 100644 --- a/packages/runtime/src/helpers.ts +++ b/packages/runtime/src/helpers.ts @@ -21,10 +21,13 @@ import { type EventOrigin, type EventStore, InvalidEventError, + type DefinitionResolveError, + type DefinitionResolver, type JsonValue, type OutstandingEffect, - type RegisteredDefinition, type RunCursor, + type SchemaInput, + type VersionedDefinition, type RunState, type SnapshotStore, } from '@looms/core' @@ -178,7 +181,7 @@ export function groupOutstanding( * event with raw input. */ export function threadStartedEvents( - definitions: ReadonlyMap, + definitions: DefinitionResolver, args: { kind: string definitionName: string @@ -187,11 +190,9 @@ export function threadStartedEvents( threadId: string parentThreadId: string | null }, -): Effect.Effect, UnknownDefinitionError> { +): Effect.Effect, UnknownDefinitionError | DefinitionResolveError> { return Effect.gen(function* () { - const def = definitions.get( - definitionKey(args.kind, args.definitionName, args.definitionVersion), - ) + const def = yield* definitions.resolve(args.kind, args.definitionName, args.definitionVersion) if (!def) { return yield* new UnknownDefinitionError( @@ -204,9 +205,10 @@ export function threadStartedEvents( const raw = args.input ?? null let startedInput = raw let validationError: string | undefined + const input = schemaInputOf(def) - if (def?.input) { - const validated = yield* validateInputEffect(def.input, raw).pipe( + if (input) { + const validated = yield* validateInputEffect(input, raw).pipe( Effect.match({ onFailure: (error) => ({ ok: false as const, error: error.message }), onSuccess: (value) => ({ ok: true as const, value }), @@ -249,6 +251,22 @@ export function threadStartedEvents( }) } +function schemaInputOf(definition: VersionedDefinition): SchemaInput | undefined { + if (!('input' in definition)) { + return undefined + } + + return isSchemaInput(definition.input) ? definition.input : undefined +} + +function isSchemaInput(value: unknown): value is SchemaInput { + if (value === null || (!Predicate.isObject(value) && !Predicate.isFunction(value))) { + return false + } + + return '~standard' in value || 'ast' in value +} + export function synthesizedThreadFailed( before: RunState, after: RunState, diff --git a/packages/runtime/src/local-effect-executor.ts b/packages/runtime/src/local-effect-executor.ts index e611713..81d978b 100644 --- a/packages/runtime/src/local-effect-executor.ts +++ b/packages/runtime/src/local-effect-executor.ts @@ -9,8 +9,8 @@ import { type ComposedRegistry, type EventEnvelope, type EventInput, + type DefinitionResolver, type OutstandingEffect, - type RegisteredDefinition, type RunLauncher, type RunLaunchSource, } from '@looms/core' @@ -31,7 +31,7 @@ import type { RunExecutionContext } from './run-execution-context' export interface LocalEffectExecutorOptions { readonly registry: ComposedRegistry readonly services: ManagedRuntime.ManagedRuntime - readonly definitions: ReadonlyMap + readonly definitionResolver: DefinitionResolver readonly executionContext: RunExecutionContext readonly liveAppends: RuntimeRepository['liveAppends'] readonly runInContext: (effect: Effect.Effect) => Promise @@ -129,7 +129,7 @@ export function executeLocalEffects( const { registry, services, - definitions, + definitionResolver, executionContext, liveAppends, runInContext, @@ -182,7 +182,7 @@ export function executeLocalEffects( const dispatched = dispatchEffect( registry, services, - definitions, + definitionResolver, launcherFor(launcher, input.runId, input.correlationId, item, launched), item.effect, { diff --git a/packages/runtime/src/looms.ts b/packages/runtime/src/looms.ts index a1fc06d..8d458f1 100644 --- a/packages/runtime/src/looms.ts +++ b/packages/runtime/src/looms.ts @@ -9,6 +9,7 @@ import { type DebuggerHost, type DefinitionInput, type DefinitionRef, + type DefinitionResolver, type EventEnvelope, type EventInput, type EventStore, @@ -42,6 +43,11 @@ import { export interface CreateLoomsOptions { /** Runtime modules to enable. Pass `agent()`, `workflow()`, `approval()`, and/or your own. */ readonly modules?: readonly AnyRuntimeModule[] + /** + * Loads a definition that was not registered on a module. Registered definitions are checked + * first. A run pins the resolved version on its start event. + */ + readonly definitionResolver?: DefinitionResolver readonly store?: EventStore | Promise | (() => Promise) readonly serve?: boolean | { port?: number; hostname?: string } readonly maxWakeIterations?: number @@ -203,6 +209,7 @@ export function createLooms(options: CreateLoomsOptions = {}): Looms { localLauncher: options.launcher === undefined, worker: options.worker, observer: options.observer, + definitionResolver: options.definitionResolver, }) if (options.rescanTimers !== false) { diff --git a/packages/runtime/src/repository.ts b/packages/runtime/src/repository.ts index d606502..84ac519 100644 --- a/packages/runtime/src/repository.ts +++ b/packages/runtime/src/repository.ts @@ -7,11 +7,14 @@ import { emptyCursor, EventStoreTrimmedError, hashRunState, + primePinnedDefinitions, summarizeRun, trimEventStoreSafely, type AppendableEvent, type AppendResult, type ComposedRegistry, + type DefinitionResolveError, + type DefinitionResolver, type EventEnvelope, type EventStore, type EventStoreAppendError, @@ -100,11 +103,19 @@ export interface RuntimeRepositoryOptions { readonly runCache: RunCursorCache readonly executionContexts: RunExecutionContexts readonly observer?: RuntimeObserver + readonly definitionResolver: DefinitionResolver } export function createRuntimeRepository(options: RuntimeRepositoryOptions) { - const { registry, snapshotStore, trimAfterSnapshot, keepSnapshots, runCache, executionContexts } = - options + const { + registry, + snapshotStore, + trimAfterSnapshot, + keepSnapshots, + runCache, + executionContexts, + definitionResolver, + } = options const headerFingerprints = new Map() @@ -196,7 +207,7 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { expectedTail?: number, ): Effect.Effect< RunCursor, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError > => Effect.gen(function* () { if (expectedTail !== undefined && cursor.seq === expectedTail) { @@ -226,6 +237,7 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { } delta = yield* readRemaining(store, runId, delta, expectedTail) + yield* primePinnedDefinitions(definitionResolver, delta) return advanceCursor(cursor, delta, registry) }) @@ -235,7 +247,7 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { runId: string, ): Effect.Effect< RunCursor, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError > => Effect.gen(function* () { let cursor = emptyCursor(runId) @@ -310,6 +322,8 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { delta = yield* readRemaining(store, runId, delta) } + yield* primePinnedDefinitions(definitionResolver, delta) + return advanceCursor(cursor, delta, registry) }) @@ -318,7 +332,7 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { runId: string, ): Effect.Effect< RunCursor, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError > => Effect.gen(function* () { const cached = runCache.get(runId) @@ -439,7 +453,7 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { commandRecords = 0, ): Effect.Effect< RunCursor, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError > => Effect.gen(function* () { const echo = assignSequences(batch, appendResult) @@ -457,6 +471,7 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { } } + yield* primePinnedDefinitions(definitionResolver, echo) next = advanceCursor(next, echo, registry) executionContexts.get(runId).liveCount = 0 @@ -484,7 +499,8 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { | EventStoreAppendError | EventStoreTruncationError | EventStoreTrimmedError - | EventStoreError, + | EventStoreError + | DefinitionResolveError, R > => Effect.gen(function* () { diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index ac0b33b..0564872 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -1,13 +1,15 @@ import { Clock, Effect, ManagedRuntime, Predicate } from 'effect' import { + cachedDefinitionResolver, + chainDefinitionResolvers, composeModules, createEvent, createThreadId, createRunId, DEFAULT_DEFINITION_VERSION, DEFAULT_SNAPSHOT_EVERY, - definitionKey, + staticDefinitionResolver, descendantThreadIds, EventStoreTag, foldRun, @@ -119,14 +121,26 @@ export function createRuntime [definitionKey(def.kind, def.name, def.version), def] as const, - ), + const staticResolver = staticDefinitionResolver( + registry.definitions.map((definition) => ({ + ...definition.value, + kind: definition.kind, + name: definition.name, + version: definition.version, + })), ) + const definitionResolver = cachedDefinitionResolver( + options.definitionResolver + ? chainDefinitionResolvers([staticResolver, options.definitionResolver]) + : staticResolver, + ) + + for (const module of options.modules) { + module.bindHost?.({ definitionResolver }) + } + // Module composition erases the heterogeneous service identifier union at this host boundary. // oxlint-disable-next-line effecttsgo/any-unknown-in-error-context const services = ManagedRuntime.make(moduleServices(options.modules)) @@ -155,6 +169,7 @@ export function createRuntime scheduler, launcher: () => launcher, }) diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 850756b..092a0ff 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -3,9 +3,11 @@ import type { Effect } from 'effect' import type { AnyRuntimeModule, ComposedRegistry, + DefinitionResolver, EventEnvelope, EventInput, WorkerCallbackInput, + DefinitionResolveError, EventStore, EventStoreAppendError, EventStoreError, @@ -42,6 +44,12 @@ export interface CreateRuntimeOptions< TModules extends readonly AnyRuntimeModule[] = readonly AnyRuntimeModule[], > { readonly modules: TModules + /** + * Loads a definition that was not registered on a module. Registered definitions are checked + * first. Lookups are cached, including misses. Start fails with `UnknownDefinitionError` + * and writes no events when the definition is missing. + */ + readonly definitionResolver?: DefinitionResolver readonly store?: EventStore /** * Durable events between mid-wake snapshots. Default 200; `0` disables. @@ -164,7 +172,10 @@ export interface LoomsRuntime< runId: string, ): Effect.Effect< RunState, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError, + | EventStoreTruncationError + | EventStoreTrimmedError + | EventStoreError + | DefinitionResolveError, EventStoreTag > getEvents( @@ -183,7 +194,10 @@ export interface LoomsRuntime< runId: string, ): Effect.Effect< RunOperationalStatus, - EventStoreError | EventStoreTrimmedError | EventStoreTruncationError, + | EventStoreError + | EventStoreTrimmedError + | EventStoreTruncationError + | DefinitionResolveError, EventStoreTag > retryEffect( @@ -212,18 +226,28 @@ export interface LoomsRuntime< /** Indexed run rows. Folds a run only when its header has not been written yet. */ readonly listRunSummaries: Effect.Effect< readonly RunSummary[], - EventStoreError | EventStoreTrimmedError | EventStoreTruncationError | SnapshotStoreError, + | EventStoreError + | EventStoreTrimmedError + | EventStoreTruncationError + | SnapshotStoreError + | DefinitionResolveError, EventStoreTag > readonly rescanTimers: Effect.Effect< number, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError, + | EventStoreTruncationError + | EventStoreTrimmedError + | EventStoreError + | DefinitionResolveError, EventStoreTag > /** Reschedule every durable timer and effect deadline after restart or repair. */ readonly recoverDeadlines: Effect.Effect< number, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError, + | EventStoreTruncationError + | EventStoreTrimmedError + | EventStoreError + | DefinitionResolveError, EventStoreTag > readonly dispose: Effect.Effect diff --git a/packages/runtime/src/wake-engine.ts b/packages/runtime/src/wake-engine.ts index 341c9d6..0150706 100644 --- a/packages/runtime/src/wake-engine.ts +++ b/packages/runtime/src/wake-engine.ts @@ -7,8 +7,8 @@ import { foldRun, isRunTerminal, type ComposedRegistry, + type DefinitionResolver, type EventEnvelope, - type RegisteredDefinition, type RunLauncher, type RunState, } from '@looms/core' @@ -44,7 +44,7 @@ function currentTimeMillis(): number { export interface WakeEngineOptions { readonly registry: ComposedRegistry readonly services: ManagedRuntime.ManagedRuntime - readonly definitions: ReadonlyMap + readonly definitionResolver: DefinitionResolver readonly executionContexts: RunExecutionContexts readonly repository: RuntimeRepository readonly snapshotEvery: number @@ -59,7 +59,7 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake const { registry, services, - definitions, + definitionResolver, executionContexts, repository, snapshotEvery, @@ -378,7 +378,7 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake { registry, services, - definitions, + definitionResolver, executionContext, liveAppends, runInContext, diff --git a/packages/schedule/src/definition-resolver.test.ts b/packages/schedule/src/definition-resolver.test.ts new file mode 100644 index 0000000..f21fb52 --- /dev/null +++ b/packages/schedule/src/definition-resolver.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { + makeMemoryEventStore, + type DefinitionResolver, + type RunSummary, + type SnapshotStore, +} from '@looms/core' +import { createLooms } from '@looms/runtime' + +import { defineSchedule } from './definitions' +import { schedule } from './module' +import { every } from './specs' + +function discardingSnapshots(): SnapshotStore { + const headers: RunSummary[] = [] + + return { + loadLatest: () => Effect.succeedNone, + save: () => Effect.void, + saveHeader: (header) => + Effect.sync(() => { + const index = headers.findIndex((item) => item.runId === header.runId) + + if (index >= 0) { + headers[index] = header + return + } + + headers.push(header) + }), + get listHeaders() { + return Effect.succeed(headers) + }, + } +} + +describe('schedule definition resolver', () => { + test('replay reads misfire from the resolved definition', async () => { + const nightly = defineSchedule({ + name: 'nightly', + version: 'v3', + target: { kind: 'workflow', name: 'job', version: 'v1' }, + spec: every(1), + misfire: 'skip', + }) + + const resolver: DefinitionResolver = { + peek: () => undefined, + resolve: (kind, name, version) => + Effect.succeed( + kind === 'schedule' && name === 'nightly' && version === 'v3' ? nightly : undefined, + ), + } + + const store = await Effect.runPromise(makeMemoryEventStore) + const snapshots = discardingSnapshots() + + const looms = createLooms({ + modules: [schedule({ minIntervalMs: 60_000 })], + store, + snapshotStore: snapshots, + definitionResolver: resolver, + rescanTimers: false, + }) + + try { + const started = await looms.start(nightly, null, { runId: 'nightly' }) + const thread = Object.values(started.state.threads).find((item) => item.kind === 'schedule') + expect(thread?.state).toMatchObject({ misfire: 'skip' }) + + const reloaded = createLooms({ + modules: [schedule({ minIntervalMs: 60_000 })], + store, + snapshotStore: snapshots, + definitionResolver: resolver, + rescanTimers: false, + }) + + try { + const state = await reloaded.getRun('nightly') + const replayed = Object.values(state.threads).find((item) => item.kind === 'schedule') + expect(replayed?.state).toMatchObject({ misfire: 'skip' }) + } finally { + await reloaded.stop() + } + } finally { + await looms.stop() + } + }) +}) diff --git a/packages/schedule/src/definitions-store.ts b/packages/schedule/src/definitions-store.ts index 0a53a1b..79fa719 100644 --- a/packages/schedule/src/definitions-store.ts +++ b/packages/schedule/src/definitions-store.ts @@ -17,5 +17,5 @@ export function makeScheduleDefinitionStore( return makeDefinitionStore('schedule', definitions) } -export const ScheduleDefinitionsLive = (definitions: ReadonlyArray) => - Layer.succeed(ScheduleDefinitionsTag, makeScheduleDefinitionStore(definitions)) +export const ScheduleDefinitionsLive = (definitions: ScheduleDefinitionStore) => + Layer.succeed(ScheduleDefinitionsTag, definitions) diff --git a/packages/schedule/src/effects.ts b/packages/schedule/src/effects.ts index 6667037..5ced07d 100644 --- a/packages/schedule/src/effects.ts +++ b/packages/schedule/src/effects.ts @@ -86,7 +86,7 @@ export const fireEffect = scheduleModule.effect({ const launcher = yield* RunLauncherTag const now = yield* Clock.currentTimeMillis - const definition = definitions.get(input.definitionName, input.definitionVersion) + const definition = yield* definitions.resolve(input.definitionName, input.definitionVersion) if (!definition) { return yield* new ScheduleEffectError({ diff --git a/packages/schedule/src/module.ts b/packages/schedule/src/module.ts index dcb2187..cdacd6e 100644 --- a/packages/schedule/src/module.ts +++ b/packages/schedule/src/module.ts @@ -1,4 +1,4 @@ -import { defineModule } from '@looms/core' +import { defineModule, makeDefinitionStore } from '@looms/core' import type { ScheduleDefinition } from './definitions' import { ScheduleDefinitionsLive } from './definitions-store' @@ -15,14 +15,18 @@ export interface ScheduleModuleOptions { } export function schedule(options: ScheduleModuleOptions = {}) { - const definitions = options.definitions ?? [] + const listed = options.definitions ?? [] + const definitions = makeDefinitionStore('schedule', listed) const minIntervalMs = options.minIntervalMs ?? DEFAULT_MIN_INTERVAL_MS return defineModule(scheduleModule, () => ({ - definitions, + definitions: listed, threads: { schedule: createScheduleThread(definitions, minIntervalMs) }, effects: { plan: planEffect, fire: fireEffect }, projections: { scheduleStatus }, services: () => ScheduleDefinitionsLive(definitions), + bindHost(host) { + definitions.useResolver(host.definitionResolver) + }, })) } diff --git a/packages/schedule/src/threads.ts b/packages/schedule/src/threads.ts index 4964dff..85a8080 100644 --- a/packages/schedule/src/threads.ts +++ b/packages/schedule/src/threads.ts @@ -2,11 +2,11 @@ import { Option, Schema } from 'effect' import { complete, - definitionKey, emit, fail, invoke, stringifyJson, + type DefinitionStore, type JsonValue, type RuntimeEffect, type ThreadDefinition, @@ -47,24 +47,15 @@ const decodeStart = Schema.decodeUnknownOption(ScheduleStartSchema) const decodeUpdate = Schema.decodeUnknownOption(ScheduleUpdatedPayloadSchema) export function createScheduleThread( - definitions: readonly ScheduleDefinition[], + definitions: DefinitionStore, minIntervalMs: number, ): ThreadDefinition { - const byKey = new Map( - definitions.map( - (definition) => - [definitionKey(definition.kind, definition.name, definition.version), definition] as const, - ), - ) - return scheduleModule.thread({ kind: 'schedule', input: ScheduleStartSchema, forgetsCompletedEffects: true, initialState: (ctx) => { - const definition = byKey.get( - definitionKey('schedule', ctx.definitionName, ctx.definitionVersion), - ) + const definition = definitions.get(ctx.definitionName, ctx.definitionVersion) const base = { input: null, @@ -178,7 +169,7 @@ export function createScheduleThread( } case 'schedule.updated': - return applyUpdate(state, event.payload, byKey, minIntervalMs) + return applyUpdate(state, event.payload, definitions, minIntervalMs) case 'schedule.paused': return { ...state, paused: true, nextAt: null } @@ -283,7 +274,7 @@ export function createScheduleThread( function applyUpdate( state: ScheduleState, payload: JsonValue, - byKey: ReadonlyMap, + definitions: DefinitionStore, minIntervalMs: number, ): ScheduleState { const decoded = decodeUpdate(payload) @@ -295,7 +286,7 @@ function applyUpdate( const patch = decoded.value const definitionName = patch.definitionName ?? state.definitionName const definitionVersion = patch.definitionVersion ?? state.definitionVersion - const definition = byKey.get(definitionKey('schedule', definitionName, definitionVersion)) + const definition = definitions.get(definitionName, definitionVersion) if ( (patch.definitionName !== undefined || patch.definitionVersion !== undefined) && diff --git a/packages/workflow/src/definitions-store.ts b/packages/workflow/src/definitions-store.ts index f66d318..fbc17bb 100644 --- a/packages/workflow/src/definitions-store.ts +++ b/packages/workflow/src/definitions-store.ts @@ -17,5 +17,5 @@ export function makeWorkflowDefinitionStore( return makeDefinitionStore('workflow', definitions) } -export const WorkflowDefinitionsLive = (definitions: ReadonlyArray) => - Layer.succeed(WorkflowDefinitionsTag, makeWorkflowDefinitionStore(definitions)) +export const WorkflowDefinitionsLive = (definitions: WorkflowDefinitionStore) => + Layer.succeed(WorkflowDefinitionsTag, definitions) diff --git a/packages/workflow/src/effects.ts b/packages/workflow/src/effects.ts index 3b5074f..8fe52d9 100644 --- a/packages/workflow/src/effects.ts +++ b/packages/workflow/src/effects.ts @@ -97,11 +97,16 @@ export const scheduleEffect = workflowModule.effect({ const workflows = yield* WorkflowDefinitionsTag const definition = input.definitionName - ? workflows.get(input.definitionName, input.definitionVersion) + ? yield* workflows.resolve(input.definitionName, input.definitionVersion) : undefined if (!definition) { - return [] + return input.definitionName + ? threadFailed( + ctx.threadId, + `Unknown workflow definition ${input.definitionName}@${input.definitionVersion}`, + ) + : [] } const nodes = { ...input.nodes } satisfies Record @@ -236,7 +241,7 @@ export const runNodeEffect = workflowModule.effect({ const workflows = yield* WorkflowDefinitionsTag const definition = input.definitionName - ? workflows.get(input.definitionName, input.definitionVersion) + ? yield* workflows.resolve(input.definitionName, input.definitionVersion) : undefined const run: NodeRun = { diff --git a/packages/workflow/src/module.ts b/packages/workflow/src/module.ts index 13ed970..6cd5cbf 100644 --- a/packages/workflow/src/module.ts +++ b/packages/workflow/src/module.ts @@ -1,4 +1,4 @@ -import { defineModule } from '@looms/core' +import { defineModule, makeDefinitionStore } from '@looms/core' import type { WorkflowDefinition } from './definitions' import { WorkflowDefinitionsLive } from './definitions-store' @@ -12,9 +12,11 @@ export interface WorkflowModuleOptions { } export function workflow(options: WorkflowModuleOptions = {}) { - const definitions = options.definitions ?? [] + const listed = options.definitions ?? [] + const definitions = makeDefinitionStore('workflow', listed) + return defineModule(workflowModule, () => ({ - definitions, + definitions: listed, threads: { workflow: workflowThread }, effects: { schedule: scheduleEffect, @@ -22,5 +24,8 @@ export function workflow(options: WorkflowModuleOptions = {}) { }, projections: { nodes }, services: () => WorkflowDefinitionsLive(definitions), + bindHost(host) { + definitions.useResolver(host.definitionResolver) + }, })) } From e98675ded941ee71a23b560c76492da6a5b55ce5 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 11:20:27 +0200 Subject: [PATCH 05/23] feat(runtime): hibernate Durable Object event sockets WebSocket subscribers replay the retained log and then receive appends without a subscription held in memory, so an idle socket can sleep. SSE uses the same hello and resync frames. Co-authored-by: Cursor --- .changeset/hibernating-websockets.md | 5 + docs/production.md | 2 + docs/protocol.md | 4 +- packages/client/src/client.test.ts | 142 ++++++++ packages/client/src/client.ts | 232 ++++++++++++- packages/client/src/index.ts | 3 + packages/client/src/sse.ts | 6 +- packages/cloudflare/src/cloudflare.test.ts | 105 +++++- packages/cloudflare/src/durable-object.ts | 43 ++- packages/cloudflare/src/websocket.test.ts | 252 ++++++++++++++ packages/cloudflare/src/websocket.ts | 325 +++++++++++++++++ packages/core/src/index.ts | 1 + packages/core/src/subscription.test.ts | 200 +++++++++++ packages/core/src/subscription.ts | 386 +++++++++++++++++++++ packages/react/src/react.ts | 88 +++-- packages/react/src/store.test.ts | 59 ++++ packages/react/src/store.ts | 155 ++++++++- packages/runtime/src/events-api.test.ts | 17 +- packages/runtime/src/index.ts | 2 + packages/runtime/src/server.ts | 1 + packages/runtime/src/sse.test.ts | 78 ++++- packages/runtime/src/sse.ts | 64 +++- packages/runtime/src/subscription.ts | 45 +++ 23 files changed, 2138 insertions(+), 77 deletions(-) create mode 100644 .changeset/hibernating-websockets.md create mode 100644 packages/cloudflare/src/websocket.test.ts create mode 100644 packages/cloudflare/src/websocket.ts create mode 100644 packages/core/src/subscription.test.ts create mode 100644 packages/core/src/subscription.ts create mode 100644 packages/runtime/src/subscription.ts diff --git a/.changeset/hibernating-websockets.md b/.changeset/hibernating-websockets.md new file mode 100644 index 0000000..152ff72 --- /dev/null +++ b/.changeset/hibernating-websockets.md @@ -0,0 +1,5 @@ +--- +'@swirls/looms': minor +--- + +Durable Objects accept a hibernating WebSocket on `GET /runs/:id/events`. The socket replays the retained log, then receives later events without a subscription held in memory. SSE sends the same `hello` and `resync` frames. A cursor below the retained head is told to resume from `head` instead of silently skipping trimmed events. `subscribeEvents` and the React store take `transport: "websocket" | "sse" | "auto"`; the default remains SSE, and a numeric reconnect delay still works. diff --git a/docs/production.md b/docs/production.md index 866dee0..834ea7b 100644 --- a/docs/production.md +++ b/docs/production.md @@ -13,6 +13,8 @@ coalesced. A deployment, eviction, or alarm retry may repeat a wake; event preconditions make that safe. Cross-object queries and transactions are not provided. Build global run lists with a projector. +Live subscribers on a Durable Object should open a WebSocket (`Upgrade: websocket` on `GET /runs/:id/events`). The object hibernates the socket, answers `ping` with `pong` without waking, and fans out events when the log is appended. SSE stays available, including on `looms.serve`, but its keepalive keeps a Durable Object awake. Writes stay on HTTP. A browser still cannot send `Authorization` on the upgrade; grant tokens are a separate change. An idle-socket billed-duration check on a real Cloudflare account has not been run. + All built-in stores reject an event over 1 MiB and a snapshot over 8 MiB, measured as serialized UTF-8, before storage is mutated. The stable error names are `EventPayloadTooLargeError` and `SnapshotPayloadTooLargeError`. diff --git a/docs/protocol.md b/docs/protocol.md index d33fd6e..aef6a81 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -32,7 +32,7 @@ Matching is by event `type` plus an optional payload subset (`match: { approvalI | `POST` | `/runs` | Start `{ kind, definitionName, input }` (`Accept: text/event-stream` or `?stream=true` streams the log live) | | `GET` | `/runs` | List run ids (`createLooms` / shared store). Actor hosts (`createLocalActorHost`, Durable Objects, celld) return `501` — use a projector lake or client-side history instead | | `GET` | `/runs/:id` | Current run state | -| `GET` | `/runs/:id/events` | Event log (`?fromSeq=&limit=`), or SSE with `live=true` / `Accept: text/event-stream`; resumes after `Last-Event-ID` | +| `GET` | `/runs/:id/events` | Event log (`?fromSeq=&limit=`). SSE with `live=true` / `Accept: text/event-stream`, or a WebSocket with `Upgrade: websocket` on a Durable Object. Resumes after `Last-Event-ID` or `fromSeq`. `threadId` and `includeEphemeral=false` filter the stream. | | `POST` | `/runs/:id/events` | Signal the run | | `GET` | `/runs/:id/threads` | Child threads | | `POST` | `/runs/:id/wake` | Resume processing | @@ -41,4 +41,6 @@ Matching is by event `type` plus an optional payload subset (`match: { approvalI `looms.fetch(request)` handles these paths. It returns `null` for everything else so you can mount Looms next to your own UI. +A live subscription speaks three frames. `{"type":"hello","head":1,"tail":4}` is sent first and reports the retained bounds. Event frames stay `{"batch":[...encoded events]}` and, on SSE, still use `id: `. `{"type":"resync","head":5,"reason":"trimmed"|"backpressure"}` means the cursor is below the retained log, or the socket fell behind; the server then closes the WebSocket with code `4000`. Resume from `head`. Clients that only read `batch` ignore the other two frames. + Use `@swirls/looms/client` (`createLoomsClient`) instead of hand-rolling fetch when you can. diff --git a/packages/client/src/client.test.ts b/packages/client/src/client.test.ts index f454244..c407f7b 100644 --- a/packages/client/src/client.test.ts +++ b/packages/client/src/client.test.ts @@ -166,6 +166,148 @@ describe('createLoomsClient', () => { stop() }) + test('subscribeEvents resumes from a resync head', async () => { + const calls: string[] = [] + let attempt = 0 + + const started = { + id: 'evt_4', + runId: 'run_1', + seq: 4, + ts: 1, + type: 'runtime.run.started', + payload: {}, + threadId: 'thr_1', + parentThreadId: null, + causationId: null, + correlationId: null, + effectId: null, + ephemeral: false, + origin: { type: 'system' }, + } + + const client = createLoomsClient({ + fetch: async (input) => { + calls.push(hrefOf(input)) + attempt += 1 + const encoder = new TextEncoder() + + const frame = + attempt === 1 + ? `data: ${JSON.stringify({ type: 'resync', head: 4, reason: 'trimmed' })}\n\n` + : `id: 4\ndata: ${JSON.stringify({ batch: [started] })}\n\n` + + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(frame)) + controller.close() + }, + }), + { headers: { 'content-type': 'text/event-stream' } }, + ) + }, + }) + + const received: number[] = [] + + const stop = client.subscribeEvents( + 'run_1', + (event) => { + received.push(event.seq) + }, + { reconnectDelayMs: 20 }, + ) + + const start = Date.now() + + while (received.length === 0 && Date.now() - start < 2_000) { + await Bun.sleep(10) + } + + expect(received).toEqual([4]) + expect(calls[0]).toContain('fromSeq=1') + expect(calls.some((href) => href.includes('fromSeq=4'))).toBe(true) + stop() + }) + + test('subscribeEvents can follow a websocket', async () => { + const received: number[] = [] + + class FakeSocket { + static latest: FakeSocket | undefined + readonly url: string + private readonly listeners = new Map void>>() + + constructor(url: string) { + this.url = url + FakeSocket.latest = this + } + + addEventListener(type: string, listener: (event: { readonly data?: unknown }) => void): void { + const current = this.listeners.get(type) ?? [] + current.push(listener) + this.listeners.set(type, current) + } + + emit(type: string, data?: string): void { + for (const listener of this.listeners.get(type) ?? []) { + listener({ data }) + } + } + + close(): void { + this.emit('close') + } + } + + const client = createLoomsClient({ baseUrl: 'http://looms.test' }) + + const stop = client.subscribeEvents( + 'run_1', + (event) => { + received.push(event.seq) + }, + { transport: 'websocket', webSocket: FakeSocket, reconnectDelayMs: 60_000 }, + ) + + const socket = FakeSocket.latest + expect(socket?.url.startsWith('ws://looms.test/runs/run_1/events?fromSeq=1')).toBe(true) + socket?.emit('message', JSON.stringify({ type: 'hello', head: 1, tail: 1 })) + + socket?.emit( + 'message', + JSON.stringify({ + batch: [ + { + id: 'evt_1', + runId: 'run_1', + seq: 1, + ts: 1, + type: 'runtime.run.started', + payload: {}, + threadId: 'thr_1', + parentThreadId: null, + causationId: null, + correlationId: null, + effectId: null, + ephemeral: false, + origin: { type: 'system' }, + }, + ], + }), + ) + + const start = Date.now() + + while (received.length === 0 && Date.now() - start < 2_000) { + await Bun.sleep(10) + } + + expect(received).toEqual([1]) + stop() + }) + test('stream yields live deltas then the completed run events', async () => { const client = createLoomsClient({ fetch: async (input, init) => { diff --git a/packages/client/src/client.ts b/packages/client/src/client.ts index 413b98e..211a235 100644 --- a/packages/client/src/client.ts +++ b/packages/client/src/client.ts @@ -1,9 +1,10 @@ -import { Data, Effect, Option, Schema, Stream } from 'effect' +import { Data, Effect, Option, Predicate, Schema, Stream } from 'effect' import { createRunId, DefinitionCatalogSchema, EventEnvelopeSchema, + readSubscriptionData, ReplayStepSchema, RunStateSchema, RunSummariesSchema, @@ -14,6 +15,7 @@ import { type EventInput, type JsonValue, type RunState, + type SubscriptionTransport, type WorkerCallbackInput, } from '@looms/core' @@ -118,14 +120,29 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { const subscribeEvents = ( runId: string, onEvent: (event: EventEnvelope) => void, - reconnectDelayMs = 1_000, + third: number | SubscribeEventsOptions = 1_000, ) => { + const settings = subscriptionSettings(third) let fromSeq = 1 const controller = new AbortController() const run = Effect.gen(function* () { while (!controller.signal.aborted) { yield* Effect.gen(function* () { + if (useWebSocket(settings)) { + return yield* consumeWebSocket({ + baseUrl, + runId, + settings, + signal: controller.signal, + currentSeq: () => fromSeq, + setSeq: (next) => { + fromSeq = next + }, + onEvent, + }) + } + const cursor = Math.max(0, fromSeq - 1) const headers = new Headers({ accept: 'text/event-stream' }) @@ -135,7 +152,7 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { const res = yield* Effect.tryPromise({ try: (signal) => - fetchImpl(`${baseUrl}/runs/${encodeURIComponent(runId)}/events?fromSeq=${fromSeq}`, { + fetchImpl(`${baseUrl}${eventsQuery(runId, fromSeq, settings)}`, { headers, signal, }), @@ -146,10 +163,22 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { if (res.ok && body) { return yield* consumeSseStreamEffect(body, (frame) => { - for (const event of eventsFromSseData(frame.data, runId)) { - onEvent(event) - fromSeq = Math.max(fromSeq, event.seq + 1) - } + let resync = false + + deliverSubscriptionData({ + data: frame.data, + runId, + currentSeq: () => fromSeq, + setSeq: (next) => { + fromSeq = next + }, + onEvent, + onResync: () => { + resync = true + }, + }) + + return resync ? false : undefined }) } @@ -162,7 +191,7 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { break } - yield* Effect.sleep(reconnectDelayMs) + yield* Effect.sleep(settings.reconnectDelayMs) } }) @@ -329,3 +358,190 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { } export type LoomsClient = ReturnType + +export interface SubscribeEventsOptions { + readonly transport?: SubscriptionTransport + readonly reconnectDelayMs?: number + readonly threadId?: string + readonly includeEphemeral?: boolean + readonly webSocket?: SubscriptionSocketConstructor +} + +export interface SubscriptionSocket { + addEventListener(type: string, listener: (event: { readonly data?: unknown }) => void): void + close(code?: number, reason?: string): void +} + +export interface SubscriptionSocketConstructor { + new (url: string): SubscriptionSocket +} + +interface SubscriptionDelivery { + readonly data: string + readonly runId: string + readonly currentSeq: () => number + readonly setSeq: (seq: number) => void + readonly onEvent: (event: EventEnvelope) => void + readonly onResync: () => void +} + +interface WebSocketSubscription { + readonly baseUrl: string + readonly runId: string + readonly settings: ResolvedSubscription + readonly signal: AbortSignal + readonly currentSeq: () => number + readonly setSeq: (seq: number) => void + readonly onEvent: (event: EventEnvelope) => void +} + +interface ResolvedSubscription extends SubscribeEventsOptions { + readonly transport: SubscriptionTransport + readonly reconnectDelayMs: number +} + +function subscriptionSettings(third: number | SubscribeEventsOptions): ResolvedSubscription { + if (Predicate.isNumber(third)) { + return { transport: 'sse', reconnectDelayMs: third } + } + + return { + transport: third.transport ?? 'sse', + reconnectDelayMs: third.reconnectDelayMs ?? 1_000, + threadId: third.threadId, + includeEphemeral: third.includeEphemeral, + webSocket: third.webSocket, + } +} + +function useWebSocket(settings: ResolvedSubscription): boolean { + if (settings.transport !== 'websocket' && settings.transport !== 'auto') { + return false + } + + return settings.webSocket !== undefined || globalWebSocket() !== undefined +} + +function eventsQuery(runId: string, fromSeq: number, settings: ResolvedSubscription): string { + const params = new URLSearchParams({ fromSeq: String(fromSeq) }) + + if (settings.threadId) { + params.set('threadId', settings.threadId) + } + + if (settings.includeEphemeral === false) { + params.set('includeEphemeral', 'false') + } + + return `/runs/${encodeURIComponent(runId)}/events?${params.toString()}` +} + +function webSocketUrl(baseUrl: string, path: string): string { + const url = new URL(path, baseUrl) + url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:' + return url.toString() +} + +function globalWebSocket(): SubscriptionSocketConstructor | undefined { + if (!('WebSocket' in globalThis) || !Predicate.isFunction(globalThis.WebSocket)) { + return undefined + } + + return globalThis.WebSocket +} + +function consumeWebSocket(subscription: WebSocketSubscription): Effect.Effect { + const Socket = subscription.settings.webSocket ?? globalWebSocket() + + if (!Socket) { + return Effect.void + } + + const url = webSocketUrl( + subscription.baseUrl, + eventsQuery(subscription.runId, subscription.currentSeq(), subscription.settings), + ) + + return Effect.callback((resume) => { + const socket = new Socket(url) + + const finish = () => { + resume(Effect.void) + } + + if (subscription.signal.aborted) { + socket.close() + finish() + return Effect.void + } + + const onAbort = () => { + socket.close() + finish() + } + + subscription.signal.addEventListener('abort', onAbort) + + socket.addEventListener('message', (event) => { + if (!Predicate.isString(event.data)) { + return + } + + deliverSubscriptionData({ + data: event.data, + runId: subscription.runId, + currentSeq: subscription.currentSeq, + setSeq: subscription.setSeq, + onEvent: subscription.onEvent, + onResync: () => { + socket.close() + }, + }) + }) + + socket.addEventListener('close', finish) + socket.addEventListener('error', finish) + + return Effect.sync(() => { + subscription.signal.removeEventListener('abort', onAbort) + socket.close() + }) + }) +} + +function deliverSubscriptionData(delivery: SubscriptionDelivery): void { + const read = readSubscriptionData(delivery.data, delivery.runId) + + switch (read.kind) { + case 'events': { + const fromSeq = delivery.currentSeq() + let next = fromSeq + + for (const event of read.events) { + if (event.seq < fromSeq) { + continue + } + + delivery.onEvent(event) + next = Math.max(next, event.seq + 1) + } + + delivery.setSeq(next) + return + } + + case 'resync': + delivery.setSeq(read.head) + delivery.onResync() + return + + case 'hello': + case 'ignore': + return + + default: { + const unexpected: never = read + return unexpected + } + } +} diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index fe3be90..a1ef0b2 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -4,6 +4,9 @@ export { type LoomsClientOptions, type StartResult, type StreamHandle, + type SubscribeEventsOptions, + type SubscriptionSocket, + type SubscriptionSocketConstructor, } from './client' export { consumeSseStream, diff --git a/packages/client/src/sse.ts b/packages/client/src/sse.ts index 6b0e656..99f3758 100644 --- a/packages/client/src/sse.ts +++ b/packages/client/src/sse.ts @@ -58,7 +58,7 @@ export class SseStreamError extends Data.TaggedError('SseStreamError')<{ export function consumeSseStreamEffect( body: ReadableStream, - onFrame: (frame: SseFrame) => void, + onFrame: (frame: SseFrame) => void | boolean, ): Effect.Effect { const decoder = new TextDecoder() @@ -85,8 +85,8 @@ export function consumeSseStreamEffect( for (const chunk of chunks) { const frame = parseSseFrame(chunk) - if (frame) { - onFrame(frame) + if (frame && onFrame(frame) === false) { + return } } } diff --git a/packages/cloudflare/src/cloudflare.test.ts b/packages/cloudflare/src/cloudflare.test.ts index d9f5cf8..29e0418 100644 --- a/packages/cloudflare/src/cloudflare.test.ts +++ b/packages/cloudflare/src/cloudflare.test.ts @@ -4,7 +4,12 @@ import { describe, expect, mock, setSystemTime, test } from 'bun:test' import { Effect, Schema } from 'effect' import { createLocalActorHost } from '@looms/actor' -import { DefinitionCatalogSchema, emptyRunState, makeMemoryEventStore } from '@looms/core' +import { + DefinitionCatalogSchema, + emptyRunState, + makeMemoryEventStore, + type SubscriptionAttachment, +} from '@looms/core' import { defineSchedule, ensureSchedule, every, once, schedule } from '@looms/schedule' import { defineWorkflow, workflow } from '@looms/workflow' @@ -319,6 +324,104 @@ describe('@looms/cloudflare', () => { const fetchRes = await doInstance.fetch(new Request('http://localhost/health')) expect(fetchRes.status).toBe(200) }) + + test('LoomsDurableObject replays a run over a hibernating socket', async () => { + class Socket { + readonly sent: string[] = [] + bufferedAmount = 0 + private attachment: SubscriptionAttachment | undefined + + send(data: string): void { + this.sent.push(data) + } + + close(): void { + // The platform drops the socket. + } + + serializeAttachment(value: SubscriptionAttachment): void { + this.attachment = value + } + + deserializeAttachment(): SubscriptionAttachment | undefined { + return this.attachment + } + } + + let server: Socket | undefined + const hadPair = 'WebSocketPair' in globalThis + const previous = globalThis.WebSocketPair + + Object.defineProperty(globalThis, 'WebSocketPair', { + configurable: true, + writable: true, + value: class { + 0: Socket + 1: Socket + + constructor() { + this[0] = new Socket() + this[1] = new Socket() + server = this[1] + } + }, + }) + + try { + const accepted: Socket[] = [] + + const ctx = { + ...objectContext('run_ws', sqliteStorage()), + acceptWebSocket(ws: Socket) { + accepted.push(ws) + }, + getWebSockets() { + return accepted + }, + } + + const flow = defineWorkflow({ + name: 'ws-flow', + nodes: [{ id: 'work', run: () => ({ ok: true }) }], + }) + + class SocketObject extends LoomsDurableObject { + override configure(): LoomsDurableObjectConfig { + return { modules: [workflow({ definitions: [flow] })] } + } + } + + // SAFETY: the test stands in for a Durable Object state. + // @ts-expect-error mock socket context in unit test + const object = new SocketObject(ctx, {}) + + await object.startRun({ + runId: 'run_ws', + kind: 'workflow', + definitionName: 'ws-flow', + input: null, + }) + + const response = await object.fetch( + new Request('http://localhost/runs/run_ws/events', { headers: { upgrade: 'websocket' } }), + ) + + expect(response.status).toBe(101) + expect(server?.sent[0]).toContain('"type":"hello"') + expect(server?.sent.some((frame) => frame.includes('"batch"'))).toBe(true) + expect(accepted).toHaveLength(1) + } finally { + if (hadPair) { + Object.defineProperty(globalThis, 'WebSocketPair', { + configurable: true, + writable: true, + value: previous, + }) + } else { + Reflect.deleteProperty(globalThis, 'WebSocketPair') + } + } + }) }) const decodeCatalog = Schema.decodeUnknownSync(DefinitionCatalogSchema) diff --git a/packages/cloudflare/src/durable-object.ts b/packages/cloudflare/src/durable-object.ts index d5746df..7f64add 100644 --- a/packages/cloudflare/src/durable-object.ts +++ b/packages/cloudflare/src/durable-object.ts @@ -13,11 +13,12 @@ import { type RunStatusSnapshot, } from '@looms/core' import type { Projector } from '@looms/projectors' -import type { Authorize, StartResult, StartRunArgs } from '@looms/runtime' +import { defaultAuthorize, type Authorize, type StartResult, type StartRunArgs } from '@looms/runtime' import { durableObjectAlarms } from './alarm-scheduler' import { durableObjectEventStore, type DurableObjectEventStore } from './event-store' import type { DurableObjectNamespaceLike } from './router' +import { attachSocketFanout, isRunSocketUpgrade, openRunSocket } from './websocket' export interface LoomsDurableObjectConfig { readonly modules: readonly AnyRuntimeModule[] @@ -46,6 +47,7 @@ export interface LoomsDurableObjectConfig { export abstract class LoomsDurableObject extends DurableObject { private cellPromise?: Promise private store?: DurableObjectEventStore + private authorize: Authorize = defaultAuthorize abstract configure(env: Env): LoomsDurableObjectConfig | Promise @@ -90,10 +92,11 @@ export abstract class LoomsDurableObject extends DurableObject extends DurableObject { + if (isRunSocketUpgrade(req)) { + return this.getCell().then(() => this.acceptRunSocket(req)) + } + return this.getCell().then((cell) => cell.fetch(req)) } + /** Client messages are ignored. Signals and other writes stay on HTTP. */ + override webSocketMessage(): void { + // A hibernated socket stays quiet. The platform delivers this only if a client writes. + } + + override webSocketClose(): void { + // The platform drops the socket. Nothing is retained for it. + } + + override webSocketError(): void { + // The platform drops the socket. Nothing is retained for it. + } + + private acceptRunSocket(req: Request): Promise { + const store = this.store + const runId = this.ctx.id.name + + if (!store || !runId) { + return Promise.resolve(Response.json({ error: 'Run is not ready' }, { status: 500 })) + } + + return Effect.runPromise( + openRunSocket({ + ctx: this.ctx, + store, + request: req, + runId, + authorize: this.authorize, + }), + ) + } + /** Commit this object's run and set its alarm. Returns before the run wakes. */ launch(args: LaunchRunArgs): Promise { return this.getCell().then((cell) => cell.launch(args)) diff --git a/packages/cloudflare/src/websocket.test.ts b/packages/cloudflare/src/websocket.test.ts new file mode 100644 index 0000000..167df24 --- /dev/null +++ b/packages/cloudflare/src/websocket.test.ts @@ -0,0 +1,252 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { createEvent, makeMemoryEventStore, type EventStore, type SubscriptionAttachment } from '@looms/core' + +import { attachSocketFanout, openRunSocket, type HibernatingSocketState } from './websocket' + +class FakeSocket { + readonly sent: string[] = [] + closed: { code: number; reason: string } | undefined + bufferedAmount = 0 + private attachment: SubscriptionAttachment | undefined + + send(data: string): void { + this.sent.push(data) + } + + close(code?: number, reason?: string): void { + this.closed = { code: code ?? 0, reason: reason ?? '' } + } + + serializeAttachment(value: SubscriptionAttachment): void { + this.attachment = value + } + + deserializeAttachment(): SubscriptionAttachment | undefined { + return this.attachment + } +} + +class FakeState implements HibernatingSocketState { + readonly accepted: FakeSocket[] = [] + autoResponse: { request: string; response: string } | undefined + + acceptWebSocket(ws: WebSocket): void { + if (ws instanceof FakeSocket) { + this.accepted.push(ws) + } + } + + getWebSockets(): readonly WebSocket[] { + return this.accepted.map(asSocket) + } + + setWebSocketAutoResponse(pair: WebSocketRequestResponsePair): void { + this.autoResponse = { request: pair.request, response: pair.response } + } +} + +function pairFrom(server: FakeSocket, client: FakeSocket) { + return () => ({ client: asSocket(client), server: asSocket(server) }) +} + +function asSocket(socket: FakeSocket): WebSocket { + // SAFETY: the test socket implements send, close, bufferedAmount, and attachment. + // oxlint-disable-next-line typescript/no-unsafe-type-assertion + return socket as WebSocket +} + +async function seededStore(): Promise { + const store = await Effect.runPromise(makeMemoryEventStore) + + await Effect.runPromise( + store.append( + 'run_1', + [ + threadEvent('thr_a'), + threadEvent('thr_b'), + ], + { expectedTail: 0 }, + ), + ) + + return store +} + +function threadEvent(threadId: string) { + return createEvent('run_1', { + type: 'runtime.thread.started', + payload: { + threadId, + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v1', + input: null, + parentThreadId: null, + }, + threadId, + origin: { type: 'system' }, + }) +} + +function upgrade(url: string): Request { + return new Request(url, { headers: { upgrade: 'websocket' } }) +} + +describe('hibernating run sockets', () => { + test('replays the log, then fans out a later append', async () => { + const store = await seededStore() + const ctx = new FakeState() + const server = new FakeSocket() + const broadcasting = attachSocketFanout(ctx, store, 'run_1') + + const response = await Effect.runPromise( + openRunSocket({ + ctx, + store, + request: upgrade('http://looms.test/runs/run_1/events'), + runId: 'run_1', + createPair: pairFrom(server, new FakeSocket()), + }), + ) + + expect(response.status).toBe(101) + const frames = server.sent.map((line) => JSON.parse(line)) + expect(frames[0]).toEqual({ type: 'hello', head: 1, tail: 2 }) + expect(frames[1]?.batch).toHaveLength(2) + + expect(frames[1]?.batch.map((item: { threadId: string }) => item.threadId)).toEqual([ + 'thr_a', + 'thr_b', + ]) + + await Effect.runPromise( + broadcasting.append('run_1', [threadEvent('thr_a')], { expectedTail: 2 }), + ) + + const live = server.sent.map((line) => JSON.parse(line)) + expect(live.at(-1)).toMatchObject({ batch: [expect.objectContaining({ seq: 3 })] }) + }) + + test('resumes after fromSeq and drops other threads', async () => { + const store = await seededStore() + const server = new FakeSocket() + + await Effect.runPromise( + openRunSocket({ + ctx: new FakeState(), + store, + request: upgrade('http://looms.test/runs/run_1/events?fromSeq=2&threadId=thr_b'), + runId: 'run_1', + createPair: pairFrom(server, new FakeSocket()), + }), + ) + + const frames = server.sent.map((line) => JSON.parse(line)) + const batches = frames.filter((frame) => 'batch' in frame) + expect(batches).toHaveLength(1) + expect(batches[0].batch[0].threadId).toBe('thr_b') + expect(batches[0].batch[0].seq).toBe(2) + }) + + test('a trimmed cursor closes with resync', async () => { + const store = await seededStore() + + if (!store.trim) { + throw new Error('memory store does not trim') + } + + await Effect.runPromise(store.trim('run_1', 2)) + const server = new FakeSocket() + + await Effect.runPromise( + openRunSocket({ + ctx: new FakeState(), + store, + request: upgrade('http://looms.test/runs/run_1/events?fromSeq=1'), + runId: 'run_1', + createPair: pairFrom(server, new FakeSocket()), + }), + ) + + expect(JSON.parse(server.sent[0] ?? '{}')).toEqual({ + type: 'resync', + head: 2, + reason: 'trimmed', + }) + + expect(server.closed).toEqual({ code: 4000, reason: 'trimmed' }) + }) + + test('backpressure closes the socket', async () => { + const store = await seededStore() + const server = new FakeSocket() + server.bufferedAmount = 2 * 1024 * 1024 + + await Effect.runPromise( + openRunSocket({ + ctx: new FakeState(), + store, + request: upgrade('http://looms.test/runs/run_1/events'), + runId: 'run_1', + createPair: pairFrom(server, new FakeSocket()), + }), + ) + + expect(server.closed?.reason).toBe('backpressure') + + expect(server.sent.map((line) => JSON.parse(line))).toContainEqual({ + type: 'resync', + head: 1, + reason: 'backpressure', + }) + }) + + test('refuses a socket the authorizer rejected', async () => { + const store = await seededStore() + const ctx = new FakeState() + + const response = await Effect.runPromise( + openRunSocket({ + ctx, + store, + request: upgrade('http://looms.test/runs/run_1/events'), + runId: 'run_1', + authorize: () => Effect.succeed({ allowed: false, status: 401, error: 'no' }), + }), + ) + + expect(response.status).toBe(401) + expect(ctx.accepted).toHaveLength(0) + }) + + test('rejects an invalid Last-Event-ID and a mismatched run', async () => { + const store = await seededStore() + + const invalid = await Effect.runPromise( + openRunSocket({ + ctx: new FakeState(), + store, + request: new Request('http://looms.test/runs/run_1/events', { + headers: { upgrade: 'websocket', 'last-event-id': 'nope' }, + }), + runId: 'run_1', + }), + ) + + expect(invalid.status).toBe(400) + + const missing = await Effect.runPromise( + openRunSocket({ + ctx: new FakeState(), + store, + request: upgrade('http://looms.test/runs/other/events'), + runId: 'run_1', + }), + ) + + expect(missing.status).toBe(404) + }) +}) diff --git a/packages/cloudflare/src/websocket.ts b/packages/cloudflare/src/websocket.ts new file mode 100644 index 0000000..2736c3e --- /dev/null +++ b/packages/cloudflare/src/websocket.ts @@ -0,0 +1,325 @@ +import { Effect, Option, Predicate } from 'effect' + +import { + attachmentFromFilter, + fanoutSubscription, + openSubscription, + subscriptionFilter, + subscriptionFromSeq, + type AttachedSubscriptionSocket, + type EventEnvelope, + type EventStore, + type SubscriptionAttachment, +} from '@looms/core' +import { + authResponse, + defaultAuthorize, + loadSubscriptionOpening, + readSubscriptionBounds, + type Authorize, +} from '@looms/runtime' + +const EVENTS_PATH = /^\/runs\/([^/]+)\/events\/?$/ + +export interface HibernatingSocketState { + acceptWebSocket(ws: WebSocket, tags?: string[]): void + getWebSockets?(tag?: string): readonly WebSocket[] + setWebSocketAutoResponse?(pair: WebSocketRequestResponsePair): void +} + +export interface OpenRunSocketOptions { + readonly ctx: HibernatingSocketState + readonly store: EventStore + readonly request: Request + readonly runId: string + readonly authorize?: Authorize + readonly createPair?: () => { readonly client: WebSocket; readonly server: WebSocket } | undefined +} + +const autoResponseInstalled = new WeakSet() + +export function isRunSocketUpgrade(request: Request): boolean { + if (request.method !== 'GET') { + return false + } + + const upgrade = request.headers.get('upgrade') + + if (upgrade === null || upgrade.toLowerCase() !== 'websocket') { + return false + } + + return runEventsPath(new URL(request.url)) !== undefined +} + +/** + * Wakes on append, not on a socket subscription, so a hibernated object can sleep + * while its WebSockets stay accepted. + */ +export function attachSocketFanout( + ctx: HibernatingSocketState, + store: EventStore, + runId: string, +): EventStore { + return tapAppend(store, (events) => + readSubscriptionBounds(store, runId).pipe( + Effect.map((bounds) => { + fanoutSubscription(acceptedSockets(ctx, runId), events, { head: bounds.head }) + }), + Effect.orElseSucceed(() => { + fanoutSubscription(acceptedSockets(ctx, runId), events, { head: 1 }) + }), + ), + ) +} + +export function openRunSocket(options: OpenRunSocketOptions): Effect.Effect { + const url = new URL(options.request.url) + const requested = runEventsPath(url) + + if (requested !== options.runId) { + return Effect.succeed(Response.json({ error: 'Run not found' }, { status: 404 })) + } + + const authorize = options.authorize ?? defaultAuthorize + + return Effect.gen(function* () { + const decision = yield* authorize(options.request, { + access: 'read', + name: 'runs.events', + runId: options.runId, + }) + + const denied = authResponse(decision) + + if (denied) { + return denied + } + + const cursor = subscriptionFromSeq(url, options.request.headers.get('last-event-id')) + + if (!cursor.ok) { + return Response.json({ error: cursor.message }, { status: 400 }) + } + + const createPair = options.createPair ?? createWebSocketPair + const pair = createPair() + + if (!pair) { + return new Response('WebSocket upgrade is not available', { status: 426 }) + } + + installAutoResponse(options.ctx) + options.ctx.acceptWebSocket(pair.server, [options.runId]) + + const filter = subscriptionFilter(url) + const attachment = attachmentFromFilter(filter, cursor.fromSeq, null) + const socket = wrapSocket(pair.server) + socket.setAttachment(attachment) + + const loaded = yield* loadSubscriptionOpening(options.store, options.runId, cursor.fromSeq).pipe( + Effect.option, + ) + + if (Option.isNone(loaded)) { + return Response.json({ error: 'Event log is unavailable' }, { status: 500 }) + } + + const current = socket.attachment() ?? attachment + const next = openSubscription(socket, current, loaded.value) + + if (next) { + socket.setAttachment(next) + } + + return new Response(null, { status: 101, webSocket: pair.client }) + }) +} + +function tapAppend( + store: EventStore, + onAppend: (events: readonly EventEnvelope[]) => Effect.Effect, +): EventStore { + return { + ...store, + append: (runId, events, appendOptions) => + Effect.gen(function* () { + const result = yield* store.append(runId, events, appendOptions) + const first = result.sequences[0] + + if (first === undefined) { + return result + } + + yield* Effect.gen(function* () { + const appended = yield* store.read(runId, { + fromSeq: first, + limit: result.sequences.length, + }) + + if (appended.length === 0) { + return + } + + yield* onAppend(appended) + }).pipe(Effect.ignore) + + return result + }), + } +} + +function acceptedSockets( + ctx: HibernatingSocketState, + runId: string, +): readonly AttachedSubscriptionSocket[] { + if (!ctx.getWebSockets) { + return [] + } + + return ctx.getWebSockets(runId).map(wrapSocket) +} + +function wrapSocket(ws: WebSocket): AttachedSubscriptionSocket { + return { + send: (data) => { + ws.send(data) + }, + close: (code, reason) => { + ws.close(code, reason) + }, + get bufferedAmount() { + return Predicate.isNumber(ws.bufferedAmount) ? ws.bufferedAmount : 0 + }, + attachment: () => readAttachment(ws), + setAttachment: (attachment) => { + writeAttachment(ws, attachment) + }, + } +} + +function readAttachment(ws: WebSocket): SubscriptionAttachment | undefined { + if (!('deserializeAttachment' in ws)) { + return undefined + } + + // oxlint-disable-next-line typescript/unbound-method -- hibernation methods are own properties of the platform socket + const read: unknown = ws.deserializeAttachment + + if (!Predicate.isFunction(read)) { + return undefined + } + + const value: unknown = read.call(ws) + + if (!Predicate.isObject(value) || !('cursor' in value) || !Predicate.isNumber(value.cursor)) { + return undefined + } + + const principal = + 'principal' in value && (value.principal === null || Predicate.isString(value.principal)) + ? value.principal + : null + + const threadId = + 'threadId' in value && Predicate.isString(value.threadId) ? value.threadId : null + + const includeEphemeral = !('includeEphemeral' in value) || value.includeEphemeral !== false + + return { + principal, + cursor: value.cursor, + threadId, + includeEphemeral, + } +} + +function writeAttachment(ws: WebSocket, attachment: SubscriptionAttachment): void { + if (!('serializeAttachment' in ws)) { + return + } + + // oxlint-disable-next-line typescript/unbound-method -- hibernation methods are own properties of the platform socket + const write: unknown = ws.serializeAttachment + + if (!Predicate.isFunction(write)) { + return + } + + write.call(ws, attachment) +} + +function installAutoResponse(ctx: HibernatingSocketState): void { + if (autoResponseInstalled.has(ctx) || !ctx.setWebSocketAutoResponse) { + return + } + + try { + ctx.setWebSocketAutoResponse(new WebSocketRequestResponsePair('ping', 'pong')) + autoResponseInstalled.add(ctx) + } catch { + // Hosts without the hibernation helper still accept the socket. + } +} + +interface WorkersSocketPair { + readonly 0: WebSocket + readonly 1: WebSocket +} + +interface WorkersGlobals { + readonly WebSocketPair?: new () => WorkersSocketPair +} + +function createWebSocketPair(): { client: WebSocket; server: WebSocket } | undefined { + // SAFETY: Workers adds WebSocketPair to the global object. Other hosts leave it absent. + // oxlint-disable-next-line typescript/no-unsafe-type-assertion + const candidate = (globalThis as WorkersGlobals).WebSocketPair + + if (!Predicate.isFunction(candidate)) { + return undefined + } + + let pair: unknown + + try { + pair = Reflect.construct(candidate, []) + } catch { + return undefined + } + + if (!isSocketPair(pair)) { + return undefined + } + + return { client: pair[0], server: pair[1] } +} + +function isSocket(value: unknown): value is WebSocket { + return ( + Predicate.isObject(value) && + 'send' in value && + Predicate.isFunction(value.send) && + 'close' in value && + Predicate.isFunction(value.close) + ) +} + +function isSocketPair(value: unknown): value is { 0: WebSocket; 1: WebSocket } { + return Predicate.isObject(value) && isSocket(value[0]) && isSocket(value[1]) +} + +function runEventsPath(url: URL): string | undefined { + const match = EVENTS_PATH.exec(url.pathname) + const raw = match?.[1] + + if (!raw) { + return undefined + } + + try { + return decodeURIComponent(raw) + } catch { + return undefined + } +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c1ee11b..536b8a4 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -16,6 +16,7 @@ export * from './envelope' export * from './catalog' export * from './effects' export * from './protocol' +export * from './subscription' export * from './thread' export * from './state' export * from './match' diff --git a/packages/core/src/subscription.test.ts b/packages/core/src/subscription.test.ts new file mode 100644 index 0000000..6f8ca51 --- /dev/null +++ b/packages/core/src/subscription.test.ts @@ -0,0 +1,200 @@ +import { describe, expect, test } from 'bun:test' + +import { createEvent } from './envelope' +import { + attachmentFromFilter, + decodeSubscriptionFrame, + encodeSubscriptionFrame, + eventMatchesSubscription, + fanoutSubscription, + openSubscription, + pushSubscriptionEvents, + readSubscriptionData, + subscriptionFilter, + subscriptionFromSeq, + subscriptionReplay, + type AttachedSubscriptionSocket, + type SubscriptionAttachment, +} from './subscription' + +function envelope(seq: number, threadId: string, ephemeral = false) { + return { + ...createEvent('run_1', { + type: 'runtime.thread.started', + payload: { + threadId, + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v1', + input: null, + parentThreadId: null, + }, + threadId, + origin: { type: 'system' }, + ephemeral, + }), + seq, + } +} + +class FakeSocket implements AttachedSubscriptionSocket { + readonly sent: string[] = [] + closed: { code: number; reason: string } | undefined + bufferedAmount = 0 + private current: SubscriptionAttachment | undefined + + constructor(attachment: SubscriptionAttachment) { + this.current = attachment + } + + send(data: string): void { + if (this.closed) { + throw new Error('closed') + } + + this.sent.push(data) + } + + close(code: number, reason: string): void { + this.closed = { code, reason } + } + + attachment(): SubscriptionAttachment | undefined { + return this.current + } + + setAttachment(attachment: SubscriptionAttachment): void { + this.current = attachment + } +} + +describe('subscription protocol', () => { + test('decodes hello, a typeless batch, and resync', () => { + expect(decodeSubscriptionFrame('{"type":"hello","head":1,"tail":4}')).toEqual({ + type: 'hello', + head: 1, + tail: 4, + }) + + const batch = decodeSubscriptionFrame('{"batch":[{"type":"runtime.run.started"}]}') + expect(batch?.type).toBe('batch') + + expect(encodeSubscriptionFrame({ type: 'batch', batch: [{ type: 'runtime.run.started' }] })).toBe( + '{"batch":[{"type":"runtime.run.started"}]}', + ) + + expect(decodeSubscriptionFrame('{"type":"resync","head":3,"reason":"trimmed"}')).toEqual({ + type: 'resync', + head: 3, + reason: 'trimmed', + }) + + expect(decodeSubscriptionFrame('not-json')).toBeUndefined() + }) + + test('cursor and filter follow the query string', () => { + const url = new URL('http://looms.test/runs/run_1/events?fromSeq=4&threadId=thr_a&includeEphemeral=0') + + expect(subscriptionFromSeq(url, null)).toEqual({ ok: true, fromSeq: 4 }) + expect(subscriptionFromSeq(url, '2')).toEqual({ ok: true, fromSeq: 3 }) + expect(subscriptionFromSeq(url, 'nope').ok).toBe(false) + expect(subscriptionFilter(url)).toEqual({ threadId: 'thr_a', includeEphemeral: false }) + }) + + test('a cursor below the retained head asks for resync', () => { + expect(subscriptionReplay(1, 1)).toEqual({ kind: 'play', fromSeq: 1 }) + expect(subscriptionReplay(5, 4)).toEqual({ kind: 'resync', head: 5 }) + }) + + test('replay sends hello then the matching events and resumes the cursor', () => { + const socket = new FakeSocket( + attachmentFromFilter({ threadId: 'thr_a', includeEphemeral: false }, 1, null), + ) + + const next = openSubscription(socket, socket.attachment()!, { + kind: 'open', + hello: { type: 'hello', head: 1, tail: 3 }, + events: [ + envelope(1, 'thr_a'), + envelope(2, 'thr_b'), + envelope(3, 'thr_a', true), + ], + }) + + expect(socket.sent.map((line) => JSON.parse(line))).toEqual([ + { type: 'hello', head: 1, tail: 3 }, + { batch: [expect.objectContaining({ seq: 1, threadId: 'thr_a' })] }, + ]) + + expect(next?.cursor).toBe(3) + expect(socket.closed).toBeUndefined() + }) + + test('a trimmed cursor closes with resync', () => { + const socket = new FakeSocket(attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, 'ada')) + + const next = openSubscription(socket, socket.attachment()!, { kind: 'resync', head: 8 }) + + expect(next).toBeUndefined() + + expect(JSON.parse(socket.sent[0] ?? '{}')).toEqual({ + type: 'resync', + head: 8, + reason: 'trimmed', + }) + + expect(socket.closed).toEqual({ code: 4000, reason: 'trimmed' }) + }) + + test('backpressure closes the socket and a later append does not write to it', () => { + const slow = new FakeSocket(attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, null)) + slow.bufferedAmount = 32 + const live = new FakeSocket(attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, null)) + + fanoutSubscription([slow, live], [envelope(1, 'thr_a')], { bufferLimitBytes: 8, head: 4 }) + + expect(slow.closed?.reason).toBe('backpressure') + expect(JSON.parse(slow.sent[0] ?? '{}')).toMatchObject({ type: 'resync', reason: 'backpressure', head: 4 }) + expect(live.closed).toBeUndefined() + expect(live.attachment()?.cursor).toBe(1) + + fanoutSubscription([slow, live], [envelope(2, 'thr_a')], { bufferLimitBytes: 8, head: 4 }) + expect(live.attachment()?.cursor).toBe(2) + expect(slow.sent).toHaveLength(1) + }) + + test('readSubscriptionData keeps events and surfaces resync', () => { + const event = envelope(1, 'thr_a') + const encoded = encodeSubscriptionFrame({ type: 'batch', batch: [event] }) + const read = readSubscriptionData(encoded, 'run_1') + + expect(read.kind).toBe('events') + + if (read.kind === 'events') { + expect(read.events[0]?.seq).toBe(1) + } + + expect(readSubscriptionData('{"type":"resync","head":9,"reason":"trimmed"}', 'run_1')).toEqual({ + kind: 'resync', + head: 9, + reason: 'trimmed', + }) + }) + + test('filter drops other threads and ephemeral events', () => { + const filter = { threadId: 'thr_a', includeEphemeral: false } + + expect(eventMatchesSubscription(envelope(1, 'thr_a'), filter)).toBe(true) + expect(eventMatchesSubscription(envelope(2, 'thr_b'), filter)).toBe(false) + expect(eventMatchesSubscription(envelope(3, 'thr_a', true), filter)).toBe(false) + }) + + test('push does not resend a seq the attachment already passed', () => { + const attachment = { ...attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, null), cursor: 1 } + const socket = new FakeSocket(attachment) + const next = pushSubscriptionEvents(socket, [envelope(1, 'thr_a'), envelope(2, 'thr_a')], attachment) + + expect(socket.sent).toHaveLength(1) + expect(next?.cursor).toBe(2) + }) +}) diff --git a/packages/core/src/subscription.ts b/packages/core/src/subscription.ts new file mode 100644 index 0000000..aaff364 --- /dev/null +++ b/packages/core/src/subscription.ts @@ -0,0 +1,386 @@ +import { Option, Predicate, Schema } from 'effect' + +import { decodeLoomsEvent, encodeLoomsEvent } from './event-codec' +import type { EventEnvelope } from './envelope' +import { stringifyJson, type JsonValue } from './types' + +/** A socket is closed with this code when the client must resume from `head`. */ +export const SUBSCRIPTION_RESYNC_CODE = 4000 + +/** Close a socket that has fallen this far behind instead of buffering without bound. */ +export const SUBSCRIPTION_BACKPRESSURE_BYTES = 1024 * 1024 + +export type SubscriptionTransport = 'websocket' | 'sse' | 'auto' + +export type SubscriptionResyncReason = 'trimmed' | 'backpressure' + +export interface SubscriptionHello { + readonly type: 'hello' + readonly head: number + readonly tail: number +} + +export interface SubscriptionBatch { + readonly type: 'batch' + readonly batch: readonly JsonValue[] +} + +export interface SubscriptionResync { + readonly type: 'resync' + readonly head: number + readonly reason: SubscriptionResyncReason +} + +export type SubscriptionFrame = SubscriptionHello | SubscriptionBatch | SubscriptionResync + +export interface SubscriptionFilter { + readonly threadId: string | null + readonly includeEphemeral: boolean +} + +/** State hibernated with a Durable Object WebSocket. Writes stay on HTTP. */ +export interface SubscriptionAttachment { + readonly principal: string | null + readonly cursor: number + readonly threadId: string | null + readonly includeEphemeral: boolean +} + +export interface SubscriptionSender { + send(data: string): void + close(code: number, reason: string): void + readonly bufferedAmount: number +} + +export interface AttachedSubscriptionSocket extends SubscriptionSender { + attachment(): SubscriptionAttachment | undefined + setAttachment(attachment: SubscriptionAttachment): void +} + +export interface SubscriptionPushOptions { + readonly bufferLimitBytes?: number + readonly head?: number +} + +export type SubscriptionOpening = + | { readonly kind: 'resync'; readonly head: number } + | { + readonly kind: 'open' + readonly hello: SubscriptionHello + readonly events: readonly EventEnvelope[] + } + +export type SubscriptionRead = + | { readonly kind: 'events'; readonly events: readonly EventEnvelope[] } + | { readonly kind: 'hello'; readonly head: number; readonly tail: number } + | { readonly kind: 'resync'; readonly head: number; readonly reason: SubscriptionResyncReason } + | { readonly kind: 'ignore' } + +const decodeJsonArray = Schema.decodeUnknownOption(Schema.Array(Schema.Json)) +const asJson = Schema.decodeUnknownOption(Schema.Json) + +export function encodeSubscriptionFrame(frame: SubscriptionFrame): string { + switch (frame.type) { + case 'hello': + return stringifyJson({ type: 'hello', head: frame.head, tail: frame.tail }) + case 'resync': + return stringifyJson({ type: 'resync', head: frame.head, reason: frame.reason }) + case 'batch': + return stringifyJson({ batch: frame.batch }) + + default: { + const unexpected: never = frame + return unexpected + } + } +} + +export function decodeSubscriptionFrame(data: string): SubscriptionFrame | undefined { + let parsed: unknown + + try { + parsed = JSON.parse(data) + } catch { + return undefined + } + + if (!Predicate.isObject(parsed)) { + return undefined + } + + if ('type' in parsed && parsed.type === 'hello') { + const head = + 'head' in parsed && Predicate.isNumber(parsed.head) && Number.isFinite(parsed.head) + ? parsed.head + : undefined + + const tail = + 'tail' in parsed && Predicate.isNumber(parsed.tail) && Number.isFinite(parsed.tail) + ? parsed.tail + : undefined + + if (head === undefined || tail === undefined || head < 1 || tail < 0) { + return undefined + } + + return { type: 'hello', head, tail } + } + + if ('type' in parsed && parsed.type === 'resync') { + const head = + 'head' in parsed && Predicate.isNumber(parsed.head) && Number.isFinite(parsed.head) + ? parsed.head + : undefined + + const reason = + 'reason' in parsed && (parsed.reason === 'trimmed' || parsed.reason === 'backpressure') + ? parsed.reason + : undefined + + if (head === undefined || reason === undefined || head < 1) { + return undefined + } + + return { type: 'resync', head, reason } + } + + if (!('batch' in parsed)) { + return undefined + } + + const batch = decodeJsonArray(parsed.batch) + + if (Option.isNone(batch)) { + return undefined + } + + return { type: 'batch', batch: batch.value } +} + +/** One cursor for SSE and WebSocket. `Last-Event-ID` is the last seen seq. */ +export function subscriptionFromSeq( + url: URL, + lastEventId: string | null, +): { readonly ok: true; readonly fromSeq: number } | { readonly ok: false; readonly message: string } { + if (lastEventId !== null && lastEventId.trim() !== '') { + const value = Number(lastEventId) + + if (!Number.isSafeInteger(value) || value < 0) { + return { ok: false, message: 'Last-Event-ID must be a non-negative integer' } + } + + return { ok: true, fromSeq: value + 1 } + } + + const raw = url.searchParams.get('fromSeq') + + if (raw === null) { + return { ok: true, fromSeq: 1 } + } + + const fromSeq = Number(raw) + + if (!Number.isSafeInteger(fromSeq) || fromSeq < 1) { + return { ok: false, message: 'fromSeq must be an integer greater than or equal to 1' } + } + + return { ok: true, fromSeq } +} + +export function subscriptionFilter(url: URL): SubscriptionFilter { + const threadId = url.searchParams.get('threadId') + const includeEphemeral = url.searchParams.get('includeEphemeral') + + return { + threadId: threadId && threadId.length > 0 ? threadId : null, + includeEphemeral: includeEphemeral !== 'false' && includeEphemeral !== '0', + } +} + +export function subscriptionReplay( + head: number, + fromSeq: number, +): { readonly kind: 'resync'; readonly head: number } | { readonly kind: 'play'; readonly fromSeq: number } { + if (fromSeq < head) { + return { kind: 'resync', head } + } + + return { kind: 'play', fromSeq } +} + +export function eventMatchesSubscription( + event: EventEnvelope, + filter: SubscriptionFilter, +): boolean { + if (!filter.includeEphemeral && event.ephemeral) { + return false + } + + if (filter.threadId !== null && event.threadId !== filter.threadId) { + return false + } + + return true +} + +export function attachmentFromFilter( + filter: SubscriptionFilter, + fromSeq: number, + principal: string | null, +): SubscriptionAttachment { + return { + principal, + cursor: fromSeq - 1, + threadId: filter.threadId, + includeEphemeral: filter.includeEphemeral, + } +} + +export function readSubscriptionData(data: string, runId: string): SubscriptionRead { + const frame = decodeSubscriptionFrame(data) + + if (!frame) { + return { kind: 'ignore' } + } + + switch (frame.type) { + case 'hello': + return { kind: 'hello', head: frame.head, tail: frame.tail } + case 'resync': + return { kind: 'resync', head: frame.head, reason: frame.reason } + + case 'batch': { + const events: EventEnvelope[] = [] + + for (const item of frame.batch) { + try { + events.push(decodeLoomsEvent(item, runId)) + } catch { + // A bad item does not drop the rest of the batch. + } + } + + return { kind: 'events', events } + } + + default: { + const unexpected: never = frame + return unexpected + } + } +} + +/** + * Sends the events this attachment still needs. Returns the advanced attachment, + * or `undefined` after the socket is closed for resync. + */ +export function pushSubscriptionEvents( + socket: SubscriptionSender, + events: readonly EventEnvelope[], + attachment: SubscriptionAttachment, + options?: SubscriptionPushOptions, +): SubscriptionAttachment | undefined { + const limit = options?.bufferLimitBytes ?? SUBSCRIPTION_BACKPRESSURE_BYTES + + if (socket.bufferedAmount > limit) { + closeForResync(socket, options?.head ?? 1, 'backpressure') + return undefined + } + + const filter: SubscriptionFilter = { + threadId: attachment.threadId, + includeEphemeral: attachment.includeEphemeral, + } + + let cursor = attachment.cursor + const batch: JsonValue[] = [] + + for (const event of events) { + if (event.seq <= cursor) { + continue + } + + cursor = event.seq + + if (eventMatchesSubscription(event, filter)) { + const parsed: unknown = JSON.parse(stringifyJson(encodeLoomsEvent(event))) + const encoded = asJson(parsed) + + if (Option.isSome(encoded)) { + batch.push(encoded.value) + } + } + } + + if (batch.length > 0 && !sendFrame(socket, { type: 'batch', batch })) { + closeForResync(socket, options?.head ?? 1, 'backpressure') + return undefined + } + + return { ...attachment, cursor } +} + +export function openSubscription( + socket: SubscriptionSender, + attachment: SubscriptionAttachment, + opening: SubscriptionOpening, + options?: SubscriptionPushOptions, +): SubscriptionAttachment | undefined { + if (opening.kind === 'resync') { + closeForResync(socket, opening.head, 'trimmed') + return undefined + } + + if (!sendFrame(socket, opening.hello)) { + closeForResync(socket, opening.hello.head, 'backpressure') + return undefined + } + + return pushSubscriptionEvents(socket, opening.events, attachment, { + ...options, + head: opening.hello.head, + }) +} + +export function fanoutSubscription( + sockets: readonly AttachedSubscriptionSocket[], + events: readonly EventEnvelope[], + options?: SubscriptionPushOptions, +): void { + for (const socket of sockets) { + const current = socket.attachment() + + if (!current) { + continue + } + + const next = pushSubscriptionEvents(socket, events, current, options) + + if (next) { + socket.setAttachment(next) + } + } +} + +function closeForResync( + socket: SubscriptionSender, + head: number, + reason: SubscriptionResyncReason, +): void { + sendFrame(socket, { type: 'resync', head, reason }) + + try { + socket.close(SUBSCRIPTION_RESYNC_CODE, reason) + } catch { + // The platform may already have dropped the socket. + } +} + +function sendFrame(socket: SubscriptionSender, frame: SubscriptionFrame): boolean { + try { + socket.send(encodeSubscriptionFrame(frame)) + return true + } catch { + return false + } +} diff --git a/packages/react/src/react.ts b/packages/react/src/react.ts index 89f2ae1..13459a2 100644 --- a/packages/react/src/react.ts +++ b/packages/react/src/react.ts @@ -20,6 +20,7 @@ import { type RegisteredEvent, type RunLaunchSource, type RunStatus, + type SubscriptionTransport, type ThreadTree, } from '@looms/core' @@ -35,11 +36,13 @@ export interface LoomsProviderProps { children: ReactNode endpoint?: string coalesce?: CoalesceOption + transport?: SubscriptionTransport } interface ProviderContextValue { endpoint: string coalesce?: CoalesceOption + transport?: SubscriptionTransport } const LoomsContext = createContext({ endpoint: '' }) @@ -53,8 +56,23 @@ interface RegistryEntry { const registry = new Map() -function registryKey(endpoint: string, runId: string): string { - return `${endpoint}\0${runId}` +interface StoreIdentity { + readonly endpoint: string + readonly reconnectDelayMs?: number + readonly coalesce?: CoalesceOption + readonly transport?: SubscriptionTransport + readonly threadId?: string + readonly includeEphemeral?: boolean +} + +function registryKey(runId: string, identity: StoreIdentity): string { + return [ + identity.endpoint, + runId, + identity.transport ?? '', + identity.threadId ?? '', + identity.includeEphemeral === false ? '0' : '1', + ].join('\0') } function disposeEntry(key: string): void { @@ -74,13 +92,8 @@ function scheduleDisposal(key: string): Fiber.Fiber { ) } -function peekStore( - runId: string, - endpoint: string, - reconnectDelayMs?: number, - coalesce?: CoalesceOption, -): LoomsClientStore { - const key = registryKey(endpoint, runId) +function peekStore(runId: string, identity: StoreIdentity): LoomsClientStore { + const key = registryKey(runId, identity) const existing = registry.get(key) if (existing) { @@ -89,9 +102,12 @@ function peekStore( const store = createLoomsStore({ storeId: runId, - endpoint, - reconnectDelayMs, - coalesce, + endpoint: identity.endpoint, + reconnectDelayMs: identity.reconnectDelayMs, + coalesce: identity.coalesce, + transport: identity.transport, + threadId: identity.threadId, + includeEphemeral: identity.includeEphemeral, }) registry.set(key, { @@ -103,14 +119,9 @@ function peekStore( return store } -function retainStore( - runId: string, - endpoint: string, - reconnectDelayMs?: number, - coalesce?: CoalesceOption, -): void { - const key = registryKey(endpoint, runId) - peekStore(runId, endpoint, reconnectDelayMs, coalesce) +function retainStore(runId: string, identity: StoreIdentity): void { + const key = registryKey(runId, identity) + peekStore(runId, identity) const entry = registry.get(key) if (!entry) { @@ -125,8 +136,8 @@ function retainStore( } } -function releaseStore(runId: string, endpoint: string): void { - const key = registryKey(endpoint, runId) +function releaseStore(runId: string, identity: StoreIdentity): void { + const key = registryKey(runId, identity) const entry = registry.get(key) if (!entry) { @@ -141,8 +152,13 @@ function releaseStore(runId: string, endpoint: string): void { } /** Root provider that supplies the Looms host endpoint to run stores. */ -export function LoomsProvider({ children, endpoint = '', coalesce }: LoomsProviderProps) { - const value = useMemo(() => ({ endpoint, coalesce }), [endpoint, coalesce]) +export function LoomsProvider({ + children, + endpoint = '', + coalesce, + transport, +}: LoomsProviderProps) { + const value = useMemo(() => ({ endpoint, coalesce, transport }), [endpoint, coalesce, transport]) return createElement(LoomsContext.Provider, { value, children }) } @@ -150,6 +166,9 @@ export interface UseRunStoreOptions { endpoint?: string reconnectDelayMs?: number coalesce?: CoalesceOption + transport?: SubscriptionTransport + threadId?: string + includeEphemeral?: boolean } /** @@ -162,12 +181,25 @@ export function useRunStore(runId: string, options?: UseRunStoreOptions): LoomsC const endpoint = options?.endpoint ?? context.endpoint const reconnectDelayMs = options?.reconnectDelayMs const coalesce = options?.coalesce ?? context.coalesce - const store = peekStore(runId, endpoint, reconnectDelayMs, coalesce) + const transport = options?.transport ?? context.transport + const threadId = options?.threadId + const includeEphemeral = options?.includeEphemeral + + const identity = { + endpoint, + reconnectDelayMs, + coalesce, + transport, + threadId, + includeEphemeral, + } + + const store = peekStore(runId, identity) useEffect(() => { - retainStore(runId, endpoint, reconnectDelayMs, coalesce) - return () => releaseStore(runId, endpoint) - }, [runId, endpoint, reconnectDelayMs, coalesce]) + retainStore(runId, identity) + return () => releaseStore(runId, identity) + }, [runId, endpoint, reconnectDelayMs, coalesce, transport, threadId, includeEphemeral]) return store } diff --git a/packages/react/src/store.test.ts b/packages/react/src/store.test.ts index a277ef2..eab144b 100644 --- a/packages/react/src/store.test.ts +++ b/packages/react/src/store.test.ts @@ -285,3 +285,62 @@ describe('createLoomsStore reconnect', () => { } }) }) + +describe('createLoomsStore websocket', () => { + test('applies batches and resumes from a resync head', async () => { + class FakeSocket { + static latest: FakeSocket | undefined + readonly url: string + private readonly listeners = new Map void>>() + + constructor(url: string) { + this.url = url + FakeSocket.latest = this + } + + addEventListener(type: string, listener: (event: { readonly data?: unknown }) => void): void { + const current = this.listeners.get(type) ?? [] + current.push(listener) + this.listeners.set(type, current) + } + + emit(type: string, data?: string): void { + for (const listener of this.listeners.get(type) ?? []) { + listener({ data }) + } + } + + close(): void { + this.emit('close') + } + } + + const store = createLoomsStore({ + storeId: 'run_1', + endpoint: 'http://127.0.0.1:9', + transport: 'websocket', + reconnectDelayMs: 20, + webSocket: FakeSocket, + }) + + await waitFor(() => FakeSocket.latest !== undefined) + const first = FakeSocket.latest + + if (!first) { + throw new Error('missing socket') + } + + first.emit('message', JSON.stringify({ type: 'hello', head: 1, tail: 0 })) + expect(store.events()).toHaveLength(0) + + first.emit( + 'message', + JSON.stringify({ batch: [encodeLoomsEvent(event(1, 'runtime.run.started'))] }), + ) + + await waitFor(() => store.events().length === 1) + first.emit('message', JSON.stringify({ type: 'resync', head: 4, reason: 'trimmed' })) + await waitFor(() => FakeSocket.latest?.url.includes('fromSeq=4') === true) + store.dispose() + }) +}) diff --git a/packages/react/src/store.ts b/packages/react/src/store.ts index 4f539ef..393c377 100644 --- a/packages/react/src/store.ts +++ b/packages/react/src/store.ts @@ -2,18 +2,21 @@ import { Data, Effect, Fiber, Option, Predicate, Schema } from 'effect' import { decodeLoomsEvent, + readSubscriptionData, stringifyJson, type AnyEventEnvelope, type EventEnvelope, type EventInput, type ProjectionDefinition, type RegisteredEvent, + type SubscriptionTransport, } from '@looms/core' +import type { SubscriptionSocketConstructor } from '@looms/client' import { EventIndex, ProjectionCache, type LiveFoldDefinition } from './derived' import { materializeEvents } from './materialize' import { createNotifyScheduler, type CoalesceOption } from './notify' -import { consumeSseStreamEffect, eventsFromSseData } from './sse' +import { consumeSseStreamEffect } from './sse' import { emptyTables, type MaterializedTables } from './tables' export type { CoalesceOption } from './notify' @@ -41,10 +44,18 @@ export interface LoomsClientStoreOptions { * Opens `${endpoint}/runs/:id/events?live=true` as an SSE stream. */ endpoint: string - /** Backoff between SSE reconnects. */ + /** Backoff between reconnects. The delay stays fixed, matching the SSE client. */ reconnectDelayMs?: number fetch?: typeof fetch coalesce?: CoalesceOption + /** + * `undefined` keeps the current SSE path. `websocket` and `auto` use a hibernating + * socket when a constructor is available. + */ + transport?: SubscriptionTransport + threadId?: string + includeEphemeral?: boolean + webSocket?: SubscriptionSocketConstructor } export interface LoomsStoreQuery { @@ -74,6 +85,24 @@ export interface LoomsClientStore void } +function socketConstructor( + options: LoomsClientStoreOptions, +): SubscriptionSocketConstructor | undefined { + if (options.transport !== 'websocket' && options.transport !== 'auto') { + return undefined + } + + if (options.webSocket) { + return options.webSocket + } + + if (!('WebSocket' in globalThis) || !Predicate.isFunction(globalThis.WebSocket)) { + return undefined + } + + return globalThis.WebSocket +} + function eventSourceCtor(): typeof EventSource | undefined { if (typeof EventSource === 'undefined') { return undefined @@ -108,6 +137,7 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS let liveStarted = false let liveAbort: AbortController | undefined let eventSource: EventSource | undefined + let socket: { close(code?: number, reason?: string): void } | undefined let reconnectFiber: Fiber.Fiber | undefined let syncing: Promise | null = null @@ -169,12 +199,39 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS notify.schedule() } - const applyEncoded = (data: string) => { - applyBatch(eventsFromSseData(data, options.storeId)) + const applyLiveData = (data: string): 'continue' | 'resync' => { + const read = readSubscriptionData(data, options.storeId) + + switch (read.kind) { + case 'events': + applyBatch(read.events.slice()) + return 'continue' + case 'resync': + fromSeq = read.head + return 'resync' + case 'hello': + case 'ignore': + return 'continue' + + default: { + const unexpected: never = read + return unexpected + } + } } const liveUrl = () => { - return `${endpoint}/runs/${encodeURIComponent(options.storeId)}/events?live=true&fromSeq=${fromSeq}` + const params = new URLSearchParams({ live: 'true', fromSeq: String(fromSeq) }) + + if (options.threadId) { + params.set('threadId', options.threadId) + } + + if (options.includeEphemeral === false) { + params.set('includeEphemeral', 'false') + } + + return `${endpoint}/runs/${encodeURIComponent(options.storeId)}/events?${params.toString()}` } const pull = Effect.gen(function* () { @@ -238,7 +295,7 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS return syncing } - const scheduleReconnect = (Ctor: typeof EventSource) => { + const scheduleReconnect = (start: () => void) => { if (disposed || !liveStarted || reconnectFiber !== undefined) { return } @@ -258,7 +315,7 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS }).pipe(Effect.ignore) if (!disposed && liveStarted) { - startEventSource(Ctor) + start() } }), ) @@ -278,7 +335,19 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS }) es.addEventListener('message', (msg) => { - applyEncoded(msg.data) + if (applyLiveData(msg.data) === 'resync') { + es.close() + + if (eventSource === es) { + eventSource = undefined + } + + setStatus('reconnecting') + + scheduleReconnect(() => { + startEventSource(Ctor) + }) + } }) es.addEventListener('error', () => { @@ -295,7 +364,9 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS eventSource = undefined } - scheduleReconnect(Ctor) + scheduleReconnect(() => { + startEventSource(Ctor) + }) } else if (!live) { setStatus('reconnecting') } @@ -336,9 +407,9 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS live = true setStatus('live') - return yield* consumeSseStreamEffect(body, (frame) => { - applyEncoded(frame.data) - }) + return yield* consumeSseStreamEffect(body, (frame) => + applyLiveData(frame.data) === 'resync' ? false : undefined, + ) }).pipe( Effect.catch(() => Effect.sync(() => { @@ -362,6 +433,57 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS void Effect.runPromise(run, { signal: controller.signal }).catch(() => undefined) } + const startWebSocket = (Ctor: SubscriptionSocketConstructor) => { + if (disposed || !liveStarted) { + return + } + + let url: string + + try { + const http = new URL(liveUrl()) + http.protocol = http.protocol === 'https:' ? 'wss:' : 'ws:' + url = http.toString() + } catch { + startFetchStream() + return + } + + const ws = new Ctor(url) + socket = ws + + ws.addEventListener('open', () => { + live = true + setStatus('live') + }) + + ws.addEventListener('message', (event) => { + if (!Predicate.isString(event.data)) { + return + } + + if (applyLiveData(event.data) === 'resync') { + ws.close() + } + }) + + ws.addEventListener('close', () => { + live = false + + if (socket === ws) { + socket = undefined + } + + if (!disposed && liveStarted) { + setStatus('reconnecting') + + scheduleReconnect(() => { + startWebSocket(Ctor) + }) + } + }) + } + const startLive = () => { if (!options.storeId || (liveStarted && !disposed)) { return @@ -369,6 +491,13 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS liveStarted = true disposed = false + const socketCtor = socketConstructor(options) + + if (socketCtor) { + startWebSocket(socketCtor) + return + } + const ctor = options.fetch === undefined ? eventSourceCtor() : undefined if (ctor) { @@ -389,6 +518,8 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS eventSource?.close() eventSource = undefined + socket?.close() + socket = undefined liveAbort?.abort() liveAbort = undefined } diff --git a/packages/runtime/src/events-api.test.ts b/packages/runtime/src/events-api.test.ts index ea89d41..d142656 100644 --- a/packages/runtime/src/events-api.test.ts +++ b/packages/runtime/src/events-api.test.ts @@ -61,10 +61,11 @@ describe('GET /runs/:id/events', () => { expect(res).not.toBeNull() expect(res!.headers.get('content-type')).toBe('text/event-stream') expect(res!.headers.get('cache-control')).toBe('no-cache, no-transform') - const frames = await readSseFrames(res!, 3) - expect(frames.length).toBeGreaterThanOrEqual(1) - expect(frames[0]).toContain('id: 1') - expect(frames[0]).toContain('data: ') + const frames = await readSseFrames(res!, 4) + const firstEvent = frames.find((frame) => frame.includes('id: 1')) + expect(firstEvent).toBeDefined() + expect(firstEvent).toContain('data: ') + expect(frames[0]).toContain('"type":"hello"') }) test('Last-Event-ID skips already-seen events', async () => { @@ -90,9 +91,11 @@ describe('GET /runs/:id/events', () => { ) expect(res).not.toBeNull() - const frames = await readSseFrames(res!, 1) - expect(frames[0]).toContain('id: 2') - expect(frames[0]).not.toContain('id: 1\n') + const frames = await readSseFrames(res!, 2) + const resumed = frames.find((frame) => frame.includes('id: 2')) + expect(resumed).toBeDefined() + expect(resumed).not.toContain('id: 1\n') + expect(frames.some((frame) => frame.includes('id: 1\n'))).toBe(false) }) test('non-live pull returns the event resource', async () => { diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index 1f041f8..e053801 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -22,6 +22,7 @@ export { type WorkerCallbackInput, } from './runtime' export { createLooms, type Looms, type CreateLoomsOptions, type StartResult } from './looms' +export { loadSubscriptionOpening, readSubscriptionBounds } from './subscription' export { serveHttp, createFetchHandler, @@ -29,6 +30,7 @@ export { type FetchHandlerOptions, authAllowed, authDenied, + authResponse, bearerAuth, defaultAuthorize, type AuthDecision, diff --git a/packages/runtime/src/server.ts b/packages/runtime/src/server.ts index b2e30b9..4218620 100644 --- a/packages/runtime/src/server.ts +++ b/packages/runtime/src/server.ts @@ -4,6 +4,7 @@ export { mountDebugger, type DebuggerMount } from './server/debugger-host' export { authAllowed, authDenied, + authResponse, bearerAuth, defaultAuthorize, type AuthDecision, diff --git a/packages/runtime/src/sse.test.ts b/packages/runtime/src/sse.test.ts index d9f4940..18ecce4 100644 --- a/packages/runtime/src/sse.test.ts +++ b/packages/runtime/src/sse.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from 'bun:test' import { Effect } from 'effect' -import { makeMemoryEventStore } from '@looms/core' +import { createEvent, makeMemoryEventStore } from '@looms/core' import { createEventStreamResponse } from './sse' @@ -55,4 +55,80 @@ describe('createEventStreamResponse', () => { await reader.cancel() abort.abort() }) + + test('tells the client to resync when the cursor was trimmed', async () => { + const store = await Effect.runPromise(makeMemoryEventStore) + + const event = createEvent('run_trim', { + type: 'runtime.thread.started', + payload: { + threadId: 'thr_a', + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v1', + input: null, + parentThreadId: null, + }, + threadId: 'thr_a', + origin: { type: 'system' }, + }) + + const second = createEvent('run_trim', { + type: 'runtime.thread.started', + payload: { + threadId: 'thr_a', + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v1', + input: null, + parentThreadId: null, + }, + threadId: 'thr_a', + origin: { type: 'system' }, + }) + + await Effect.runPromise(store.append('run_trim', [event, second], { expectedTail: 0 })) + + if (!store.trim) { + throw new Error('memory store does not trim') + } + + await Effect.runPromise(store.trim('run_trim', 2)) + + const abort = new AbortController() + + const res = createEventStreamResponse({ + signal: abort.signal, + store, + runId: 'run_trim', + fromSeq: 1, + heartbeatMs: 60_000, + }) + + const reader = res.body?.getReader() + + if (!reader) { + throw new Error('missing stream body') + } + + const decoder = new TextDecoder() + let text = '' + + while (!text.includes('resync')) { + const chunk = await reader.read() + + if (chunk.done) { + break + } + + text += decoder.decode(chunk.value) + } + + expect(text.startsWith(': keepalive\n\n')).toBe(true) + expect(text).toContain('"type":"resync"') + expect(text).toContain('"head":2') + expect(text).toContain('"reason":"trimmed"') + await reader.cancel() + abort.abort() + }) }) diff --git a/packages/runtime/src/sse.ts b/packages/runtime/src/sse.ts index 8e5ef99..db9ff5d 100644 --- a/packages/runtime/src/sse.ts +++ b/packages/runtime/src/sse.ts @@ -1,6 +1,13 @@ import { Effect, Fiber, Schedule, Stream } from 'effect' -import { encodeLoomsEvent, stringifyJson, type EventEnvelope, type EventStore } from '@looms/core' +import { + encodeLoomsEvent, + encodeSubscriptionFrame, + type EventEnvelope, + type EventStore, +} from '@looms/core' + +import { readSubscriptionBounds } from './subscription' export interface EventStreamOptions { signal: AbortSignal @@ -102,23 +109,52 @@ export function createEventStreamResponse(options: EventStreamOptions): Response } } - subscriptionFiber = Effect.runFork( - store.subscribe(runId, { fromSeq }).pipe( - Stream.tap((event: EventEnvelope) => - Effect.sync(() => { - write( - `id: ${event.seq}\ndata: ${stringifyJson({ batch: [encodeLoomsEvent(event)] })}\n\n`, - ) - }), - ), - Stream.runDrain, - ), - ) - // Send an immediate keepalive so bytes and headers flush over the wire immediately, // preventing dev server proxies and Bun.serve from aborting quiet connections. + // It has to be enqueued before the subscription fiber, which writes hello synchronously. write(': keepalive\n\n') + subscriptionFiber = Effect.runFork( + Effect.gen(function* () { + const bounds = yield* readSubscriptionBounds(store, runId) + + if (fromSeq < bounds.head) { + write( + `data: ${encodeSubscriptionFrame({ + type: 'resync', + head: bounds.head, + reason: 'trimmed', + } )}\n\n`, + ) + + close() + return + } + + write( + `data: ${encodeSubscriptionFrame({ + type: 'hello', + head: bounds.head, + tail: bounds.tail, + })}\n\n`, + ) + + yield* store.subscribe(runId, { fromSeq }).pipe( + Stream.tap((event: EventEnvelope) => + Effect.sync(() => { + write( + `id: ${event.seq}\ndata: ${encodeSubscriptionFrame({ + type: 'batch', + batch: [encodeLoomsEvent(event)], + })}\n\n`, + ) + }), + ), + Stream.runDrain, + ) + }), + ) + heartbeatFiber = Effect.runFork( Effect.repeat( Effect.sleep(heartbeatMs).pipe( diff --git a/packages/runtime/src/subscription.ts b/packages/runtime/src/subscription.ts new file mode 100644 index 0000000..903e0b1 --- /dev/null +++ b/packages/runtime/src/subscription.ts @@ -0,0 +1,45 @@ +import { Effect } from 'effect' + +import { + subscriptionReplay, + type EventEnvelope, + type EventStore, + type EventStoreError, + type SubscriptionHello, + type SubscriptionOpening, +} from '@looms/core' + +export function readSubscriptionBounds( + store: EventStore, + runId: string, +): Effect.Effect<{ readonly head: number; readonly tail: number }, EventStoreError> { + if (store.bounds) { + return store.bounds(runId) + } + + return store.tail(runId).pipe(Effect.map((tail) => ({ head: 1, tail }))) +} + +/** Hello plus the retained events, or a resync when `fromSeq` is below the log head. */ +export function loadSubscriptionOpening( + store: EventStore, + runId: string, + fromSeq: number, +): Effect.Effect { + return Effect.gen(function* () { + const bounds = yield* readSubscriptionBounds(store, runId) + const decision = subscriptionReplay(bounds.head, fromSeq) + + if (decision.kind === 'resync') { + return { kind: 'resync', head: decision.head } + } + + const events: readonly EventEnvelope[] = yield* store.read(runId, { + fromSeq: decision.fromSeq, + }) + + const hello: SubscriptionHello = { type: 'hello', head: bounds.head, tail: bounds.tail } + + return { kind: 'open', hello, events } + }) +} From 1b5e8da26cb164db38061f42b47795ccb86f1e70 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 11:48:21 +0200 Subject: [PATCH 06/23] feat(runtime)!: launch workflow children as their own runs Spawn, map, fanout, and loop can start a child run that signals the parent when it finishes. Cancelling the parent either stops those children immediately or waits until they finish cancelling. BREAKING CHANGE: RunLauncher implementations must provide signal and cancel. Co-authored-by: Cursor --- .changeset/child-runs.md | 9 + docs/modules.md | 2 + docs/production.md | 2 + packages/actor/src/cell.ts | 12 +- packages/actor/src/host.ts | 3 + packages/cloudflare/src/durable-object.ts | 26 ++- packages/core/src/effects.ts | 72 ++++++ packages/core/src/fold.ts | 55 ++++- packages/core/src/ids.ts | 5 + packages/core/src/launcher.ts | 38 ++++ packages/core/src/projection.ts | 3 + packages/core/src/protocol.ts | 18 ++ packages/core/src/state.ts | 24 +- .../debugger/src/events/protocol-summarize.ts | 7 + packages/runtime/src/cancellation.ts | 46 +++- packages/runtime/src/child-run.test.ts | 201 +++++++++++++++++ packages/runtime/src/child-run.ts | 86 +++++++ packages/runtime/src/dispatch-effect.ts | 16 ++ packages/runtime/src/local-effect-executor.ts | 47 +++- packages/runtime/src/planners.test.ts | 3 + packages/runtime/src/runtime.ts | 41 +++- packages/runtime/src/types.ts | 11 +- packages/runtime/src/wake-engine.ts | 50 ++++ packages/workflow/src/definitions.ts | 15 ++ packages/workflow/src/effects.ts | 15 +- packages/workflow/src/events.ts | 12 + packages/workflow/src/threads.ts | 213 +++++++++++++++--- 27 files changed, 971 insertions(+), 61 deletions(-) create mode 100644 .changeset/child-runs.md create mode 100644 packages/runtime/src/child-run.test.ts create mode 100644 packages/runtime/src/child-run.ts diff --git a/.changeset/child-runs.md b/.changeset/child-runs.md new file mode 100644 index 0000000..3be0a62 --- /dev/null +++ b/.changeset/child-runs.md @@ -0,0 +1,9 @@ +--- +'@swirls/looms': minor +--- + +feat(runtime)!: launch workflow children as their own runs + +A workflow can set `childExecution: 'run'` so spawn, map, fanout, and loop start a child run instead of a thread. The child run id is stable across retries. When the child finishes it signals `runtime.child.settled`, which completes the parent's wait. `childClose` is `terminate` or `wait`. The default remains in-process threads. + +BREAKING CHANGE: `RunLauncher` implementations must provide `signal` and `cancel`. diff --git a/docs/modules.md b/docs/modules.md index d8600a3..d8e929a 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -44,6 +44,8 @@ Each module owns its definitions. `workflow({ definitions: [checkout] })` instal At initialization, Looms rejects duplicate `(kind, name)` registrations and definitions whose kind is not implemented by their owning module. Starting or spawning an unregistered definition fails before any run events are written. An unsupported thread kind is rejected the same way; if such an event somehow appears in a log, fold marks the thread failed instead of inventing empty state. Pass all child definitions to their owning modules too. +`defineWorkflow({ childExecution: 'run' })` launches each spawned, mapped, or looped child as its own run instead of a thread in the parent. The child run id is derived from the parent run and the child thread id, so a retry finds the same run. The child appends `runtime.child.settled` to the parent when it finishes, and that event completes the parent's wait. `childClose: 'terminate'` (the default) cancels those runs when the parent is cancelled. `childClose: 'wait'` cancels the children first and cancels the parent only after they have finished. A missing `RunLauncher` fails the launch. The default `childExecution` is `'thread'`, which keeps the child inside the parent run. + `defineModule(options, setup)` returns a complete runtime module. The callback receives typed `effect`, `projection`, `thread`, `input`, and `emit` builders. Only returned members are installed; access them through `payments.effects.charge` and `payments.projections.ledger`. Event schemas can be inline or supplied as a reusable `defineEventCatalog`. Modules that implement a custom thread kind can return `definitions` alongside `threads`. The runtime gathers these for `start`, HTTP starts, and cross-module child spawning. diff --git a/docs/production.md b/docs/production.md index 866dee0..bcde618 100644 --- a/docs/production.md +++ b/docs/production.md @@ -13,6 +13,8 @@ coalesced. A deployment, eviction, or alarm retry may repeat a wake; event preconditions make that safe. Cross-object queries and transactions are not provided. Build global run lists with a projector. +A workflow can launch children with `childExecution: 'run'`. Each child is its own run, addressed by a deterministic id, and reports `runtime.child.settled` back to the parent. Cancelling the parent cancels those children. `childClose: 'wait'` records `runtime.child.close.requested` and cancels the parent only after the children reach a terminal status. `childClose: 'terminate'` cancels them as the parent closes. Children stay threads in the parent run unless `childExecution` is set. + All built-in stores reject an event over 1 MiB and a snapshot over 8 MiB, measured as serialized UTF-8, before storage is mutated. The stable error names are `EventPayloadTooLargeError` and `SnapshotPayloadTooLargeError`. diff --git a/packages/actor/src/cell.ts b/packages/actor/src/cell.ts index b5540b3..3f05e35 100644 --- a/packages/actor/src/cell.ts +++ b/packages/actor/src/cell.ts @@ -62,8 +62,14 @@ export interface ActorCell< startRun(args: StartRunArgs): Promise signal( events: ReadonlyArray, - options?: { idempotencyKey?: string }, + options?: { idempotencyKey?: string; coalesce?: boolean }, ): Promise + /** Append runtime-originated events. Not the client signal path. */ + accept( + events: ReadonlyArray, + options?: { idempotencyKey?: string; coalesce?: boolean }, + ): Promise + cancel(): Promise getRun(): Promise } @@ -138,6 +144,10 @@ export function createActorCell< signal: (events, signalOptions) => provide(runtime.signal(runId, events, signalOptions)), + accept: (events, signalOptions) => provide(runtime.accept(runId, events, signalOptions)), + + cancel: () => provide(runtime.cancel(runId).pipe(Effect.asVoid)), + getRun: () => provide(runtime.getRun(runId)), } } diff --git a/packages/actor/src/host.ts b/packages/actor/src/host.ts index 7976f53..fd685b2 100644 --- a/packages/actor/src/host.ts +++ b/packages/actor/src/host.ts @@ -101,6 +101,9 @@ export function createLocalActorHost< const launcher = createRemoteRunLauncher({ launch: (runId, args) => getCell(runId).then((cell) => cell.launch(args)), status: (runId) => getCell(runId).then((cell) => cell.status()), + accept: (runId, events, signalOptions) => + getCell(runId).then((cell) => cell.accept(events, signalOptions).then(() => undefined)), + cancel: (runId) => getCell(runId).then((cell) => cell.cancel()), }) getCell = (runId: string): Promise> => { diff --git a/packages/cloudflare/src/durable-object.ts b/packages/cloudflare/src/durable-object.ts index d5746df..ac84267 100644 --- a/packages/cloudflare/src/durable-object.ts +++ b/packages/cloudflare/src/durable-object.ts @@ -133,11 +133,22 @@ export abstract class LoomsDurableObject extends DurableObject, - options?: { idempotencyKey?: string }, + options?: { idempotencyKey?: string; coalesce?: boolean }, ): Promise { return this.getCell().then((cell) => cell.signal(events, options)) } + accept( + events: ReadonlyArray, + options?: { idempotencyKey?: string; coalesce?: boolean }, + ): Promise { + return this.getCell().then((cell) => cell.accept(events, options)) + } + + cancel(): Promise { + return this.getCell().then((cell) => cell.cancel()) + } + getRun(): Promise { return this.getCell().then((cell) => cell.getRun()) } @@ -165,8 +176,13 @@ export interface DurableObjectRunStub { startRun(args: StartRunArgs): Promise signal( events: ReadonlyArray, - options?: { idempotencyKey?: string }, + options?: { idempotencyKey?: string; coalesce?: boolean }, + ): Promise + accept( + events: ReadonlyArray, + options?: { idempotencyKey?: string; coalesce?: boolean }, ): Promise + cancel(): Promise getRun(): Promise } @@ -181,6 +197,12 @@ export function durableObjectLauncher(namespace: { getByName(name: string): obje return createRemoteRunLauncher({ launch: (runId, args) => runs.getByName(runId).launch(args), status: (runId) => runs.getByName(runId).status(), + accept: (runId, events, options) => + runs + .getByName(runId) + .accept(events, options) + .then(() => undefined), + cancel: (runId) => runs.getByName(runId).cancel(), }) } diff --git a/packages/core/src/effects.ts b/packages/core/src/effects.ts index 57035c6..233f52a 100644 --- a/packages/core/src/effects.ts +++ b/packages/core/src/effects.ts @@ -60,6 +60,23 @@ export type CancelEffect = { tag?: string } +export type LaunchEffect = { + type: 'runtime.launch' + runId: string + kind: string + definitionName: string + definitionVersion: string + input: JsonValue + close: 'terminate' | 'wait' + tag?: string +} + +export type CancelRunEffect = { + type: 'runtime.cancelRun' + runId: string + tag?: string +} + export interface RetryPolicy { readonly maxAttempts: number readonly backoffMs?: number @@ -86,6 +103,8 @@ export type PrimitiveEffect = | CompleteEffect | FailEffect | CancelEffect + | LaunchEffect + | CancelRunEffect export type RuntimeEffect = PrimitiveEffect | InvokeEffect @@ -132,6 +151,25 @@ export const RuntimeEffectSchema = Schema.Union([ threadId: Schema.String, tag: Schema.optional(Schema.String), }), + Schema.Struct({ + type: Schema.Literal('runtime.launch'), + runId: Schema.String, + kind: Schema.String, + definitionName: Schema.String, + definitionVersion: Schema.String.pipe( + Schema.withDecodingDefaultKey(Effect.succeed(DEFAULT_DEFINITION_VERSION)), + ), + input: Schema.Json, + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('terminate' as const)), + ), + tag: Schema.optional(Schema.String), + }), + Schema.Struct({ + type: Schema.Literal('runtime.cancelRun'), + runId: Schema.String, + tag: Schema.optional(Schema.String), + }), Schema.Struct({ type: Schema.String, input: Schema.Json, @@ -148,6 +186,8 @@ export function isPrimitiveEffect(effect: RuntimeEffect): effect is PrimitiveEff case 'runtime.complete': case 'runtime.fail': case 'runtime.cancel': + case 'runtime.launch': + case 'runtime.cancelRun': return true default: return false @@ -413,6 +453,38 @@ export function cancel(threadId: string, tag?: string): CancelEffect { : { type: 'runtime.cancel', threadId, tag } } +export function launch(args: { + runId: string + kind: string + definitionName: string + definitionVersion?: string + input: JsonValue + close?: 'terminate' | 'wait' + tag?: string +}): LaunchEffect { + const effect: LaunchEffect = { + type: 'runtime.launch', + runId: args.runId, + kind: args.kind, + definitionName: args.definitionName, + definitionVersion: args.definitionVersion ?? DEFAULT_DEFINITION_VERSION, + input: args.input, + close: args.close ?? 'terminate', + } + + if (args.tag !== undefined) { + effect.tag = args.tag + } + + return effect +} + +export function cancelRun(runId: string, tag?: string): CancelRunEffect { + return tag === undefined + ? { type: 'runtime.cancelRun', runId } + : { type: 'runtime.cancelRun', runId, tag } +} + export interface InvokeOptions { /** Wait until this epoch millisecond before the host runs the effect. */ readonly at?: number diff --git a/packages/core/src/fold.ts b/packages/core/src/fold.ts index 535df71..6610a95 100644 --- a/packages/core/src/fold.ts +++ b/packages/core/src/fold.ts @@ -50,6 +50,9 @@ function cloneState(state: RunState): RunState { processedIdempotencyKeys: state.processedIdempotencyKeys ? [...state.processedIdempotencyKeys] : [], + launchedRuns: state.launchedRuns ? [...state.launchedRuns] : [], + closeWaiting: state.closeWaiting ?? false, + childCloseSent: state.childCloseSent ?? false, } } @@ -464,7 +467,45 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi return next } - case 'runtime.run.launched': + case 'runtime.run.launched': { + const childRunId = readString(payload, 'runId') + + if (!childRunId || state.launchedRuns.some((run) => run.runId === childRunId)) { + return state + } + + const close = readString(payload, 'close') + + if (close !== 'terminate' && close !== 'wait') { + return state + } + + return { + ...state, + launchedRuns: [...state.launchedRuns, { runId: childRunId, close }], + } + } + + case 'runtime.child.settled': { + const childRunId = readString(payload, 'runId') + + if (!childRunId) { + return state + } + + return { + ...state, + launchedRuns: state.launchedRuns.filter((run) => run.runId !== childRunId), + } + } + + case 'runtime.child.close.requested': + return { + ...state, + childCloseSent: true, + closeWaiting: readString(payload, 'policy') === 'wait' ? true : state.closeWaiting, + } + case 'runtime.snapshot.taken': case 'runtime.thread.cancel.requested': case 'runtime.timer.set': @@ -500,6 +541,18 @@ export function stableEffectId(threadId: string, effect: RuntimeEffect): string return createEffectId(threadId, `cancel_${effect.threadId}`) } + if (effect.type === 'runtime.launch' && 'runId' in effect && Predicate.isString(effect.runId)) { + return createEffectId(threadId, `launch_${effect.runId}`) + } + + if ( + effect.type === 'runtime.cancelRun' && + 'runId' in effect && + Predicate.isString(effect.runId) + ) { + return createEffectId(threadId, `cancel_${effect.runId}`) + } + return undefined } diff --git a/packages/core/src/ids.ts b/packages/core/src/ids.ts index be3709e..c404df2 100644 --- a/packages/core/src/ids.ts +++ b/packages/core/src/ids.ts @@ -34,6 +34,11 @@ function fnv1a32(value: string, seed: number): string { return (hash >>> 0).toString(16).padStart(8, '0') } +/** A stable `run_` id derived from `parts`, so a retry launches the same child run. */ +export function deterministicRunId(...parts: readonly (string | number)[]): string { + return `run_${deterministicThreadId(...parts).replaceAll('-', '')}` +} + /** * A UUID-shaped id derived only from `parts`, so a reducer that replays the same events names the same * child thread every time. diff --git a/packages/core/src/launcher.ts b/packages/core/src/launcher.ts index 70d7370..2b65cc0 100644 --- a/packages/core/src/launcher.ts +++ b/packages/core/src/launcher.ts @@ -1,7 +1,12 @@ import { Context, Data, Effect, Option, Schema } from 'effect' +import type { EventInput } from './envelope' import type { JsonValue, RunStatus } from './types' +export const ChildCloseSchema = Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]) + +export type ChildClose = Schema.Schema.Type + /** The effect that launched a run, recorded on the launched run's `runtime.run.started`. */ export const RunLaunchSourceSchema = Schema.Struct({ runId: Schema.String, @@ -36,6 +41,11 @@ export interface LaunchRunArgs { * correlation id, or to the launching run's id when that run has none. */ readonly correlationId?: string + /** + * What the launching run does with this child when the launching run closes. + * `terminate` cancels the child. `wait` lets the child finish cancelling first. + */ + readonly close?: ChildClose } /** Enough of a run to decide whether a previous fire is still in progress. */ @@ -50,6 +60,16 @@ export interface RunStatusSnapshot { export interface RunLauncher { readonly launch: (args: LaunchRunArgs) => Effect.Effect readonly status: (runId: string) => Effect.Effect + /** + * Append events to a run and wake it. When that run is already waking, the wake coalesces + * instead of waiting, so a child can notify its parent from inside the parent's wake. + */ + readonly signal: ( + runId: string, + events: readonly EventInput[], + options?: { readonly idempotencyKey?: string }, + ) => Effect.Effect + readonly cancel: (runId: string) => Effect.Effect } export class RunLauncherTag extends Context.Service()( @@ -85,12 +105,20 @@ export class RunLauncherUnavailableError extends Data.TaggedError('RunLauncherUn export const unavailableRunLauncher: RunLauncher = { launch: () => Effect.fail(new RunLauncherUnavailableError()), status: () => Effect.fail(new RunLauncherUnavailableError()), + signal: () => Effect.fail(new RunLauncherUnavailableError()), + cancel: () => Effect.fail(new RunLauncherUnavailableError()), } /** Adapts a per-`runId` RPC or cell lookup into a `RunLauncher`. */ export function createRemoteRunLauncher(target: { readonly launch: (runId: string, args: LaunchRunArgs) => Promise readonly status: (runId: string) => Promise + readonly accept: ( + runId: string, + events: readonly EventInput[], + options?: { readonly idempotencyKey?: string; readonly coalesce?: boolean }, + ) => Promise + readonly cancel: (runId: string) => Promise }): RunLauncher { return { launch: (args) => { @@ -110,5 +138,15 @@ export function createRemoteRunLauncher(target: { try: () => target.status(runId), catch: runLauncherError, }), + signal: (runId, events, options) => + Effect.tryPromise({ + try: () => target.accept(runId, events, { ...options, coalesce: true }), + catch: runLauncherError, + }), + cancel: (runId) => + Effect.tryPromise({ + try: () => target.cancel(runId), + catch: runLauncherError, + }), } } diff --git a/packages/core/src/projection.ts b/packages/core/src/projection.ts index e0fbf05..eb97b19 100644 --- a/packages/core/src/projection.ts +++ b/packages/core/src/projection.ts @@ -302,6 +302,9 @@ export function toThreadTree(state: TreeBuildState): ThreadTree { effectExecutions: {}, completedEffectIds: [], processedIdempotencyKeys: [], + launchedRuns: [], + closeWaiting: false, + childCloseSent: false, }) } diff --git a/packages/core/src/protocol.ts b/packages/core/src/protocol.ts index 527ceaf..1169e5a 100644 --- a/packages/core/src/protocol.ts +++ b/packages/core/src/protocol.ts @@ -34,6 +34,22 @@ export const protocolCatalog = defineEventCatalog('runtime', { definitionVersion: DefinitionVersionSchema, /** In the payload, not the envelope: an envelope `effectId` would complete the effect. */ effectId: Schema.String, + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('terminate' as const)), + ), + }), + 'child.settled': Schema.Struct({ + runId: Schema.String, + status: Schema.Union([ + Schema.Literal('completed'), + Schema.Literal('failed'), + Schema.Literal('cancelled'), + ]), + output: Schema.NullOr(JsonValueSchema), + error: Schema.NullOr(Schema.String), + }), + 'child.close.requested': Schema.Struct({ + policy: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]), }), 'run.forked': Schema.Struct({ sourceRunId: Schema.String, @@ -167,6 +183,8 @@ export const PROTOCOL_TYPES = [ 'runtime.run.completed', 'runtime.run.cancelled', 'runtime.run.launched', + 'runtime.child.settled', + 'runtime.child.close.requested', 'runtime.run.forked', 'runtime.fork.elided', 'runtime.thread.started', diff --git a/packages/core/src/state.ts b/packages/core/src/state.ts index 58d370b..176638e 100644 --- a/packages/core/src/state.ts +++ b/packages/core/src/state.ts @@ -7,7 +7,7 @@ import { type WaitCondition, } from './effects' import { JsonValueSchema } from './envelope' -import { RunLaunchSourceSchema, type RunLaunchSource } from './launcher' +import { RunLaunchSourceSchema, type ChildClose, type RunLaunchSource } from './launcher' import { DEFAULT_DEFINITION_VERSION } from './module' import { isTerminalStatus } from './thread' import type { JsonValue, ThreadStatus, RunStatus } from './types' @@ -79,6 +79,11 @@ export interface RunForkSource { seq: number } +export interface LaunchedRun { + runId: string + close: ChildClose +} + export interface RunState { runId: string status: RunStatus @@ -91,6 +96,12 @@ export interface RunState { effectExecutions: { [effectId: string]: EffectExecutionRecord } completedEffectIds: readonly string[] processedIdempotencyKeys: readonly string[] + /** Child runs this run launched and has not yet seen settle. */ + launchedRuns: readonly LaunchedRun[] + /** The run is cancelling and will not finish until `wait` children settle. */ + closeWaiting: boolean + /** Child runs were already asked to stop because this run closed. */ + childCloseSent: boolean } export const RunStateSchema = Schema.Struct({ @@ -198,6 +209,14 @@ export const RunStateSchema = Schema.Struct({ processedIdempotencyKeys: Schema.Array(Schema.String).pipe( Schema.withDecodingDefaultKey(Effect.succeed([])), ), + launchedRuns: Schema.Array( + Schema.Struct({ + runId: Schema.String, + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]), + }), + ).pipe(Schema.withDecodingDefaultKey(Effect.succeed([]))), + closeWaiting: Schema.Boolean.pipe(Schema.withDecodingDefaultKey(Effect.succeed(false))), + childCloseSent: Schema.Boolean.pipe(Schema.withDecodingDefaultKey(Effect.succeed(false))), }) export function emptyRunState(runId: string): RunState { @@ -213,6 +232,9 @@ export function emptyRunState(runId: string): RunState { effectExecutions: {}, completedEffectIds: [], processedIdempotencyKeys: [], + launchedRuns: [], + closeWaiting: false, + childCloseSent: false, } } diff --git a/packages/debugger/src/events/protocol-summarize.ts b/packages/debugger/src/events/protocol-summarize.ts index 5ea918e..6d7dc01 100644 --- a/packages/debugger/src/events/protocol-summarize.ts +++ b/packages/debugger/src/events/protocol-summarize.ts @@ -34,6 +34,13 @@ export function summarizeProtocolEvent(event: DebuggerEvent): EventSummary | und title: 'run launched', detail: `${jsonText(payload.kind) ?? 'run'}:${jsonText(payload.definitionName) ?? 'unknown'} · ${jsonText(payload.runId) ?? 'unknown'}`, } + case 'runtime.child.settled': + return { + title: 'child settled', + detail: `${jsonText(payload.status) ?? 'done'} · ${jsonText(payload.runId) ?? ''}`, + } + case 'runtime.child.close.requested': + return { title: 'child close requested', detail: jsonText(payload.policy) } case 'runtime.run.forked': return { title: 'run forked', diff --git a/packages/runtime/src/cancellation.ts b/packages/runtime/src/cancellation.ts index e16d237..60b9669 100644 --- a/packages/runtime/src/cancellation.ts +++ b/packages/runtime/src/cancellation.ts @@ -9,6 +9,7 @@ import { type EffectExecutionRecord, type EventEnvelope, type OutstandingEffect, + type RunLauncher, type RunState, } from '@looms/core' @@ -29,6 +30,7 @@ export interface CancellationOptions { readonly observer?: RuntimeObserver readonly wake: LoomsRuntime['wake'] readonly withRunIngress: RunIngress + readonly launcher: () => RunLauncher } interface WorkerCancellation { @@ -146,6 +148,37 @@ export function createCancellation(options: CancellationOptions): LoomsRuntime[' const now = yield* Clock.currentTimeMillis return yield* repository.commit(store, runId, (current) => { + const rootCancel = threadId === undefined || threadId === current.state.rootThreadId + + const deferClose = + rootCancel && + !current.state.childCloseSent && + current.state.launchedRuns.some((child) => child.close === 'wait') + + if (deferClose) { + const requested = createEvent(runId, { + type: 'runtime.child.close.requested', + payload: { policy: 'wait' }, + threadId: current.state.rootThreadId, + origin: { type: 'system' }, + idempotencyKey: `child-close:${current.state.rootThreadId ?? runId}`, + }) + + return Effect.succeed({ + events: [requested], + value: { + plan: { + events: [], + threadIds: new Set(), + workerCancellations: [], + } satisfies CancellationPlan, + now, + childRunIds: current.state.launchedRuns.map((child) => child.runId), + }, + idempotencyKey: `signal:child-close:${current.state.rootThreadId ?? runId}`, + }) + } + const plan = planCancellation({ runId, seq: current.seq, @@ -156,16 +189,25 @@ export function createCancellation(options: CancellationOptions): LoomsRuntime[' now, }) + const childRunIds: readonly string[] = [] + return Effect.succeed({ events: plan.events, - value: { plan, now }, + value: { plan, now, childRunIds }, idempotencyKey: `signal:cancel:${threadId ?? current.state.rootThreadId ?? ''}`, }) }) }), ) - const { plan, now } = committed.value + const { plan, now, childRunIds } = committed.value + + if (childRunIds.length > 0) { + yield* Effect.forEach(childRunIds, (childRunId) => options.launcher().cancel(childRunId), { + concurrency: 1, + discard: true, + }) + } if (committed.append) { executionContexts.abort(runId, plan.threadIds, 'Effect cancelled') diff --git a/packages/runtime/src/child-run.test.ts b/packages/runtime/src/child-run.test.ts new file mode 100644 index 0000000..a1bdeda --- /dev/null +++ b/packages/runtime/src/child-run.test.ts @@ -0,0 +1,201 @@ +import { describe, expect, test } from 'bun:test' + +import { Predicate } from 'effect' + +import type { RunState } from '@looms/core' +import { defineWorkflow, workflow } from '@looms/workflow' + +import { createLooms } from './looms' + +const child = defineWorkflow({ + name: 'child-value', + nodes: [{ id: 'out', run: () => 7 }], + output: ({ results }) => results.out, +}) + +const parent = defineWorkflow({ + name: 'parent-value', + childExecution: 'run', + nodes: [{ id: 'spawn', run: (ctx) => ctx.spawn(child, {}) }], + output: ({ results }) => results.spawn, +}) + +const mappedChild = defineWorkflow({ + name: 'mapped-child', + nodes: [{ id: 'out', run: (ctx) => ctx.input }], + output: ({ input }) => input, +}) + +const mappedParent = defineWorkflow({ + name: 'mapped-parent', + childExecution: 'run', + nodes: [{ id: 'items', run: (ctx) => ctx.map(mappedChild, [1, 2]) }], + output: ({ results }) => results.items, +}) + +const sleepyChild = defineWorkflow({ + name: 'sleepy-child', + nodes: [{ id: 'nap', run: (ctx) => ctx.sleep(60_000) }], +}) + +function rootOutput(state: RunState) { + return state.rootThreadId ? state.threads[state.rootThreadId]?.output : undefined +} + +async function waitForStatus( + looms: { getRun(runId: string): Promise }, + runId: string, + status: RunState['status'], +) { + const deadline = Date.now() + 2_000 + let state = await looms.getRun(runId) + + while (state.status !== status && Date.now() < deadline) { + await Bun.sleep(10) + state = await looms.getRun(runId) + } + + return state +} + +describe('child runs', () => { + test('returns a child run result to the spawning node', async () => { + const looms = createLooms({ modules: [workflow({ definitions: [child, parent] })] }) + const started = await looms.start(parent, {}) + const state = await waitForStatus(looms, started.runId, 'completed') + const events = await looms.getEvents(started.runId) + const launched = events.find((event) => event.type === 'runtime.run.launched') + + const childRunId = + launched && Predicate.isObject(launched.payload) && Predicate.isString(launched.payload.runId) + ? launched.payload.runId + : undefined + + expect(state.status).toBe('completed') + expect(rootOutput(state)).toBe(7) + expect(childRunId).toEqual(expect.any(String)) + + const childState = await looms.getRun(childRunId ?? '') + expect(childState.status).toBe('completed') + expect(childState.startIdentity?.launchedBy?.runId).toBe(started.runId) + expect(events.some((event) => event.type === 'runtime.child.settled')).toBe(true) + + await looms.stop() + }) + + test('collects mapped child runs in item order', async () => { + const looms = createLooms({ + modules: [workflow({ definitions: [mappedChild, mappedParent] })], + }) + + const started = await looms.start(mappedParent, {}) + const state = await waitForStatus(looms, started.runId, 'completed') + const events = await looms.getEvents(started.runId) + const launched = events.filter((event) => event.type === 'runtime.run.launched') + + expect(state.status).toBe('completed') + expect(rootOutput(state)).toEqual([1, 2]) + expect(launched).toHaveLength(2) + + await looms.stop() + }) + + test('keeps the default of running a child in the parent run', async () => { + const threaded = defineWorkflow({ + name: 'threaded-parent', + nodes: [{ id: 'spawn', run: (ctx) => ctx.spawn(child, {}) }], + }) + + const looms = createLooms({ modules: [workflow({ definitions: [child, threaded] })] }) + const started = await looms.start(threaded, {}) + const events = await looms.getEvents(started.runId) + + expect(started.state.status).toBe('completed') + expect(events.some((event) => event.type === 'runtime.run.launched')).toBe(false) + expect(rootOutput(started.state)).toEqual({ spawn: 7 }) + + await looms.stop() + }) + + test('terminates child runs when the parent is cancelled', async () => { + const terminating = defineWorkflow({ + name: 'terminating-parent', + childExecution: 'run', + childClose: 'terminate', + nodes: [{ id: 'spawn', run: (ctx) => ctx.spawn(sleepyChild, {}) }], + }) + + const looms = createLooms({ + modules: [workflow({ definitions: [sleepyChild, terminating] })], + }) + + const started = await looms.start(terminating, {}) + const eventsBefore = await looms.getEvents(started.runId) + const launched = eventsBefore.find((event) => event.type === 'runtime.run.launched') + + const childRunId = + launched && Predicate.isObject(launched.payload) && Predicate.isString(launched.payload.runId) + ? launched.payload.runId + : '' + + expect(started.state.status).toBe('running') + + const cancelled = await looms.cancel(started.runId) + const events = await looms.getEvents(started.runId) + const childState = await looms.getRun(childRunId) + + expect(cancelled.status).toBe('cancelled') + expect(childState.status).toBe('cancelled') + + expect( + events.some( + (event) => + event.type === 'runtime.child.close.requested' && + Predicate.isObject(event.payload) && + event.payload.policy === 'wait', + ), + ).toBe(false) + + await looms.stop() + }) + + test('waits for child runs to finish cancelling', async () => { + const waiting = defineWorkflow({ + name: 'waiting-parent', + childExecution: 'run', + childClose: 'wait', + nodes: [{ id: 'spawn', run: (ctx) => ctx.spawn(sleepyChild, {}) }], + }) + + const looms = createLooms({ modules: [workflow({ definitions: [sleepyChild, waiting] })] }) + const started = await looms.start(waiting, {}) + + const launched = (await looms.getEvents(started.runId)).find( + (event) => event.type === 'runtime.run.launched', + ) + + const childRunId = + launched && Predicate.isObject(launched.payload) && Predicate.isString(launched.payload.runId) + ? launched.payload.runId + : '' + + const cancelled = await looms.cancel(started.runId) + const events = await looms.getEvents(started.runId) + const types = events.map((event) => event.type) + const childState = await looms.getRun(childRunId) + + expect(cancelled.status).toBe('cancelled') + expect(childState.status).toBe('cancelled') + expect(types.indexOf('runtime.child.close.requested')).toBeGreaterThanOrEqual(0) + + expect(types.indexOf('runtime.child.close.requested')).toBeLessThan( + types.indexOf('runtime.run.cancelled'), + ) + + expect(events.find((event) => event.type === 'runtime.child.close.requested')).toMatchObject({ + payload: { policy: 'wait' }, + }) + + await looms.stop() + }) +}) diff --git a/packages/runtime/src/child-run.ts b/packages/runtime/src/child-run.ts new file mode 100644 index 0000000..806b327 --- /dev/null +++ b/packages/runtime/src/child-run.ts @@ -0,0 +1,86 @@ +import { Effect } from 'effect' + +import { + descendantThreadIds, + isTerminalStatus, + type EventInput, + type RunLauncher, + type RunState, +} from '@looms/core' + +/** Tell the launching run that this child has reached a terminal status. */ +export function notifyLaunchingParent( + launcher: RunLauncher, + runId: string, + state: RunState, +): Effect.Effect { + const launchedBy = state.startIdentity?.launchedBy + const status = state.status + + if (!launchedBy || status === 'running') { + return Effect.void + } + + const root = state.rootThreadId ? state.threads[state.rootThreadId] : undefined + const error = status === 'completed' ? null : (root?.error ?? status) + + return launcher.signal( + launchedBy.runId, + [ + { + type: 'runtime.child.settled', + payload: { + runId, + status, + output: root?.output ?? null, + error, + }, + threadId: launchedBy.threadId, + idempotencyKey: `child-settled:${runId}`, + origin: { type: 'run', runId }, + }, + ], + { idempotencyKey: `child-settled:${runId}` }, + ) +} + +/** + * When a run asked to wait for child cancellation and those children are gone, cancel the + * run's own threads. The caller appends these events from the parent wake, which must not + * call `runtime.cancel` on itself. + */ +export function childCloseCancellations(state: RunState): EventInput[] { + if (!state.closeWaiting || state.launchedRuns.some((run) => run.close === 'wait')) { + return [] + } + + const rootId = state.rootThreadId + + if (!rootId) { + return [] + } + + const root = state.threads[rootId] + + if (!root || isTerminalStatus(root.status)) { + return [] + } + + const ids = [...descendantThreadIds(state, rootId).toReversed(), rootId] + + return ids.flatMap((threadId) => { + const record = state.threads[threadId] + + if (!record || isTerminalStatus(record.status)) { + return [] + } + + return [ + { + type: 'runtime.thread.cancelled', + payload: { threadId, reason: 'cancelled' }, + threadId, + }, + ] + }) +} diff --git a/packages/runtime/src/dispatch-effect.ts b/packages/runtime/src/dispatch-effect.ts index 93a533d..abb0d1c 100644 --- a/packages/runtime/src/dispatch-effect.ts +++ b/packages/runtime/src/dispatch-effect.ts @@ -99,6 +99,22 @@ export function dispatchEffect( ]) } + case 'runtime.launch': + return launcher + .launch({ + runId: eff.runId, + kind: eff.kind, + definitionName: eff.definitionName, + definitionVersion: eff.definitionVersion, + input: eff.input, + idempotencyKey: eff.runId, + close: eff.close, + }) + .pipe(Effect.map(() => [])) + + case 'runtime.cancelRun': + return launcher.cancel(eff.runId).pipe(Effect.map(() => [])) + default: { const exhaustiveCheck: never = eff return exhaustiveCheck diff --git a/packages/runtime/src/local-effect-executor.ts b/packages/runtime/src/local-effect-executor.ts index e611713..18ca38c 100644 --- a/packages/runtime/src/local-effect-executor.ts +++ b/packages/runtime/src/local-effect-executor.ts @@ -11,6 +11,7 @@ import { type EventInput, type OutstandingEffect, type RegisteredDefinition, + type LaunchRunArgs, type RunLauncher, type RunLaunchSource, } from '@looms/core' @@ -48,6 +49,24 @@ export interface ExecuteLocalEffectsInput { readonly now: number } +function runLaunchedPayload(args: LaunchRunArgs, targetRunId: string, effectId: string) { + const definitionVersion = args.definitionVersion ?? DEFAULT_DEFINITION_VERSION + + const base = { + runId: targetRunId, + kind: args.kind, + definitionName: args.definitionName, + definitionVersion, + effectId, + } + + if (args.close === undefined) { + return base + } + + return { ...base, close: args.close } +} + /** * Stamps every launch with the effect that made it, replacing anything the handler passed, and * records a `runtime.run.launched` event for each launch the target accepts. @@ -77,13 +96,7 @@ function launcherFor( launched.push( createEvent(runId, { type: 'runtime.run.launched', - payload: { - runId: targetRunId, - kind: args.kind, - definitionName: args.definitionName, - definitionVersion: args.definitionVersion ?? DEFAULT_DEFINITION_VERSION, - effectId: item.effectId, - }, + payload: runLaunchedPayload(args, targetRunId, item.effectId), threadId: item.threadId, causationId: item.causingEventId, correlationId, @@ -95,6 +108,8 @@ function launcherFor( ) }, status: launcher.status, + signal: (targetRunId, events, options) => launcher.signal(targetRunId, events, options), + cancel: (targetRunId) => launcher.cancel(targetRunId), } } @@ -264,7 +279,23 @@ export function executeLocalEffects( const { outcomes } = settled - if (outcomes.length === 0) { + const sideEffect = + item.effect.type === 'runtime.launch' || item.effect.type === 'runtime.cancelRun' + + if (outcomes.length === 0 && sideEffect) { + const attempt = input.attempts.get(item.effectId) ?? 1 + + produced.push( + createEvent(input.runId, { + type: 'runtime.effect.completed', + payload: { effectId: item.effectId, attempt }, + threadId: item.threadId, + effectId: item.effectId, + causationId: item.causingEventId, + origin: { type: 'system' }, + }), + ) + } else if (outcomes.length === 0) { produced.push(createEffectFailedEvent(input.runId, item, 'empty-outcome')) } diff --git a/packages/runtime/src/planners.test.ts b/packages/runtime/src/planners.test.ts index 53eefd5..aeb1089 100644 --- a/packages/runtime/src/planners.test.ts +++ b/packages/runtime/src/planners.test.ts @@ -30,6 +30,9 @@ function runningState(overrides: Partial = {}): RunState { effectExecutions: {}, completedEffectIds: [], processedIdempotencyKeys: [], + launchedRuns: [], + closeWaiting: false, + childCloseSent: false, ...overrides, } } diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index ac0b33b..29d2cbf 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -190,13 +190,24 @@ export function createRuntime event.type === 'runtime.child.settled', + ) + + if (terminal && !settlementOnly) { return yield* new RunTerminalError(runId, current.state.status) } const folded = foldRun(batch, registry, { runId, initial: current.state }) - const satisfiedEvents = waitSatisfiedEvents(folded, batch).map((input) => + const forWaits = + terminal || current.state.closeWaiting + ? batch.filter((event) => event.type !== 'runtime.child.settled') + : batch + + const satisfiedEvents = waitSatisfiedEvents(folded, forWaits).map((input) => createEvent(runId, input), ) @@ -360,8 +371,21 @@ export function createRuntime launcher, }) + const appendAndWake: LoomsRuntime['accept'] = (runId, events, signalOpts) => + withRunIngress(runId, signalAdmitted(runId, events, signalOpts)).pipe( + Effect.flatMap((state) => { + if (signalOpts?.coalesce && executionContexts.get(runId).waking) { + executionContexts.get(runId).wakeAgain = true + return Effect.succeed(state) + } + + return runtime.wake(runId) + }), + ) + runtime = { modules: options.modules, registry, @@ -555,12 +579,11 @@ export function createRuntime admitExternalEvents(events).pipe( - Effect.flatMap((admitted) => - withRunIngress(runId, signalAdmitted(runId, admitted, signalOpts)), - ), - Effect.andThen(runtime.wake(runId)), + Effect.flatMap((admitted) => appendAndWake(runId, admitted, signalOpts)), ), + accept: (runId, events, signalOpts) => appendAndWake(runId, events, signalOpts), + cancel: cancellation, fork: (sourceRunId, forkOptions) => { @@ -648,6 +671,12 @@ export function createRuntime + runtime + .accept(runId, events, { ...signalOptions, coalesce: true }) + .pipe(Effect.provideService(EventStoreTag, store), Effect.asVoid), + cancel: (runId) => + runtime.cancel(runId).pipe(Effect.provideService(EventStoreTag, store), Effect.asVoid), } : unavailableRunLauncher) diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 850756b..2217ebd 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -157,7 +157,16 @@ export interface LoomsRuntime< signal( runId: string, events: ReadonlyArray, - options?: { idempotencyKey?: string }, + options?: { idempotencyKey?: string; coalesce?: boolean }, + ): Effect.Effect + /** + * Append events the runtime itself produced for another run. Unlike `signal`, this does not + * reject reserved runtime event types or rewrite their origin. + */ + accept( + runId: string, + events: ReadonlyArray, + options?: { idempotencyKey?: string; coalesce?: boolean }, ): Effect.Effect wake(runId: string): Effect.Effect getRun( diff --git a/packages/runtime/src/wake-engine.ts b/packages/runtime/src/wake-engine.ts index 341c9d6..c768fbe 100644 --- a/packages/runtime/src/wake-engine.ts +++ b/packages/runtime/src/wake-engine.ts @@ -8,11 +8,14 @@ import { isRunTerminal, type ComposedRegistry, type EventEnvelope, + type EventStore, type RegisteredDefinition, + type RunCursor, type RunLauncher, type RunState, } from '@looms/core' +import { childCloseCancellations, notifyLaunchingParent } from './child-run' import { nextRuntimeDeadline, planDeadlineTransitions, @@ -76,6 +79,40 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake const observe = (event: Parameters[0]) => notifyObserver(options.observer, event) + const closeLaunchedRuns = (store: EventStore, runId: string, cursor: RunCursor) => + Effect.gen(function* () { + let next = cursor + const state = next.state + + if (isRunTerminal(state) && !state.childCloseSent && state.launchedRuns.length > 0) { + const requested = createEvent(runId, { + type: 'runtime.child.close.requested', + payload: { policy: 'terminate' }, + threadId: state.rootThreadId, + origin: { type: 'system' }, + idempotencyKey: `child-close:${runId}`, + }) + + const appended = yield* appendObserved(store, runId, stripSeq([requested])) + next = yield* applyAppend(store, runId, next, stripSeq([requested]), appended) + + yield* Effect.forEach( + state.launchedRuns, + (child) => options.launcher().cancel(child.runId), + { + concurrency: 1, + discard: true, + }, + ) + } + + if (isRunTerminal(next.state)) { + yield* notifyLaunchingParent(options.launcher(), runId, next.state) + } + + return next + }) + return (runId) => Effect.gen(function* () { observe({ type: 'run.wake', runId, at: currentTimeMillis() }) @@ -146,6 +183,13 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake do { let cursor = yield* loadCursor(store, runId) + const closeEvents = childCloseCancellations(cursor.state) + + if (closeEvents.length > 0) { + const created = closeEvents.map((input) => createEvent(runId, input)) + const appended = yield* appendObserved(store, runId, stripSeq(created)) + cursor = yield* applyAppend(store, runId, cursor, stripSeq(created), appended) + } if (isRunTerminal(cursor.state)) { const now = yield* Clock.currentTimeMillis @@ -161,6 +205,8 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake cursor = yield* applyAppend(store, runId, cursor, stripSeq(events), appended) } + cursor = yield* closeLaunchedRuns(store, runId, cursor) + const nextDeadline = nextRuntimeDeadline(cursor.state, now) if (nextDeadline !== undefined) { @@ -461,6 +507,10 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake cursor = yield* applyAppend(store, runId, cursor, stripSeq([completed]), appendResult) } + if (isRunTerminal(cursor.state)) { + cursor = yield* closeLaunchedRuns(store, runId, cursor) + } + // Snapshot on park: the next wake (on any instance) starts from here. cursor = yield* persistSnapshot(store, runId, cursor) const nextDeadline = nextRuntimeDeadline(cursor.state, plannedAt) diff --git a/packages/workflow/src/definitions.ts b/packages/workflow/src/definitions.ts index b0dc151..ef37418 100644 --- a/packages/workflow/src/definitions.ts +++ b/packages/workflow/src/definitions.ts @@ -127,9 +127,24 @@ export interface WorkflowDefinition< readonly inputLimitBytes?: number readonly outputLimitBytes?: number readonly checkpointLimitBytes?: number + /** + * `thread` keeps spawned, mapped, and looped children in this run. + * `run` launches each child as its own run. + */ + readonly childExecution?: 'thread' | 'run' + /** + * When this run is cancelled, `terminate` cancels child runs immediately and `wait` cancels + * this run only after those children have finished cancelling. Ignored when children are threads. + */ + readonly childClose?: 'terminate' | 'wait' output?(ctx: { input: TInput; results: { [nodeId: string]: JsonValue | null } }): TOutput } +export interface WorkflowChildMode { + readonly execution: 'thread' | 'run' + readonly close: 'terminate' | 'wait' +} + export type AnyWorkflowDefinition = WorkflowDefinition function compatibleNodeStatus(status: string | undefined): NodeStatus { diff --git a/packages/workflow/src/effects.ts b/packages/workflow/src/effects.ts index 3b5074f..9bc0818 100644 --- a/packages/workflow/src/effects.ts +++ b/packages/workflow/src/effects.ts @@ -13,6 +13,7 @@ import { nodeResultBuilders, type NodeResult, type NodeState, + type WorkflowChildMode, type WorkflowDefinition, type WorkflowIteration, } from './definitions' @@ -352,7 +353,7 @@ function nodeResultEvents( ? nodeChildWorkflowId(threadId, nodeId, attempt) : createThreadId()) - return spawnRequested(run, childThreadId, result, result.input) + return spawnRequested(definition, run, childThreadId, result, result.input) } case 'sleep': { @@ -401,6 +402,7 @@ function nodeResultEvents( items: result.items, concurrency: result.concurrency, limitBytes, + ...childPolicy(definition), }, threadId, }, @@ -415,7 +417,7 @@ function nodeResultEvents( } const childThreadId = iterationChildWorkflowId(threadId, nodeId, 'loop', attempt, index) - return spawnRequested(run, childThreadId, result, result.input, index) + return spawnRequested(definition, run, childThreadId, result, result.input, index) } default: { @@ -426,7 +428,15 @@ function nodeResultEvents( }) } +function childPolicy(definition: WorkflowDefinition): WorkflowChildMode { + return { + execution: definition.childExecution ?? 'thread', + close: definition.childClose ?? 'terminate', + } +} + function spawnRequested( + definition: WorkflowDefinition, run: NodeRun, childThreadId: string, child: { readonly kind: string; readonly name: string; readonly version: string }, @@ -445,6 +455,7 @@ function spawnRequested( definitionVersion: child.version, input, loopIndex, + ...childPolicy(definition), }, threadId: run.threadId, }, diff --git a/packages/workflow/src/events.ts b/packages/workflow/src/events.ts index 0695e46..f06d2ed 100644 --- a/packages/workflow/src/events.ts +++ b/packages/workflow/src/events.ts @@ -42,6 +42,12 @@ export const WorkflowSpawnRequestedPayloadSchema = Schema.Struct({ input: Schema.Json, /** Set for `loop` iterations: the node re-runs after this child instead of finishing. */ loopIndex: Schema.NullOr(Schema.Finite).pipe(Schema.withDecodingDefaultKey(Effect.succeed(null))), + execution: Schema.Union([Schema.Literal('thread'), Schema.Literal('run')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('thread' as const)), + ), + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('terminate' as const)), + ), }) export const WorkflowMapRequestedPayloadSchema = Schema.Struct({ @@ -53,6 +59,12 @@ export const WorkflowMapRequestedPayloadSchema = Schema.Struct({ items: Schema.Array(Schema.Json), concurrency: Schema.Finite, limitBytes: Schema.Finite, + execution: Schema.Union([Schema.Literal('thread'), Schema.Literal('run')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('thread' as const)), + ), + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('terminate' as const)), + ), }) export const WorkflowSleepRequestedPayloadSchema = Schema.Struct({ diff --git a/packages/workflow/src/threads.ts b/packages/workflow/src/threads.ts index c31e75f..52040e2 100644 --- a/packages/workflow/src/threads.ts +++ b/packages/workflow/src/threads.ts @@ -2,14 +2,17 @@ import { Effect, Predicate, Schema } from 'effect' import { cancel, + cancelRun, createWaitId, DEFAULT_DEFINITION_VERSION, + deterministicRunId, EventInputSchema, emit, fail, invoke, isJsonObject, isWithdrawnError, + launch, RuntimeEffectSchema, spawn, stableEffectId, @@ -21,7 +24,7 @@ import { waitOutcome, } from '@looms/core' -import { NodeStateSchema, type NodeState } from './definitions' +import { NodeStateSchema, type NodeState, type WorkflowChildMode } from './definitions' import { runNodeEffect, scheduleEffect } from './effects' import { iterationChildWorkflowId, nodeRunKey, WorkflowValueTooLargeError } from './graph' import { workflowModule } from './scope' @@ -39,6 +42,12 @@ export const WorkflowPendingSpawnSchema = Schema.Struct({ attempt: attemptNumber, input: Schema.Json, loopIndex: Schema.NullOr(Schema.Finite).pipe(Schema.withDecodingDefaultKey(Effect.succeed(null))), + execution: Schema.Union([Schema.Literal('thread'), Schema.Literal('run')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('thread' as const)), + ), + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('terminate' as const)), + ), }) export type WorkflowPendingSpawn = Schema.Schema.Type @@ -54,6 +63,12 @@ export const WorkflowPendingMapSchema = Schema.Struct({ /** Items before this index have been started; later ones wait for a free slot. */ nextIndex: Schema.Finite, outputs: Schema.Record(Schema.String, Schema.Json), + execution: Schema.Union([Schema.Literal('thread'), Schema.Literal('run')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('thread' as const)), + ), + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('terminate' as const)), + ), }) export type WorkflowPendingMap = Schema.Schema.Type @@ -136,12 +151,37 @@ function runNodeInvocation(state: WorkflowState, nodeId: string) { } function spawnAndWait( + runId: string, threadId: string, - child: Pick, + child: Pick< + WorkflowPendingSpawn, + 'kind' | 'definitionName' | 'definitionVersion' | 'execution' | 'close' + >, childThreadId: string, input: JsonValue, tag: { readonly [key: string]: JsonValue }, ): RuntimeEffect[] { + if (child.execution === 'run') { + const childRunId = deterministicRunId(runId, childThreadId) + + return [ + launch({ + runId: childRunId, + kind: child.kind, + definitionName: child.definitionName, + definitionVersion: child.definitionVersion, + input, + close: child.close, + tag: `launch_${childRunId}`, + }), + wait({ + waitId: createWaitId(threadId, `spawn_${childRunId}`), + on: { type: 'runtime.child.settled', match: { runId: childRunId } }, + tag, + }), + ] + } + return [ spawn({ childThreadId, @@ -161,6 +201,16 @@ function spawnAndWait( ] } +function childMode( + execution: JsonValue | undefined, + close: JsonValue | undefined, +): WorkflowChildMode { + return { + execution: execution === 'run' ? 'run' : 'thread', + close: close === 'wait' ? 'wait' : 'terminate', + } +} + function mapChildThreadId(threadId: string, map: WorkflowPendingMap, index: number): string { return iterationChildWorkflowId(threadId, map.nodeId, 'map', map.attempt, index) } @@ -171,7 +221,7 @@ interface NodeEffect { } /** Every effect a node currently needs, attributed to that node. */ -function nodeEffects(state: WorkflowState, threadId: string): NodeEffect[] { +function nodeEffects(state: WorkflowState, threadId: string, runId: string): NodeEffect[] { const effects: NodeEffect[] = [] const add = (nodeId: string, items: readonly RuntimeEffect[]) => { @@ -189,7 +239,7 @@ function nodeEffects(state: WorkflowState, threadId: string): NodeEffect[] { add( item.nodeId, - spawnAndWait(threadId, item, item.childThreadId, item.input, { + spawnAndWait(runId, threadId, item, item.childThreadId, item.input, { ...tag, loopIndex: item.loopIndex, }), @@ -202,6 +252,7 @@ function nodeEffects(state: WorkflowState, threadId: string): NodeEffect[] { add( map.nodeId, spawnAndWait( + runId, threadId, map, mapChildThreadId(threadId, map, index), @@ -226,29 +277,75 @@ function nodeEffects(state: WorkflowState, threadId: string): NodeEffect[] { return effects } -function nodeIdForEffect(state: WorkflowState, threadId: string, effectId: string): string | null { - const match = nodeEffects(state, threadId).find( +function nodeIdForEffect( + state: WorkflowState, + threadId: string, + runId: string, + effectId: string, +): string | null { + const match = nodeEffects(state, threadId, runId).find( (item) => stableEffectId(threadId, item.effect) === effectId, ) return match?.nodeId ?? null } +interface UnsettledChild { + readonly key: string + readonly target: string + readonly execution: 'thread' | 'run' + readonly close: 'terminate' | 'wait' +} + +function childTarget(runId: string, childThreadId: string, execution: 'thread' | 'run'): string { + return execution === 'run' ? deterministicRunId(runId, childThreadId) : childThreadId +} + /** Children a node started that have not reported back yet. */ -function unsettledChildren(state: WorkflowState, nodeId: string, threadId: string): string[] { - const children = state.pendingSpawns - .filter((item) => item.nodeId === nodeId) - .map((item) => item.childThreadId) +function unsettledChildren( + state: WorkflowState, + nodeId: string, + threadId: string, + runId: string, +): UnsettledChild[] { + const children: UnsettledChild[] = [] + + for (const item of state.pendingSpawns) { + if (item.nodeId !== nodeId) { + continue + } + + const execution = item.execution === 'run' ? 'run' : 'thread' + + children.push({ + key: item.childThreadId, + target: childTarget(runId, item.childThreadId, execution), + execution, + close: item.close === 'wait' ? 'wait' : 'terminate', + }) + } for (const map of state.pendingMaps) { if (map.nodeId !== nodeId) { continue } + const execution = map.execution === 'run' ? 'run' : 'thread' + const close = map.close === 'wait' ? 'wait' : 'terminate' + for (let index = 0; index < map.nextIndex; index++) { - if (!(String(index) in map.outputs)) { - children.push(mapChildThreadId(threadId, map, index)) + if (String(index) in map.outputs) { + continue } + + const key = mapChildThreadId(threadId, map, index) + + children.push({ + key, + target: childTarget(runId, key, execution), + execution, + close, + }) } } @@ -257,32 +354,55 @@ function unsettledChildren(state: WorkflowState, nodeId: string, threadId: strin /** * Drops a node's pending work and cancels the children it leaves running. `settled` is the child - * whose result is finishing the node, which must not be cancelled. + * whose result is finishing the node, which must not be cancelled. A `wait` child run stays + * pending until it settles, so the node does not finish ahead of that cancellation. */ function withoutPendingWork( state: WorkflowState, nodeId: string, threadId: string, + runId: string, settled: string | null = null, ): WorkflowState { - const cancels = unsettledChildren(state, nodeId, threadId).filter( - (childThreadId) => childThreadId !== settled && !state.pendingCancels.includes(childThreadId), + const children = unsettledChildren(state, nodeId, threadId, runId) + + const cancels = children + .map((child) => child.target) + .filter((target) => target !== settled && !state.pendingCancels.includes(target)) + + const keptSpawns = new Set( + children + .filter( + (child) => child.execution === 'run' && child.close === 'wait' && child.target !== settled, + ) + .map((child) => child.key), + ) + + const keepMap = children.some( + (child) => child.execution === 'run' && child.close === 'wait' && child.target !== settled, ) return { ...state, pendingCancels: cancels.length === 0 ? state.pendingCancels : [...state.pendingCancels, ...cancels], - pendingSpawns: state.pendingSpawns.filter((item) => item.nodeId !== nodeId), - pendingMaps: state.pendingMaps.filter((item) => item.nodeId !== nodeId), + pendingSpawns: state.pendingSpawns.filter( + (item) => item.nodeId !== nodeId || keptSpawns.has(item.childThreadId), + ), + pendingMaps: state.pendingMaps.filter((item) => item.nodeId !== nodeId || keepMap), pendingSleeps: state.pendingSleeps.filter((item) => item.nodeId !== nodeId), pendingEffects: state.pendingEffects.filter((item) => item.nodeId !== nodeId), } } -function stopNode(state: WorkflowState, nodeId: string, threadId: string): WorkflowState { +function stopNode( + state: WorkflowState, + nodeId: string, + threadId: string, + runId: string, +): WorkflowState { return { - ...withoutPendingWork(state, nodeId, threadId), + ...withoutPendingWork(state, nodeId, threadId, runId), runningNodes: state.runningNodes.filter((id) => id !== nodeId), } } @@ -294,6 +414,7 @@ function finishNode( result: JsonValue | null, error: string | null, threadId: string, + runId: string, settled: string | null = null, ): WorkflowState { const id = `finished_${nodeId}` @@ -305,7 +426,7 @@ function finishNode( } return { - ...withoutPendingWork(state, nodeId, threadId, settled), + ...withoutPendingWork(state, nodeId, threadId, runId, settled), pendingEmits: [...state.pendingEmits.filter((item) => item.id !== id), { id, event }], } } @@ -316,23 +437,25 @@ function recordMapOutput( index: number, output: JsonValue | null, threadId: string, + runId: string, ): WorkflowState { if (String(index) in map.outputs) { return state } - const settled = mapChildThreadId(threadId, map, index) + const childThreadId = mapChildThreadId(threadId, map, index) + const settled = childTarget(runId, childThreadId, map.execution === 'run' ? 'run' : 'thread') const outputs = { ...map.outputs, [String(index)]: output } const size = utf8JsonBytes(outputs) if (size > map.limitBytes) { const error = new WorkflowValueTooLargeError(`Node ${map.nodeId} outputs`, size, map.limitBytes) - return finishNode(state, map.nodeId, null, error.message, threadId, settled) + return finishNode(state, map.nodeId, null, error.message, threadId, runId, settled) } if (Object.keys(outputs).length === map.items.length) { const ordered = map.items.map((_, i) => outputs[String(i)] ?? null) - return finishNode(state, map.nodeId, ordered, null, threadId, settled) + return finishNode(state, map.nodeId, ordered, null, threadId, runId, settled) } const nextIndex = Math.min(map.items.length, map.nextIndex + 1) @@ -437,7 +560,7 @@ export const workflowThread = workflowModule.thread({ const failed = Predicate.isString(error) && error.length > 0 return { - ...stopNode(state, nodeId, ctx.threadId), + ...stopNode(state, nodeId, ctx.threadId, ctx.runId), nodes: { ...state.nodes, [nodeId]: { @@ -457,7 +580,7 @@ export const workflowThread = workflowModule.thread({ case 'workflow.node.skipped': { const { nodeId, reason, cause } = event.payload return { - ...stopNode(state, nodeId, ctx.threadId), + ...stopNode(state, nodeId, ctx.threadId, ctx.runId), nodes: { ...state.nodes, [nodeId]: { @@ -483,7 +606,10 @@ export const workflowThread = workflowModule.thread({ ...state, pendingSpawns: [ ...state.pendingSpawns.filter((item) => item.nodeId !== nodeId), - event.payload, + { + ...event.payload, + ...childMode(event.payload.execution, event.payload.close), + }, ], } } @@ -501,6 +627,7 @@ export const workflowThread = workflowModule.thread({ ...state.pendingMaps.filter((item) => item.nodeId !== nodeId), { ...event.payload, + ...childMode(event.payload.execution, event.payload.close), nextIndex: Math.min(event.payload.concurrency, event.payload.items.length), outputs: {}, }, @@ -568,7 +695,7 @@ export const workflowThread = workflowModule.thread({ const effectId = event.payload.effectId const nodeId = Predicate.isString(effectId) - ? nodeIdForEffect(state, ctx.threadId, effectId) + ? nodeIdForEffect(state, ctx.threadId, ctx.runId, effectId) : null if (!nodeId) { @@ -578,7 +705,14 @@ export const workflowThread = workflowModule.thread({ const message = `Node ${nodeId} failed: ${error}` return { - ...finishNode(stopNode(state, nodeId, ctx.threadId), nodeId, null, message, ctx.threadId), + ...finishNode( + stopNode(state, nodeId, ctx.threadId, ctx.runId), + nodeId, + null, + message, + ctx.threadId, + ctx.runId, + ), nodes: { ...state.nodes, [nodeId]: { @@ -608,7 +742,7 @@ export const workflowThread = workflowModule.thread({ const map = state.pendingMaps.find((item) => item.nodeId === nodeId) if (error === null && map && Predicate.isNumber(tag.index)) { - return recordMapOutput(state, map, tag.index, result, ctx.threadId) + return recordMapOutput(state, map, tag.index, result, ctx.threadId, ctx.runId) } const node = state.nodes[nodeId] @@ -627,12 +761,15 @@ export const workflowThread = workflowModule.thread({ const satisfied = event.payload.event const satisfiedPayload = Predicate.isObject(satisfied) ? satisfied.payload : undefined - const settled = - Predicate.isObject(satisfiedPayload) && Predicate.isString(satisfiedPayload.threadId) - ? satisfiedPayload.threadId - : null + const settled = Predicate.isObject(satisfiedPayload) + ? Predicate.isString(satisfiedPayload.runId) + ? satisfiedPayload.runId + : Predicate.isString(satisfiedPayload.threadId) + ? satisfiedPayload.threadId + : null + : null - return finishNode(state, nodeId, result, error, ctx.threadId, settled) + return finishNode(state, nodeId, result, error, ctx.threadId, ctx.runId, settled) } default: @@ -646,7 +783,7 @@ export const workflowThread = workflowModule.thread({ effects.push(scheduleInvocation(state)) } - for (const item of nodeEffects(state, ctx.threadId)) { + for (const item of nodeEffects(state, ctx.threadId, ctx.runId)) { effects.push(item.effect) } @@ -654,8 +791,10 @@ export const workflowThread = workflowModule.thread({ effects.push(emit(emitReq.event)) } - for (const childThreadId of state.pendingCancels) { - effects.push(cancel(childThreadId)) + for (const childId of state.pendingCancels) { + effects.push( + childId.startsWith('run_') ? cancelRun(childId, `cancel_${childId}`) : cancel(childId), + ) } if (state.status === 'failed') { From d004ad4265d4c1bc2574c017282a324378110956 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 11:53:09 +0200 Subject: [PATCH 07/23] feat(runtime): authorize browser event streams with a signed run grant EventSource and browser WebSockets cannot send an Authorization header. A short-lived HMAC grant in the query string or WebSocket subprotocol lets one principal read one run. Co-authored-by: Cursor --- .changeset/run-grants.md | 7 + docs/production.md | 31 +++ packages/client/src/client.test.ts | 24 ++ packages/client/src/client.ts | 17 +- packages/runtime/src/index.ts | 8 + packages/runtime/src/looms.ts | 3 +- packages/runtime/src/server.ts | 10 + packages/runtime/src/server/fetch-handler.ts | 2 +- packages/runtime/src/server/grant.test.ts | 107 ++++++++ packages/runtime/src/server/grant.ts | 269 +++++++++++++++++++ 10 files changed, 472 insertions(+), 6 deletions(-) create mode 100644 .changeset/run-grants.md create mode 100644 packages/runtime/src/server/grant.test.ts create mode 100644 packages/runtime/src/server/grant.ts diff --git a/.changeset/run-grants.md b/.changeset/run-grants.md new file mode 100644 index 0000000..0d308fe --- /dev/null +++ b/.changeset/run-grants.md @@ -0,0 +1,7 @@ +--- +'@swirls/looms': minor +--- + +feat(runtime): authorize browser event streams with a signed run grant + +`grantAuth` checks a short-lived HMAC grant from the `grant` query parameter or a `v1.` WebSocket subprotocol. The grant allows one principal to read one run, which is the credential a browser can send on EventSource or WebSocket. diff --git a/docs/production.md b/docs/production.md index 866dee0..9a95504 100644 --- a/docs/production.md +++ b/docs/production.md @@ -60,6 +60,37 @@ Each configured level requires `Authorization: Bearer `; levels without a token keep the default behavior. The client's `workerCallbackToken` option sends the worker token on `workerCallback` requests. +Browsers cannot set headers on `EventSource` or `WebSocket`. `grantAuth` verifies +a short-lived HMAC grant from the `grant` query parameter, or from a `v1.` value +in `Sec-WebSocket-Protocol` when the query parameter is absent. The grant names +one run, one principal, and `read` access. It does not authorize writes, worker +callbacks, operations, the debugger, or a different run. `exp` is an exclusive +Unix millisecond timestamp. A host that accepts the grant as a WebSocket +subprotocol must echo that protocol on the upgrade response; the query parameter +needs no echo. + +```ts +import { Effect } from 'effect' +import { grantAuth, issueRunGrant } from '@swirls/looms/runtime' + +const secret = env.GRANT_SECRET +const grant = await Effect.runPromise( + issueRunGrant(secret, { + runId, + principal: userId, + access: 'read', + exp: Date.now() + 5 * 60_000, + }), +) + +createLooms({ + modules, + authorize: grantAuth({ secret }), +}) + +const events = new EventSource(`/runs/${runId}/events?live=true&grant=${grant}`) +``` + ## Retention and compaction Snapshots accelerate runtime recovery but do not rebuild arbitrary projections. diff --git a/packages/client/src/client.test.ts b/packages/client/src/client.test.ts index f454244..b5c5585 100644 --- a/packages/client/src/client.test.ts +++ b/packages/client/src/client.test.ts @@ -166,6 +166,30 @@ describe('createLoomsClient', () => { stop() }) + test('subscribeEvents puts the read grant on the event stream', async () => { + const calls: string[] = [] + + const client = createLoomsClient({ + grant: 'v1.abc.def', + fetch: async (input) => { + calls.push(hrefOf(input)) + + return new Response(null, { status: 401 }) + }, + }) + + const stop = client.subscribeEvents('run_1', () => undefined) + const start = Date.now() + + while (calls.length === 0 && Date.now() - start < 2_000) { + await Bun.sleep(10) + } + + stop() + expect(calls[0]).toContain('/runs/run_1/events') + expect(calls[0]).toContain('grant=v1.abc.def') + }) + test('stream yields live deltas then the completed run events', async () => { const client = createLoomsClient({ fetch: async (input, init) => { diff --git a/packages/client/src/client.ts b/packages/client/src/client.ts index 413b98e..1e406b9 100644 --- a/packages/client/src/client.ts +++ b/packages/client/src/client.ts @@ -68,6 +68,11 @@ export interface LoomsClientOptions { baseUrl?: string fetch?: typeof fetch workerCallbackToken?: string + /** + * Appended to the event-stream URL as `grant`. A read grant cannot authorize other requests, so + * it is not sent on them. + */ + grant?: string } export interface StartResult { @@ -83,6 +88,7 @@ export interface StreamHandle extends AsyncIterable { export function createLoomsClient(options: LoomsClientOptions = {}) { const baseUrl = (options.baseUrl ?? '').replace(/\/$/, '') const fetchImpl = options.fetch ?? fetch + const grantQuery = options.grant ? `&grant=${encodeURIComponent(options.grant)}` : '' function request( path: string, @@ -135,10 +141,13 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { const res = yield* Effect.tryPromise({ try: (signal) => - fetchImpl(`${baseUrl}/runs/${encodeURIComponent(runId)}/events?fromSeq=${fromSeq}`, { - headers, - signal, - }), + fetchImpl( + `${baseUrl}/runs/${encodeURIComponent(runId)}/events?fromSeq=${fromSeq}${grantQuery}`, + { + headers, + signal, + }, + ), catch: (cause) => new LoomsClientError('Event subscription failed', cause), }) diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index 1f041f8..64fe818 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -31,6 +31,14 @@ export { authDenied, bearerAuth, defaultAuthorize, + grantAuth, + issueRunGrant, + runGrantToken, + verifyRunGrant, + GrantError, + type GrantAuthOptions, + type GrantFailure, + type RunGrant, type AuthDecision, type AuthorizedAccess, type Authorize, diff --git a/packages/runtime/src/looms.ts b/packages/runtime/src/looms.ts index a1fc06d..ef8a2b9 100644 --- a/packages/runtime/src/looms.ts +++ b/packages/runtime/src/looms.ts @@ -71,7 +71,8 @@ export interface CreateLoomsOptions { * and operations routes return 503; `bearerAuth({ worker, operations })` enables them. * * A mounted debugger is authorized as `{ access: 'read', name: 'debugger' }`. Bearer tokens on - * reads block browser navigation, so hosts that need auth should use a cookie or session policy. + * reads block browser navigation. `grantAuth` covers a browser EventSource or WebSocket that can + * only send a query parameter or a subprotocol. */ readonly authorize?: Authorize /** diff --git a/packages/runtime/src/server.ts b/packages/runtime/src/server.ts index b2e30b9..9e3150b 100644 --- a/packages/runtime/src/server.ts +++ b/packages/runtime/src/server.ts @@ -13,6 +13,16 @@ export { type RouteAccess, type RouteInfo, } from './server/auth' +export { + grantAuth, + issueRunGrant, + runGrantToken, + verifyRunGrant, + GrantError, + type GrantAuthOptions, + type GrantFailure, + type RunGrant, +} from './server/grant' export interface RunningServer { port: number diff --git a/packages/runtime/src/server/fetch-handler.ts b/packages/runtime/src/server/fetch-handler.ts index 831604d..fe576b5 100644 --- a/packages/runtime/src/server/fetch-handler.ts +++ b/packages/runtime/src/server/fetch-handler.ts @@ -17,7 +17,7 @@ export interface FetchHandlerOptions { readonly store: EventStore /** * Decides whether a request may use a route. Defaults to open reads and writes with worker and - * operations routes unavailable; use `bearerAuth` or your own policy to change that. + * operations routes unavailable; use `bearerAuth`, `grantAuth`, or your own policy to change that. */ readonly authorize?: Authorize /** Debugger UI served beside the API and authorized as `{ access: 'read', name: 'debugger' }`. */ diff --git a/packages/runtime/src/server/grant.test.ts b/packages/runtime/src/server/grant.test.ts new file mode 100644 index 0000000..c016a4a --- /dev/null +++ b/packages/runtime/src/server/grant.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { createLooms, grantAuth, issueRunGrant, verifyRunGrant } from '../index' + +const secret = 'grant-secret' +const runId = 'run_1' + +const claims = { + runId, + principal: 'user_1', + access: 'read' as const, + exp: 5_000, +} + +describe('run grants', () => { + test('round-trips a signed grant and rejects a different secret', async () => { + const token = await Effect.runPromise(issueRunGrant(secret, claims)) + const grant = await Effect.runPromise(verifyRunGrant(secret, token, 4_000)) + + expect(grant).toEqual(claims) + + const forged = await Effect.runPromise( + verifyRunGrant('other-secret', token, 4_000).pipe(Effect.flip), + ) + + expect(forged.reason).toBe('signature') + }) + + test('rejects a tampered payload, a bad shape, and an empty principal', async () => { + const token = await Effect.runPromise(issueRunGrant(secret, claims)) + const [version, payload, signature] = token.split('.') + const flipped = payload?.[0] === 'A' ? 'B' : 'A' + const tampered = `${version}.${flipped}${payload?.slice(1)}.${signature}` + + const forged = await Effect.runPromise( + verifyRunGrant(secret, tampered, 4_000).pipe(Effect.flip), + ) + + expect(forged.reason).toBe('signature') + + const malformed = await Effect.runPromise( + verifyRunGrant(secret, 'not-a-grant', 4_000).pipe(Effect.flip), + ) + + expect(malformed.reason).toBe('malformed') + + const empty = await Effect.runPromise( + issueRunGrant(secret, { ...claims, principal: '' }).pipe(Effect.flip), + ) + + expect(empty.reason).toBe('claims') + }) + + test('expires at exp and honors clock skew', async () => { + const token = await Effect.runPromise(issueRunGrant(secret, claims)) + const expired = await Effect.runPromise(verifyRunGrant(secret, token, 5_000).pipe(Effect.flip)) + + expect(expired.reason).toBe('expired') + + const skewed = await Effect.runPromise(verifyRunGrant(secret, token, 5_000, 1)) + + expect(skewed.principal).toBe('user_1') + }) + + test('authorizes only a read of the granted run', async () => { + const token = await Effect.runPromise(issueRunGrant(secret, claims)) + + const looms = createLooms({ + modules: [], + authorize: grantAuth({ secret, now: () => 4_000 }), + }) + + const status = (input: RequestInfo, init?: RequestInit) => looms.fetch(new Request(input, init)) + + try { + const missing = await status(`http://looms.test/runs/${runId}/threads`) + const query = await status(`http://looms.test/runs/${runId}/threads?grant=${token}`) + + const protocol = await status(`http://looms.test/runs/${runId}/threads`, { + headers: { 'sec-websocket-protocol': `chat, ${token}` }, + }) + + const otherRun = await status(`http://looms.test/runs/run_2/threads?grant=${token}`) + const unscoped = await status(`http://looms.test/runs?grant=${token}`) + + const write = await status(`http://looms.test/runs/${runId}/wake?grant=${token}`, { + method: 'POST', + }) + + const badQuery = await status(`http://looms.test/runs/${runId}/threads?grant=nope`, { + headers: { 'sec-websocket-protocol': token }, + }) + + expect(missing?.status).toBe(401) + expect(query?.status).toBe(200) + expect(protocol?.status).toBe(200) + expect(otherRun?.status).toBe(403) + expect(unscoped?.status).toBe(403) + expect(write?.status).toBe(403) + expect(badQuery?.status).toBe(401) + } finally { + await looms.stop() + } + }) +}) diff --git a/packages/runtime/src/server/grant.ts b/packages/runtime/src/server/grant.ts new file mode 100644 index 0000000..fec702f --- /dev/null +++ b/packages/runtime/src/server/grant.ts @@ -0,0 +1,269 @@ +import { Data, Effect, Option, Schema } from 'effect' + +import { authAllowed, authDenied, type AuthDecision, type Authorize, type RouteInfo } from './auth' + +const textEncoder = new TextEncoder() +const textDecoder = new TextDecoder() + +const unauthorized = authDenied(401, 'Unauthorized') +const forbidden = authDenied(403, 'Forbidden') + +const GrantPayloadSchema = Schema.Struct({ + v: Schema.Literal(1), + runId: Schema.String, + principal: Schema.String, + access: Schema.Literal('read'), + exp: Schema.Finite, +}) + +const GrantPayloadJson = Schema.fromJsonString(GrantPayloadSchema) +const encodeGrantPayload = Schema.encodeSync(GrantPayloadJson) +const decodeGrantPayload = Schema.decodeUnknownOption(GrantPayloadJson) + +export type GrantFailure = 'malformed' | 'signature' | 'expired' | 'claims' + +export class GrantError extends Data.TaggedError('GrantError')<{ + readonly reason: GrantFailure +}> {} + +/** Claims carried by a run grant. `exp` is an exclusive expiry in Unix milliseconds. */ +export interface RunGrant { + readonly runId: string + readonly principal: string + readonly access: 'read' + readonly exp: number +} + +export interface GrantAuthOptions { + readonly secret: string + /** Clock used for expiry. Defaults to `Date.now`. */ + readonly now?: () => number + /** + * Extra time after `exp` that still verifies. Covers clock skew between the issuer and the host. + * Defaults to 0. + */ + readonly clockSkewMs?: number +} + +interface GrantPayload { + readonly v: 1 + readonly runId: string + readonly principal: string + readonly access: 'read' + readonly exp: number +} + +/** + * Signs a read grant for one run. The token is URL-safe and is also a legal WebSocket subprotocol + * value, so a browser can send it as a query parameter or as `Sec-WebSocket-Protocol`. + */ +export function issueRunGrant(secret: string, claims: RunGrant): Effect.Effect { + return Effect.gen(function* () { + const payload = grantPayload(claims) + + if (!payload) { + return yield* new GrantError({ reason: 'claims' }) + } + + const json = yield* Effect.try({ + try: () => encodeGrantPayload(payload), + catch: () => new GrantError({ reason: 'claims' }), + }) + + const encoded = encodeBase64Url(textEncoder.encode(json)) + const key = yield* importHmacKey(secret) + + const signature = yield* Effect.tryPromise({ + try: () => crypto.subtle.sign('HMAC', key, textEncoder.encode(encoded)), + catch: () => new GrantError({ reason: 'signature' }), + }) + + return `v1.${encoded}.${encodeBase64Url(new Uint8Array(signature))}` + }) +} + +/** Verifies the HMAC and expiry. A grant for another run fails later, when a route is checked. */ +export function verifyRunGrant( + secret: string, + token: string, + now: number, + clockSkewMs = 0, +): Effect.Effect { + return Effect.gen(function* () { + const parts = token.split('.') + + if (parts.length !== 3 || parts[0] !== 'v1') { + return yield* new GrantError({ reason: 'malformed' }) + } + + const encoded = parts[1] ?? '' + const signature = decodeBase64Url(parts[2] ?? '') + const payloadBytes = decodeBase64Url(encoded) + + if (!signature || !payloadBytes) { + return yield* new GrantError({ reason: 'malformed' }) + } + + const key = yield* importHmacKey(secret) + + const authentic = yield* Effect.tryPromise({ + try: () => crypto.subtle.verify('HMAC', key, signature, textEncoder.encode(encoded)), + catch: () => new GrantError({ reason: 'signature' }), + }) + + if (!authentic) { + return yield* new GrantError({ reason: 'signature' }) + } + + const decoded = decodeGrantPayload(textDecoder.decode(payloadBytes)) + + if ( + Option.isNone(decoded) || + decoded.value.runId.length === 0 || + decoded.value.principal.length === 0 + ) { + return yield* new GrantError({ reason: 'claims' }) + } + + const skew = finiteSkew(clockSkewMs) + + if (decoded.value.exp <= now - skew) { + return yield* new GrantError({ reason: 'expired' }) + } + + return { + runId: decoded.value.runId, + principal: decoded.value.principal, + access: decoded.value.access, + exp: decoded.value.exp, + } + }) +} + +/** + * Authorizes a read of the granted run. The token comes from the `grant` query parameter, or, when + * that is absent, from a `v1.` WebSocket subprotocol. Writes, worker callbacks, operations, and + * reads of any other run are refused. A host that accepts the subprotocol has to echo it on the + * WebSocket response. + */ +export function grantAuth(options: GrantAuthOptions): Authorize { + const now = options.now ?? Date.now + const skew = finiteSkew(options.clockSkewMs) + + return (request, route) => + Effect.gen(function* () { + const token = runGrantToken(request) + + if (token === undefined) { + return unauthorized + } + + return yield* verifyRunGrant(options.secret, token, now(), skew).pipe( + Effect.map((grant) => grantDecision(grant, route)), + Effect.catchTag('GrantError', () => Effect.succeed(unauthorized)), + ) + }) +} + +/** The grant token on this request, if the client sent one. */ +export function runGrantToken(request: Request): string | undefined { + const query = new URL(request.url).searchParams.get('grant') + + if (query) { + return query + } + + return websocketGrant(request.headers.get('sec-websocket-protocol')) +} + +function grantDecision(grant: RunGrant, route: RouteInfo): AuthDecision { + if (route.access !== 'read' || route.runId !== grant.runId) { + return forbidden + } + + return authAllowed +} + +function grantPayload(claims: RunGrant): GrantPayload | undefined { + if (claims.runId.length === 0 || claims.principal.length === 0 || !Number.isFinite(claims.exp)) { + return undefined + } + + return { + v: 1, + runId: claims.runId, + principal: claims.principal, + access: 'read', + exp: claims.exp, + } +} + +function finiteSkew(clockSkewMs: number | undefined): number { + if (clockSkewMs === undefined || !Number.isFinite(clockSkewMs) || clockSkewMs < 0) { + return 0 + } + + return clockSkewMs +} + +function websocketGrant(header: string | null): string | undefined { + if (header === null) { + return undefined + } + + for (const part of header.split(',')) { + const protocol = part.trim() + + if (protocol.startsWith('v1.')) { + return protocol + } + } + + return undefined +} + +function importHmacKey(secret: string): Effect.Effect { + return Effect.tryPromise({ + try: () => + crypto.subtle.importKey( + 'raw', + textEncoder.encode(secret), + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign', 'verify'], + ), + catch: () => new GrantError({ reason: 'signature' }), + }) +} + +function encodeBase64Url(bytes: Uint8Array): string { + let binary = '' + + for (const byte of bytes) { + binary += String.fromCharCode(byte) + } + + let encoded = btoa(binary).replaceAll('+', '-').replaceAll('/', '_') + + while (encoded.endsWith('=')) { + encoded = encoded.slice(0, -1) + } + + return encoded +} + +function decodeBase64Url(value: string): Uint8Array | undefined { + if (value.length === 0 || value.length % 4 === 1 || !/^[A-Za-z0-9_-]+$/.test(value)) { + return undefined + } + + const padded = value + '='.repeat((4 - (value.length % 4)) % 4) + const binary = atob(padded.replaceAll('-', '+').replaceAll('_', '/')) + const bytes = new Uint8Array(new ArrayBuffer(binary.length)) + + for (let index = 0; index < binary.length; index += 1) { + bytes[index] = binary.charCodeAt(index) + } + + return bytes +} From 2bfe492b10f5a69009deeeb74b843ce5e2814588 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 12:45:41 +0200 Subject: [PATCH 08/23] feat(cloudflare): add native Cloudflare Queues effect worker and push/pull consumers Add cloudflareQueueWorker producer implementing EffectWorker with dispatchSemantics: 'acknowledged', queue push/pull consumers with single-retry authority and mid-flight cancellation abort, and maxInFlightWorkerEffects fairness guard per run. Co-authored-by: Cursor --- .changeset/queue-worker.md | 7 + docs/production.md | 17 + packages/cloudflare/src/index.ts | 17 + packages/cloudflare/src/queue-worker.test.ts | 250 ++++++++++++ packages/cloudflare/src/queue-worker.ts | 363 ++++++++++++++++++ packages/runtime/src/looms.ts | 3 + packages/runtime/src/runtime.ts | 1 + packages/runtime/src/types.ts | 8 + packages/runtime/src/wake-engine.ts | 2 + .../runtime/src/worker-effect-dispatch.ts | 131 ++++--- packages/runtime/src/worker-protocol.test.ts | 65 ++++ 11 files changed, 812 insertions(+), 52 deletions(-) create mode 100644 .changeset/queue-worker.md create mode 100644 packages/cloudflare/src/queue-worker.test.ts create mode 100644 packages/cloudflare/src/queue-worker.ts diff --git a/.changeset/queue-worker.md b/.changeset/queue-worker.md new file mode 100644 index 0000000..ea61b99 --- /dev/null +++ b/.changeset/queue-worker.md @@ -0,0 +1,7 @@ +--- +'@swirls/looms': minor +--- + +feat(cloudflare): add native Cloudflare Queues effect worker and push/pull consumers + +Adds `cloudflareQueueWorker` producer implementing `EffectWorker` with `dispatchSemantics: 'acknowledged'`, queue push/pull consumers with single-retry authority and mid-flight cancellation abort, and `maxInFlightWorkerEffects` fairness guard per run. diff --git a/docs/production.md b/docs/production.md index 866dee0..ee0085c 100644 --- a/docs/production.md +++ b/docs/production.md @@ -32,6 +32,23 @@ verify the destination's idempotency record, then use `retryEffect` or `POST /runs/:runId/effects/:effectId/retry`. Never blindly retry a non-idempotent effect. +### Cloudflare Queues transport + +`cloudflareQueueWorker` connects Durable Objects to external workers or background fleets over Cloudflare Queues: + +```ts +import { cloudflareQueueWorker } from '@swirls/looms/cloudflare' + +const worker = cloudflareQueueWorker({ + routes: [{ handles: ['sandbox.exec', 'batch.convert'], queue: env.WORKER_QUEUE }], +}) +``` + +- **Acknowledged dispatch**: `dispatchSemantics: 'acknowledged'` treats a thrown send (including 429 rate limits) as an un-enqueued failure retried with backoff, eliminating ambiguous dispatch operator interventions. +- **One retry authority**: Consumers acknowledge the queue message immediately once the `started` callback succeeds. From that point on, Looms heartbeats and start-to-close deadlines govern execution liveness. +- **Fairness guard**: `maxInFlightWorkerEffects` caps concurrent in-flight dispatches per run, preventing a single fanout from saturating a shared queue. +- **In-flight cancellation**: Because enqueued messages cannot be recalled from Cloudflare Queues, in-flight attempts learn of cancellation from heartbeat responses and abort the task signal. + ## Inspection and repair - `inspectRun(runId)` and `GET /runs/:runId/status` expose stream bounds, state, diff --git a/packages/cloudflare/src/index.ts b/packages/cloudflare/src/index.ts index 47e4c3c..bc12469 100644 --- a/packages/cloudflare/src/index.ts +++ b/packages/cloudflare/src/index.ts @@ -16,3 +16,20 @@ export { type DurableObjectNamespaceLike, type DurableObjectStubLike, } from './router' +export { + cloudflareQueueWorker, + createMemoryQueue, + processQueueMessage, + queuePushConsumer, + type CloudflareQueueWorkerOptions, + type MemoryQueue, + type MessageBatchLike, + type QueueConsumerOptions, + type QueueEffectMessage, + type QueueLike, + type QueueMessageLike, + type QueueRoute, + type QueueSendOptions, + type QueueTaskContext, + type QueueTaskHandler, +} from './queue-worker' diff --git a/packages/cloudflare/src/queue-worker.test.ts b/packages/cloudflare/src/queue-worker.test.ts new file mode 100644 index 0000000..b75e4b8 --- /dev/null +++ b/packages/cloudflare/src/queue-worker.test.ts @@ -0,0 +1,250 @@ +import { describe, expect, test } from 'bun:test' + +import { Predicate } from 'effect' + +import { + cloudflareQueueWorker, + createMemoryQueue, + processQueueMessage, + queuePushConsumer, + type QueueTaskHandler, +} from './queue-worker' + +function urlOf(input: RequestInfo | URL): string { + if (Predicate.isString(input)) { + return input + } + + if (input instanceof URL) { + return input.href + } + + return input.url +} + +describe('cloudflare queues effect worker', () => { + test('producer enqueues formatted QueueEffectMessage', async () => { + const queue = createMemoryQueue() + + const worker = cloudflareQueueWorker({ + routes: [{ handles: ['sandbox.run', 'payment.charge'], queue }], + }) + + expect(worker.dispatchSemantics).toBe('acknowledged') + expect(Predicate.isFunction(worker.handles) && worker.handles('sandbox.run')).toBe(true) + expect(Predicate.isFunction(worker.handles) && worker.handles('unknown.task')).toBe(false) + + await worker.dispatch({ + runId: 'run_1', + effectId: 'eff_1', + attempt: 1, + type: 'sandbox.run', + input: { command: 'echo hello' }, + }) + + expect(queue.enqueued).toHaveLength(1) + const enqueued = queue.enqueued[0] + + expect(enqueued).toMatchObject({ + v: 1, + runId: 'run_1', + effectId: 'eff_1', + attempt: 1, + type: 'sandbox.run', + input: { command: 'echo hello' }, + }) + }) + + test('producer throws on 429 for acknowledged retry', async () => { + const queue = createMemoryQueue() + queue.setSimulation({ rateLimit429: true }) + + const worker = cloudflareQueueWorker({ + routes: [{ handles: ['sandbox.run'], queue }], + }) + + expect( + worker.dispatch({ + runId: 'run_1', + effectId: 'eff_1', + attempt: 1, + type: 'sandbox.run', + input: {}, + }), + ).rejects.toThrow('429 Too Many Requests') + }) + + test('producer offloads inputs exceeding threshold', async () => { + const queue = createMemoryQueue() + let offloaded = false + + const worker = cloudflareQueueWorker({ + routes: [{ handles: ['sandbox.run'], queue }], + offloadInput: async (input) => { + offloaded = true + return { $ref: 'blob_123', size: JSON.stringify(input).length } + }, + }) + + const largeInput = { payload: 'x'.repeat(120_000) } + + await worker.dispatch({ + runId: 'run_1', + effectId: 'eff_1', + attempt: 1, + type: 'sandbox.run', + input: largeInput, + }) + + expect(offloaded).toBe(true) + expect(queue.enqueued[0]?.input).toEqual({ $ref: 'blob_123', size: 120_014 }) + }) + + test('consumer acknowledges queue message immediately once started is accepted', async () => { + const queue = createMemoryQueue() + + await queue.send({ + v: 1, + runId: 'run_1', + effectId: 'eff_1', + attempt: 1, + type: 'sandbox.run', + input: { cmd: 'test' }, + deadlines: {}, + }) + + const calls: string[] = [] + + const mockFetch: typeof fetch = async (input) => { + const url = urlOf(input) + calls.push(url) + + if (url.includes('/started')) { + return new Response(JSON.stringify({ status: 'accepted' }), { status: 200 }) + } + + if (url.includes('/complete')) { + return new Response(JSON.stringify({ status: 'accepted' }), { status: 200 }) + } + + return new Response(null, { status: 200 }) + } + + const handler: QueueTaskHandler = async () => { + // At this point in handler execution, the queue message must already be acknowledged + expect(queue.acknowledged).toHaveLength(1) + return [{ type: 'sandbox.finished', payload: { ok: true } }] + } + + const consumer = queuePushConsumer({ + handlers: { 'sandbox.run': handler }, + callbackUrl: 'http://looms.internal', + fetch: mockFetch, + }) + + const batch = queue.receiveBatch() + await consumer.consumeBatch(batch) + + expect(calls.some((c) => c.includes('/started'))).toBe(true) + expect(calls.some((c) => c.includes('/complete'))).toBe(true) + expect(queue.acknowledged).toHaveLength(1) + }) + + test('stale attempt is acknowledged on queue without running handler', async () => { + const queue = createMemoryQueue() + + await queue.send({ + v: 1, + runId: 'run_1', + effectId: 'eff_1', + attempt: 1, + type: 'sandbox.run', + input: {}, + deadlines: {}, + }) + + let handlerRan = false + + const mockFetch: typeof fetch = async (input) => { + const url = urlOf(input) + + if (url.includes('/started')) { + return new Response(JSON.stringify({ status: 'stale' }), { status: 200 }) + } + + return new Response(null, { status: 200 }) + } + + const handler: QueueTaskHandler = async () => { + handlerRan = true + return [] + } + + const consumer = queuePushConsumer({ + handlers: { 'sandbox.run': handler }, + callbackUrl: 'http://looms.internal', + fetch: mockFetch, + }) + + const batch = queue.receiveBatch() + await consumer.consumeBatch(batch) + + expect(handlerRan).toBe(false) + expect(queue.acknowledged).toHaveLength(1) + }) + + test('cancellation mid-flight signals handler abort', async () => { + const queue = createMemoryQueue() + + await queue.send({ + v: 1, + runId: 'run_1', + effectId: 'eff_1', + attempt: 1, + type: 'sandbox.run', + input: {}, + deadlines: {}, + }) + + const mockFetch: typeof fetch = async (input) => { + const url = urlOf(input) + + if (url.includes('/started')) { + return new Response(JSON.stringify({ status: 'accepted' }), { status: 200 }) + } + + if (url.includes('/heartbeat')) { + return new Response(JSON.stringify({ status: 'accepted', cancelRequested: true }), { + status: 200, + }) + } + + return new Response(null, { status: 200 }) + } + + let aborted = false + + const handler: QueueTaskHandler = async (_task, ctx) => { + ctx.signal.addEventListener('abort', () => { + aborted = true + }) + + // Send a heartbeat which returns cancelRequested + await ctx.heartbeat({ progress: 50 }) + return [] + } + + const batch = queue.receiveBatch() + const message = batch.messages[0] + + if (message) { + await processQueueMessage(message, { + handlers: { 'sandbox.run': handler }, + callbackUrl: 'http://looms.internal', + fetch: mockFetch, + }) + } + + expect(aborted).toBe(true) + }) +}) diff --git a/packages/cloudflare/src/queue-worker.ts b/packages/cloudflare/src/queue-worker.ts new file mode 100644 index 0000000..bb049ab --- /dev/null +++ b/packages/cloudflare/src/queue-worker.ts @@ -0,0 +1,363 @@ +import { DateTime, Predicate, Schema } from 'effect' + +import { stringifyJson, type EventInput, type JsonValue } from '@looms/core' +import type { EffectWorker, EffectWorkerTask } from '@looms/runtime' + +const CallbackResponseSchema = Schema.Struct({ + status: Schema.Union([ + Schema.Literal('accepted'), + Schema.Literal('stale'), + Schema.Literal('cancelled'), + ]), + cancelRequested: Schema.optional(Schema.Boolean), +}) + +const decodeCallbackResponse = Schema.decodeUnknownOption(CallbackResponseSchema) + +function utcNow(): Date { + return DateTime.toDateUtc(DateTime.nowUnsafe()) +} + +export interface QueueEffectMessage { + readonly v: 1 + readonly runId: string + readonly effectId: string + readonly attempt: number + readonly type: string + readonly input: JsonValue + readonly traceparent?: string + readonly deadlines: { + readonly scheduleToStartTimeoutMs?: number + readonly startToCloseTimeoutMs?: number + readonly heartbeatTimeoutMs?: number + } +} + +export interface QueueSendOptions { + readonly contentType?: 'json' | 'text' | 'bytes' | 'v8' +} + +export interface QueueLike { + send(message: Body, options?: QueueSendOptions): Promise +} + +export interface QueueRoute { + readonly handles: ReadonlyArray | ((type: string) => boolean) + readonly queue: QueueLike +} + +export interface CloudflareQueueWorkerOptions { + readonly routes: readonly QueueRoute[] + /** Optional offload hook when input exceeds ~100 KB (Cloudflare Queues cap is 128 KB). */ + readonly offloadInput?: (input: JsonValue) => Promise +} + +export function cloudflareQueueWorker(options: CloudflareQueueWorkerOptions): EffectWorker { + const isHandled = (type: string) => + options.routes.some((route) => + Predicate.isFunction(route.handles) ? route.handles(type) : route.handles.includes(type), + ) + + return { + dispatchSemantics: 'acknowledged', + handles: isHandled, + dispatch: (task: EffectWorkerTask): Promise => { + const route = options.routes.find((candidate) => + Predicate.isFunction(candidate.handles) + ? candidate.handles(task.type) + : candidate.handles.includes(task.type), + ) + + if (!route) { + return Promise.reject( + new Error(`No Cloudflare Queue route handles effect type "${task.type}"`), + ) + } + + const input = task.input + const estimatedBytes = stringifyJson(input).length + + const offloadPromise = + estimatedBytes > 100_000 && options.offloadInput + ? options.offloadInput(task.input) + : Promise.resolve(input) + + return offloadPromise.then((resolvedInput) => { + const message: QueueEffectMessage = { + v: 1, + runId: task.runId, + effectId: task.effectId, + attempt: task.attempt, + type: task.type, + input: resolvedInput, + deadlines: {}, + } + + return route.queue.send(message, { contentType: 'json' }) + }) + }, + cancel: () => undefined, + } +} + +export interface QueueTaskContext { + readonly signal: AbortSignal + readonly heartbeat: (details?: JsonValue) => Promise + readonly secrets: () => Promise> +} + +export type QueueTaskHandler = ( + task: QueueEffectMessage, + ctx: QueueTaskContext, +) => Promise | readonly EventInput[] + +export interface QueueMessageLike { + readonly id: string + readonly timestamp: Date + readonly body: Body + readonly attempts?: number + ack(): void + retry(options?: { delaySeconds?: number }): void +} + +export interface MessageBatchLike { + readonly queue: string + readonly messages: readonly QueueMessageLike[] + ackAll(): void + retryAll(options?: { delaySeconds?: number }): void +} + +export interface QueueConsumerOptions { + readonly handlers: Record + readonly callbackUrl: string + readonly workerToken?: string + readonly fetch?: typeof fetch +} + +const emptySecrets = (): Promise> => Promise.resolve({}) + +export function processQueueMessage( + message: QueueMessageLike, + options: QueueConsumerOptions, +): Promise { + const fetchImpl = options.fetch ?? fetch + const task = message.body + const handler = options.handlers[task.type] + + if (!handler) { + message.retry() + return Promise.resolve() + } + + const baseUrl = options.callbackUrl.replace(/\/$/, '') + + const headers = new Headers({ + 'content-type': 'application/json', + }) + + if (options.workerToken) { + headers.set('authorization', `Bearer ${options.workerToken}`) + } + + const startedUrl = `${baseUrl}/runs/${encodeURIComponent(task.runId)}/effects/${encodeURIComponent(task.effectId)}/${task.attempt}/started` + + return fetchImpl(startedUrl, { + method: 'POST', + headers, + body: stringifyJson({}), + }) + .then((startedRes) => { + if (!startedRes.ok) { + message.retry() + return undefined + } + + return startedRes.json().then((startedJson) => { + const startedData = decodeCallbackResponse(startedJson) + + if (startedData._tag === 'None') { + message.retry() + return undefined + } + + const { status, cancelRequested } = startedData.value + + if (status === 'stale' || status === 'cancelled') { + message.ack() + return undefined + } + + message.ack() + + const controller = new AbortController() + + if (cancelRequested) { + controller.abort() + } + + const heartbeatUrl = `${baseUrl}/runs/${encodeURIComponent(task.runId)}/effects/${encodeURIComponent(task.effectId)}/${task.attempt}/heartbeat` + + const heartbeat = (details?: JsonValue): Promise => + fetchImpl(heartbeatUrl, { + method: 'POST', + headers, + body: stringifyJson(details === undefined ? {} : { details }), + }).then((res) => { + if (res.ok) { + return res.json().then((dataJson) => { + const data = decodeCallbackResponse(dataJson) + + if (data._tag === 'Some' && data.value.cancelRequested) { + controller.abort() + } + }) + } + + return undefined + }) + + const handlerPromise = Promise.resolve( + handler(task, { + signal: controller.signal, + heartbeat, + secrets: emptySecrets, + }), + ) + + return handlerPromise + .then((events) => { + const completeUrl = `${baseUrl}/runs/${encodeURIComponent(task.runId)}/effects/${encodeURIComponent(task.effectId)}/${task.attempt}/complete` + + return fetchImpl(completeUrl, { + method: 'POST', + headers, + body: stringifyJson({ events: [...events] }), + }).then(() => undefined) + }) + .catch((cause: unknown) => { + const error = cause instanceof Error ? cause.message : String(cause) + const failUrl = `${baseUrl}/runs/${encodeURIComponent(task.runId)}/effects/${encodeURIComponent(task.effectId)}/${task.attempt}/fail` + + return fetchImpl(failUrl, { + method: 'POST', + headers, + body: stringifyJson({ error }), + }).then(() => undefined) + }) + }) + }) + .catch(() => { + message.retry() + }) +} + +export function queuePushConsumer(options: QueueConsumerOptions) { + return { + consumeBatch: (batch: MessageBatchLike): Promise => + batch.messages.reduce( + (prev, msg) => prev.then(() => processQueueMessage(msg, options)), + Promise.resolve(), + ), + } +} + +export interface MemoryQueueSimulation { + duplicateDelivery?: boolean + crashBeforeStarted?: boolean + crashAfterStarted?: boolean + rateLimit429?: boolean +} + +export interface MemoryQueue extends QueueLike { + readonly messages: readonly QueueEffectMessage[] + readonly enqueued: readonly QueueEffectMessage[] + readonly acknowledged: readonly string[] + readonly retried: readonly string[] + receiveBatch(count?: number): MessageBatchLike + setSimulation(sim: MemoryQueueSimulation): void +} + +export function createMemoryQueue(): MemoryQueue { + const enqueuedList: QueueEffectMessage[] = [] + const acknowledgedList: string[] = [] + const retriedList: string[] = [] + let simulation: MemoryQueueSimulation = {} + + return { + get messages() { + return [...enqueuedList] + }, + get enqueued() { + return [...enqueuedList] + }, + get acknowledged() { + return [...acknowledgedList] + }, + get retried() { + return [...retriedList] + }, + setSimulation(sim: MemoryQueueSimulation) { + simulation = { ...simulation, ...sim } + }, + send: (message: QueueEffectMessage): Promise => { + if (simulation.rateLimit429) { + return Promise.reject(new Error('429 Too Many Requests: Queue rate limit exceeded')) + } + + enqueuedList.push(message) + return Promise.resolve() + }, + receiveBatch: (count = 10): MessageBatchLike => { + const selected = enqueuedList.slice(0, count) + + const queueMessages: QueueMessageLike[] = selected.map((msg, idx) => { + const id = `msg_${idx}_${msg.effectId}` + + return { + id, + timestamp: utcNow(), + body: msg, + ack: () => { + acknowledgedList.push(id) + }, + retry: () => { + retriedList.push(id) + }, + } + }) + + if (simulation.duplicateDelivery && queueMessages.length > 0) { + const first = queueMessages[0] + + if (first) { + queueMessages.push({ + id: `${first.id}_dup`, + timestamp: utcNow(), + body: first.body, + ack: () => { + acknowledgedList.push(`${first.id}_dup`) + }, + retry: () => { + retriedList.push(`${first.id}_dup`) + }, + }) + } + } + + return { + queue: 'looms-effects-queue', + messages: queueMessages, + ackAll: () => { + for (const m of queueMessages) { + m.ack() + } + }, + retryAll: () => { + for (const m of queueMessages) { + m.retry() + } + }, + } + }, + } +} diff --git a/packages/runtime/src/looms.ts b/packages/runtime/src/looms.ts index a1fc06d..810b03d 100644 --- a/packages/runtime/src/looms.ts +++ b/packages/runtime/src/looms.ts @@ -64,6 +64,8 @@ export interface CreateLoomsOptions { /** Starts runs from effect handlers. Defaults to launching into this runtime's own store. */ readonly launcher?: RunLauncher readonly worker?: EffectWorker + /** Maximum concurrent in-flight worker effects dispatched per run (fairness guard). */ + readonly maxInFlightWorkerEffects?: number /** Failure-isolated runtime observation sink. */ readonly observer?: RuntimeObserver /** @@ -202,6 +204,7 @@ export function createLooms(options: CreateLoomsOptions = {}): Looms { launcher: options.launcher, localLauncher: options.launcher === undefined, worker: options.worker, + maxInFlightWorkerEffects: options.maxInFlightWorkerEffects, observer: options.observer, }) diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index ac0b33b..846dbd6 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -338,6 +338,7 @@ export function createRuntime scheduler, diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 850756b..49c4fc4 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -50,6 +50,8 @@ export interface CreateRuntimeOptions< */ readonly snapshotEvery?: number readonly maxWakeIterations?: number + /** Maximum concurrent in-flight worker effects dispatched per run (fairness guard). */ + readonly maxInFlightWorkerEffects?: number /** Maximum active and queued mutation operations per run. Default 256. */ readonly maxPendingRunOperations?: number /** Defaults to the EventStore's attached store or an in-memory SnapshotStore. */ @@ -101,6 +103,12 @@ export interface EffectWorker { readonly canDispatch?: (task: EffectWorkerTask) => boolean dispatch(task: EffectWorkerTask): void | Promise cancel?(task: EffectWorkerTask): void | Promise + /** + * Dispatch semantics: + * - 'unacknowledged' (default): failed dispatch is treated as ambiguous (wait for schedule-to-start or operator). + * - 'acknowledged': resolved send means enqueued, thrown error means not enqueued (retryable dispatch with backoff). + */ + readonly dispatchSemantics?: 'unacknowledged' | 'acknowledged' } export interface StartRunArgs { diff --git a/packages/runtime/src/wake-engine.ts b/packages/runtime/src/wake-engine.ts index 341c9d6..361dae5 100644 --- a/packages/runtime/src/wake-engine.ts +++ b/packages/runtime/src/wake-engine.ts @@ -49,6 +49,7 @@ export interface WakeEngineOptions { readonly repository: RuntimeRepository readonly snapshotEvery: number readonly maxWakeIterations: number + readonly maxInFlightWorkerEffects?: number readonly worker?: EffectWorker readonly observer?: RuntimeObserver readonly scheduler: () => WakeScheduler @@ -247,6 +248,7 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake registry, worker: options.worker, now, + maxInFlightWorkerEffects: options.maxInFlightWorkerEffects, }) if (workerPlans.length > 0) { diff --git a/packages/runtime/src/worker-effect-dispatch.ts b/packages/runtime/src/worker-effect-dispatch.ts index 478a88d..7c2e1b9 100644 --- a/packages/runtime/src/worker-effect-dispatch.ts +++ b/packages/runtime/src/worker-effect-dispatch.ts @@ -103,56 +103,69 @@ export function planWorkerEffects(input: { readonly registry: ComposedRegistry readonly worker?: EffectWorker readonly now: number + readonly maxInFlightWorkerEffects?: number }): ReadonlyArray { - return input.state.outstandingEffects - .filter((item) => { - const dueAt = deferredUntil(item.effect) - - if (dueAt !== undefined && dueAt > input.now + 5) { - return false - } - - const execution = input.state.effectExecutions[item.effectId] - - if (!isWorkerEffect({ ...input, item, execution })) { - return false - } - - return ( - !execution || - execution.status === 'queued' || - (execution.status === 'retry_wait' && - execution.nextAttemptAt !== null && - execution.nextAttemptAt <= input.now + 5) - ) - }) - .map((item) => { - const existing = input.state.effectExecutions[item.effectId] - - const attempt = - existing?.status === 'queued' ? existing.attempt : (existing?.attempt ?? 0) + 1 - - const retry = input.registry.effects.get(item.effect.type)?.retry - - return { - item, - attempt, - queuedEvent: - existing?.status === 'queued' - ? undefined - : createEvent(input.runId, { - type: 'runtime.effect.queued', - payload: { effectId: item.effectId, attempt, deadlineAt: null }, - threadId: item.threadId, - causationId: item.causingEventId, - origin: { type: 'system' }, - }), - task: workerTask(input.runId, item, attempt), - scheduleToStartDeadlineAt: retry?.scheduleToStartTimeoutMs - ? input.now + retry.scheduleToStartTimeoutMs - : null, - } - }) + const inFlightStatuses = new Set(['dispatched', 'started', 'heartbeat', 'cancel_requested']) + + const currentInFlight = Object.values(input.state.effectExecutions).filter((execution) => + inFlightStatuses.has(execution.status), + ).length + + const limit = + input.maxInFlightWorkerEffects !== undefined && input.maxInFlightWorkerEffects > 0 + ? Math.max(0, input.maxInFlightWorkerEffects - currentInFlight) + : undefined + + const candidates = input.state.outstandingEffects.filter((item) => { + const dueAt = deferredUntil(item.effect) + + if (dueAt !== undefined && dueAt > input.now + 5) { + return false + } + + const execution = input.state.effectExecutions[item.effectId] + + if (!isWorkerEffect({ ...input, item, execution })) { + return false + } + + return ( + !execution || + execution.status === 'queued' || + (execution.status === 'retry_wait' && + execution.nextAttemptAt !== null && + execution.nextAttemptAt <= input.now + 5) + ) + }) + + const selected = limit === undefined ? candidates : candidates.slice(0, limit) + + return selected.map((item) => { + const existing = input.state.effectExecutions[item.effectId] + + const attempt = existing?.status === 'queued' ? existing.attempt : (existing?.attempt ?? 0) + 1 + + const retry = input.registry.effects.get(item.effect.type)?.retry + + return { + item, + attempt, + queuedEvent: + existing?.status === 'queued' + ? undefined + : createEvent(input.runId, { + type: 'runtime.effect.queued', + payload: { effectId: item.effectId, attempt, deadlineAt: null }, + threadId: item.threadId, + causationId: item.causingEventId, + origin: { type: 'system' }, + }), + task: workerTask(input.runId, item, attempt), + scheduleToStartDeadlineAt: retry?.scheduleToStartTimeoutMs + ? input.now + retry.scheduleToStartTimeoutMs + : null, + } + }) } /** @@ -198,10 +211,24 @@ export function dispatchWorkerEffect( new RuntimeExecutionError(cause instanceof Error ? cause.message : String(cause)), }).pipe( Effect.match({ - onFailure: (error) => - plan.scheduleToStartDeadlineAt === null + onFailure: (error) => { + if (worker.dispatchSemantics === 'acknowledged') { + return createEvent(runId, { + type: 'runtime.effect.failed', + payload: { + effectId: plan.item.effectId, + error: `Worker dispatch failed: ${error.message}`, + }, + threadId: plan.item.threadId, + causationId: plan.item.causingEventId, + origin: { type: 'system' }, + }) + } + + return plan.scheduleToStartDeadlineAt === null ? ambiguous(`worker dispatch outcome is ambiguous: ${error.message}`) - : dispatched, + : dispatched + }, onSuccess: () => dispatched, }), ) diff --git a/packages/runtime/src/worker-protocol.test.ts b/packages/runtime/src/worker-protocol.test.ts index fe4f539..e6ad2bd 100644 --- a/packages/runtime/src/worker-protocol.test.ts +++ b/packages/runtime/src/worker-protocol.test.ts @@ -703,4 +703,69 @@ describe('durable external effect workers', () => { expect((await looms.getRun(started.runId)).status).toBe('cancelled') await looms.stop() }) + + test('acknowledged dispatch semantics fails and retries without marking ambiguous', async () => { + let attempts = 0 + + const looms = createLooms({ + modules: [workerModule({ maxAttempts: 3, backoffMs: 1 })], + worker: { + dispatchSemantics: 'acknowledged', + dispatch: () => { + attempts += 1 + throw new Error('429 Queue Rate Exceeded') + }, + }, + }) + + const started = await looms.start(workerDefinition, {}) + const events = await looms.getEvents(started.runId) + + expect(attempts).toBe(1) + expect(events.some((event) => event.type === 'runtime.effect.failed')).toBe(true) + expect(events.some((event) => event.type === 'runtime.effect.ambiguous')).toBe(false) + + await looms.stop() + }) + + test('maxInFlightWorkerEffects caps planned dispatches per run', async () => { + const dispatched: EffectWorkerTask[] = [] + + const multiEffect = defineEffect({ type: 'worker.multi' }) + + const fanoutModule = defineRuntimeModule({ + namespace: 'fanout', + protocolVersion: '1.0.0', + effects: { multiEffect }, + threads: { + fanout: defineThread({ + kind: 'fanout', + initialState: () => ({}), + step: (state) => state, + effects: () => [ + invoke(multiEffect, {}, 'm1'), + invoke(multiEffect, {}, 'm2'), + invoke(multiEffect, {}, 'm3'), + invoke(multiEffect, {}, 'm4'), + ], + }), + }, + definitions: [{ kind: 'fanout', name: 'fanout', version: 'v1' }], + }) + + const looms = createLooms({ + modules: [fanoutModule], + maxInFlightWorkerEffects: 2, + worker: { + dispatch: (task) => { + dispatched.push(task) + }, + }, + }) + + await looms.start({ kind: 'fanout', name: 'fanout', version: 'v1' }, {}) + expect(dispatched).toHaveLength(2) + + await looms.stop() + }) }) From 6ca8aa4c7ddb7e8df8e5fe3f3b24abdb21e98044 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 13:32:39 +0200 Subject: [PATCH 09/23] feat(otel): propagate W3C trace context across runs, effects, worker tasks, and child runs Store W3C traceparent and tracestate on RunStartIdentity, pass child trace contexts to in-process effect handlers (EffectContext) and external worker tasks (EffectWorkerTask), link child runs spawned from effects with child span IDs, and export trace context parsing/injection utilities from @looms/otel. Co-authored-by: Cursor --- .changeset/otel-trace.md | 7 + docs/production.md | 11 ++ packages/client/src/client.ts | 2 + packages/core/src/effects.ts | 4 + packages/core/src/fold.ts | 7 +- packages/core/src/index.ts | 1 + packages/core/src/launcher.ts | 4 + packages/core/src/protocol.ts | 4 + packages/core/src/state.ts | 4 + packages/core/src/trace.ts | 136 ++++++++++++++++++ packages/otel/package.json | 1 + packages/otel/src/index.test.ts | 51 ++++++- packages/otel/src/index.ts | 88 ++++++++++++ packages/runtime/src/cancellation.ts | 12 +- packages/runtime/src/helpers.ts | 14 +- .../runtime/src/launch-provenance.test.ts | 58 ++++++++ packages/runtime/src/local-effect-executor.ts | 84 ++++++++--- packages/runtime/src/looms.ts | 14 +- packages/runtime/src/runtime.ts | 18 ++- packages/runtime/src/server/routes/runs.ts | 6 + packages/runtime/src/types.ts | 6 + packages/runtime/src/wake-engine.ts | 2 + .../runtime/src/worker-effect-dispatch.ts | 22 ++- 23 files changed, 521 insertions(+), 35 deletions(-) create mode 100644 .changeset/otel-trace.md create mode 100644 packages/core/src/trace.ts diff --git a/.changeset/otel-trace.md b/.changeset/otel-trace.md new file mode 100644 index 0000000..65a9b0f --- /dev/null +++ b/.changeset/otel-trace.md @@ -0,0 +1,7 @@ +--- +'@swirls/looms': minor +--- + +feat(otel): propagate W3C trace context across runs, effects, worker tasks, and child runs + +Stores W3C `traceparent` and `tracestate` on `RunStartIdentity`, exposes child spans on `EffectContext` and `EffectWorkerTask`, propagates trace links into child runs, and exports trace context utilities from `@looms/otel`. diff --git a/docs/production.md b/docs/production.md index 866dee0..bc69f1b 100644 --- a/docs/production.md +++ b/docs/production.md @@ -106,3 +106,14 @@ and startup deadline recovery. Attach a `RuntimeObserver` and core. Alert on append conflicts/errors, append and snapshot latency, timer lateness, ambiguous effects, timeout/retry volume, cancellation ambiguity, projector lag, and dead letters. + +### W3C trace propagation + +When a trigger passes `traceparent` (and optional `tracestate`) as HTTP headers or in `startRun`, Looms stores it in `RunStartIdentity` and propagates child trace contexts across: + +- In-process effect handlers via `EffectContext.traceparent` +- External worker dispatches via `EffectWorkerTask.traceparent` +- Cloudflare Queue effect messages via `QueueEffectMessage.traceparent` +- Child runs spawned from effects with child span IDs linked to the parent trace ID + +Use `@looms/otel` (`createTraceparent`, `childTraceparent`, `injectTraceCarrier`, `extractTraceCarrier`) for W3C carrier manipulation. diff --git a/packages/client/src/client.ts b/packages/client/src/client.ts index 413b98e..243749e 100644 --- a/packages/client/src/client.ts +++ b/packages/client/src/client.ts @@ -179,6 +179,8 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { definitionVersion?: string input?: JsonValue runId?: string + traceparent?: string + tracestate?: string }) => request('/runs', StartResultSchema, { method: 'POST', body: stringifyJson(args) }) const workerCallback = (runId: string, input: WorkerCallbackInput) => { diff --git a/packages/core/src/effects.ts b/packages/core/src/effects.ts index 57035c6..912a255 100644 --- a/packages/core/src/effects.ts +++ b/packages/core/src/effects.ts @@ -187,6 +187,8 @@ export interface EffectContext { readonly threadId: string readonly causingEventId: string readonly signal: AbortSignal + readonly traceparent?: string + readonly tracestate?: string emit(event: EventInput): Promise } @@ -196,6 +198,8 @@ export interface ScopedEffectContext { readonly threadId: string readonly causingEventId: string readonly signal: AbortSignal + readonly traceparent?: string + readonly tracestate?: string emit(event: E): Promise } diff --git a/packages/core/src/fold.ts b/packages/core/src/fold.ts index 535df71..fc6fa6d 100644 --- a/packages/core/src/fold.ts +++ b/packages/core/src/fold.ts @@ -115,7 +115,10 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi const definitionVersion = readString(payload, 'definitionVersion') ?? DEFAULT_DEFINITION_VERSION - const startIdentity = + const traceparent = readString(payload, 'traceparent') + const tracestate = readString(payload, 'tracestate') + + const startIdentity: RunStartIdentity | null = rootThreadId && kind && definitionName && definitionVersion ? { kind, @@ -127,6 +130,8 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi rootThreadId, launchedBy: readRunLaunchSource(payload.launchedBy), correlationId: event.correlationId ?? null, + ...(traceparent ? { traceparent } : {}), + ...(tracestate ? { tracestate } : {}), } : null diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c1ee11b..cc9fc41 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -40,5 +40,6 @@ export * from './event-index' export * from './limits' export * from './definition-store' export * from './launcher' +export * from './trace' export * from './published' export type { DebuggerHost, DebuggerRequest } from './debugger-host' diff --git a/packages/core/src/launcher.ts b/packages/core/src/launcher.ts index 70d7370..630c416 100644 --- a/packages/core/src/launcher.ts +++ b/packages/core/src/launcher.ts @@ -36,6 +36,10 @@ export interface LaunchRunArgs { * correlation id, or to the launching run's id when that run has none. */ readonly correlationId?: string + /** W3C traceparent propagated from the launching effect or parent run. */ + readonly traceparent?: string + /** W3C tracestate accompanying traceparent. */ + readonly tracestate?: string } /** Enough of a run to decide whether a previous fire is still in progress. */ diff --git a/packages/core/src/protocol.ts b/packages/core/src/protocol.ts index 527ceaf..7d11701 100644 --- a/packages/core/src/protocol.ts +++ b/packages/core/src/protocol.ts @@ -19,6 +19,8 @@ export const protocolCatalog = defineEventCatalog('runtime', { input: JsonValueSchema, requestedThreadId: Schema.optional(Schema.NullOr(Schema.String)), launchedBy: Schema.optional(RunLaunchSourceSchema), + traceparent: Schema.optional(Schema.String), + tracestate: Schema.optional(Schema.String), }), 'run.completed': Schema.Struct({ output: Schema.NullOr(JsonValueSchema), @@ -34,6 +36,8 @@ export const protocolCatalog = defineEventCatalog('runtime', { definitionVersion: DefinitionVersionSchema, /** In the payload, not the envelope: an envelope `effectId` would complete the effect. */ effectId: Schema.String, + traceparent: Schema.optional(Schema.String), + tracestate: Schema.optional(Schema.String), }), 'run.forked': Schema.Struct({ sourceRunId: Schema.String, diff --git a/packages/core/src/state.ts b/packages/core/src/state.ts index 58d370b..3b3085b 100644 --- a/packages/core/src/state.ts +++ b/packages/core/src/state.ts @@ -71,6 +71,8 @@ export interface RunStartIdentity { rootThreadId: string launchedBy: RunLaunchSource | null correlationId: string | null + traceparent?: string + tracestate?: string } /** The run and last sequence a fork copied its history from. */ @@ -117,6 +119,8 @@ export const RunStateSchema = Schema.Struct({ correlationId: Schema.NullOr(Schema.String).pipe( Schema.withDecodingDefaultKey(Effect.succeed(null)), ), + traceparent: Schema.optional(Schema.String), + tracestate: Schema.optional(Schema.String), }), ).pipe(Schema.withDecodingDefaultKey(Effect.succeed(null))), forkedFrom: Schema.NullOr(Schema.Struct({ runId: Schema.String, seq: Schema.Finite })).pipe( diff --git a/packages/core/src/trace.ts b/packages/core/src/trace.ts new file mode 100644 index 0000000..e6778fb --- /dev/null +++ b/packages/core/src/trace.ts @@ -0,0 +1,136 @@ +import { Predicate } from 'effect' + +const W3C_TRACEPARENT_REGEX = /^([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/i + +const ALL_ZEROS_32 = '00000000000000000000000000000000' +const ALL_ZEROS_16 = '0000000000000000' + +export interface TraceContext { + readonly traceparent: string + readonly tracestate?: string +} + +export interface ParsedTraceparent { + readonly version: string + readonly traceId: string + readonly spanId: string + readonly flags: string + readonly sampled: boolean +} + +export interface CreateTraceparentOptions { + readonly traceId?: string + readonly spanId?: string + readonly sampled?: boolean +} + +function randomHex(charCount: number): string { + // Synchronous entropy for W3C trace IDs without an Effect runtime. + // oxlint-disable-next-line effecttsgo/crypto-random-uuid + const entropy = globalThis.crypto.randomUUID().replaceAll('-', '') + + if (entropy.length >= charCount) { + return entropy.slice(0, charCount) + } + + // oxlint-disable-next-line effecttsgo/crypto-random-uuid + return (entropy + globalThis.crypto.randomUUID().replaceAll('-', '')).slice(0, charCount) +} + +/** Generates a 16-byte (32-character) random hex trace id. */ +export function generateTraceId(): string { + const id = randomHex(32) + return id === ALL_ZEROS_32 ? generateTraceId() : id +} + +/** Generates an 8-byte (16-character) random hex span id. */ +export function generateSpanId(): string { + const id = randomHex(16) + return id === ALL_ZEROS_16 ? generateSpanId() : id +} + +/** Validates whether `value` is a compliant W3C traceparent header. */ +export function isValidTraceparent(value: string | null | undefined): value is string { + if (!Predicate.isString(value)) { + return false + } + + const match = W3C_TRACEPARENT_REGEX.exec(value.trim()) + + if (!match) { + return false + } + + const version = match[1] + const traceId = match[2] + const spanId = match[3] + + if (version !== '00' || traceId === ALL_ZEROS_32 || spanId === ALL_ZEROS_16) { + return false + } + + return true +} + +/** Parses a W3C traceparent into its component identifiers and sampling flag. */ +export function parseTraceparent(value: string | null | undefined): ParsedTraceparent | null { + if (!Predicate.isString(value)) { + return null + } + + const trimmed = value.trim() + const match = W3C_TRACEPARENT_REGEX.exec(trimmed) + + if (!match) { + return null + } + + const version = match[1] + const traceId = match[2] + const spanId = match[3] + const flags = match[4] + + if (!version || !traceId || !spanId || !flags) { + return null + } + + if (version !== '00' || traceId === ALL_ZEROS_32 || spanId === ALL_ZEROS_16) { + return null + } + + const flagsNum = parseInt(flags, 16) + const sampled = (flagsNum & 1) === 1 + + return { + version, + traceId: traceId.toLowerCase(), + spanId: spanId.toLowerCase(), + flags: flags.toLowerCase(), + sampled, + } +} + +/** Creates a W3C traceparent string conforming to version 00. */ +export function createTraceparent(options?: CreateTraceparentOptions): string { + const traceId = options?.traceId ?? generateTraceId() + const spanId = options?.spanId ?? generateSpanId() + const flags = options?.sampled === false ? '00' : '01' + + return `00-${traceId}-${spanId}-${flags}` +} + +/** + * Creates a child traceparent preserving the parent trace ID and sampling flag, + * but assigning a fresh span ID for the child hop (run or effect). + */ +export function childTraceparent(parent: string | null | undefined, childSpanId?: string): string { + const parsed = parseTraceparent(parent) + + if (!parsed) { + return createTraceparent({ spanId: childSpanId }) + } + + const spanId = childSpanId ?? generateSpanId() + + return `00-${parsed.traceId}-${spanId}-${parsed.flags}` +} diff --git a/packages/otel/package.json b/packages/otel/package.json index 18d015d..f9af034 100644 --- a/packages/otel/package.json +++ b/packages/otel/package.json @@ -20,6 +20,7 @@ "test": "bun test src" }, "dependencies": { + "@looms/core": "workspace:*", "@looms/projectors": "workspace:*", "@looms/runtime": "workspace:*" }, diff --git a/packages/otel/src/index.test.ts b/packages/otel/src/index.test.ts index 6e9a1db..f7580c7 100644 --- a/packages/otel/src/index.test.ts +++ b/packages/otel/src/index.test.ts @@ -1,9 +1,58 @@ import { expect, test } from 'bun:test' -import { createOpenTelemetryObservers } from './index' +import { + childTraceparent, + createOpenTelemetryObservers, + createTraceparent, + extractTraceCarrier, + injectTraceCarrier, + isValidTraceparent, + parseTraceparent, +} from './index' test('requires an explicitly supplied OpenTelemetry meter', () => { expect(() => createOpenTelemetryObservers({})).toThrow( 'An OpenTelemetry meter or meterProvider is required', ) }) + +test('creates and parses valid W3C traceparents', () => { + const tp = createTraceparent() + expect(isValidTraceparent(tp)).toBe(true) + + const parsed = parseTraceparent(tp) + expect(parsed).not.toBeNull() + expect(parsed?.version).toBe('00') + expect(parsed?.traceId).toHaveLength(32) + expect(parsed?.spanId).toHaveLength(16) + expect(parsed?.sampled).toBe(true) +}) + +test('childTraceparent keeps parent traceId and updates spanId', () => { + const parent = createTraceparent() + const parentParsed = parseTraceparent(parent)! + const child = childTraceparent(parent) + const childParsed = parseTraceparent(child)! + + expect(childParsed.traceId).toBe(parentParsed.traceId) + expect(childParsed.spanId).not.toBe(parentParsed.spanId) + expect(childParsed.flags).toBe(parentParsed.flags) +}) + +test('injectTraceCarrier and extractTraceCarrier round-trip', () => { + const tp = createTraceparent() + const ts = 'congo=t61rcWkgMzE' + const carrier = injectTraceCarrier(tp, ts) + + expect(carrier.traceparent).toBe(tp) + expect(carrier.tracestate).toBe(ts) + + const extracted = extractTraceCarrier(carrier) + expect(extracted?.traceparent).toBe(tp) + expect(extracted?.tracestate).toBe(ts) + + const headers = new Headers({ traceparent: tp, tracestate: ts }) + const fromHeaders = extractTraceCarrier(headers) + expect(fromHeaders?.traceparent).toBe(tp) + expect(fromHeaders?.tracestate).toBe(ts) +}) diff --git a/packages/otel/src/index.ts b/packages/otel/src/index.ts index aaa3f10..57fb024 100644 --- a/packages/otel/src/index.ts +++ b/packages/otel/src/index.ts @@ -1,4 +1,5 @@ import type { Attributes, Meter, MeterProvider } from '@opentelemetry/api' +import { Predicate } from 'effect' import type { ProjectorObservation, ProjectorObserver } from '@looms/projectors' import type { RuntimeObservation, RuntimeObserver } from '@looms/runtime' @@ -70,3 +71,90 @@ export function createOpenTelemetryObservers( return { runtimeObserver, projectorObserver } } + +export { + childTraceparent, + createTraceparent, + generateSpanId, + generateTraceId, + isValidTraceparent, + parseTraceparent, + type CreateTraceparentOptions, + type ParsedTraceparent, + type TraceContext, +} from '@looms/core' + +export interface TraceCarrier { + traceparent?: string + tracestate?: string +} + +/** Injects active or provided W3C trace context into a plain carrier object. */ +export function injectTraceCarrier(traceparent?: string, tracestate?: string): TraceCarrier { + const carrier: TraceCarrier = {} + + if (traceparent) { + carrier.traceparent = traceparent + } + + if (tracestate) { + carrier.tracestate = tracestate + } + + return carrier +} + +export type TraceCarrierInput = + | Headers + | Request + | TraceCarrier + | { readonly [key: string]: string | undefined } + | null + | undefined + +/** Extracts traceparent and tracestate from a carrier, headers object, or request. */ +export function extractTraceCarrier(carrier: TraceCarrierInput): TraceCarrier | null { + if (!carrier) { + return null + } + + if (carrier instanceof Headers) { + const traceparent = carrier.get('traceparent') + const tracestate = carrier.get('tracestate') + + if (!traceparent) { + return null + } + + return { + traceparent, + tracestate: tracestate ?? undefined, + } + } + + if (carrier instanceof Request) { + const traceparent = carrier.headers.get('traceparent') + const tracestate = carrier.headers.get('tracestate') + + if (!traceparent) { + return null + } + + return { + traceparent, + tracestate: tracestate ?? undefined, + } + } + + const traceparent = 'traceparent' in carrier ? carrier.traceparent : undefined + const tracestate = 'tracestate' in carrier ? carrier.tracestate : undefined + + if (Predicate.isString(traceparent) && traceparent.length > 0) { + return { + traceparent, + tracestate: Predicate.isString(tracestate) ? tracestate : undefined, + } + } + + return null +} diff --git a/packages/runtime/src/cancellation.ts b/packages/runtime/src/cancellation.ts index e16d237..19b57ae 100644 --- a/packages/runtime/src/cancellation.ts +++ b/packages/runtime/src/cancellation.ts @@ -189,7 +189,17 @@ export function createCancellation(options: CancellationOptions): LoomsRuntime[' ({ item, execution }) => Effect.tryPromise({ try: () => - Promise.resolve(worker.cancel?.(workerTask(runId, item, execution.attempt))), + Promise.resolve( + worker.cancel?.( + workerTask( + runId, + item, + execution.attempt, + committed.cursor.state.startIdentity?.traceparent, + committed.cursor.state.startIdentity?.tracestate, + ), + ), + ), catch: (cause) => new RuntimeExecutionError(cause instanceof Error ? cause.message : String(cause)), }).pipe(Effect.ignore), diff --git a/packages/runtime/src/helpers.ts b/packages/runtime/src/helpers.ts index c3d4961..a79c365 100644 --- a/packages/runtime/src/helpers.ts +++ b/packages/runtime/src/helpers.ts @@ -35,14 +35,26 @@ export function workerTask( runId: string, item: OutstandingEffect, attempt: number, + traceparent?: string | null, + tracestate?: string | null, ): EffectWorkerTask { - return { + const base: EffectWorkerTask = { runId, effectId: item.effectId, attempt, type: item.effect.type, input: 'input' in item.effect ? item.effect.input : null, } + + if (traceparent) { + return { + ...base, + traceparent, + tracestate: tracestate ?? undefined, + } + } + + return base } export class UnknownDefinitionError extends Data.TaggedError('UnknownDefinitionError')<{ diff --git a/packages/runtime/src/launch-provenance.test.ts b/packages/runtime/src/launch-provenance.test.ts index e2b4cb8..2083142 100644 --- a/packages/runtime/src/launch-provenance.test.ts +++ b/packages/runtime/src/launch-provenance.test.ts @@ -175,4 +175,62 @@ describe('launch provenance', () => { await looms.stop() } }) + + test('propagates traceparent from start across effects, worker tasks, and child runs', async () => { + let capturedEffectContext: { traceparent?: string; tracestate?: string } | undefined + + const tracingEffect = defineEffect({ + type: 'relay.trace-check', + execute: (_input, ctx) => { + capturedEffectContext = { + traceparent: ctx.traceparent, + tracestate: ctx.tracestate, + } + + return [] + }, + }) + + const tracingThread = defineThread({ + kind: 'relay', + initialState: () => ({}), + step: (state) => state, + effects: () => [invoke(tracingEffect, {}, 'trace-step'), complete({ ok: true })], + }) + + const tracingModule = defineRuntimeModule({ + namespace: 'tracing', + protocolVersion: '1.0.0', + effects: { tracingEffect }, + threads: { relay: tracingThread }, + definitions: [{ kind: 'relay', name: 'tracing-relay', version: 'v1' }], + }) + + const looms = createLooms({ modules: [tracingModule] }) + + try { + const traceparent = '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01' + const tracestate = 'congo=t61rcWkgMzE' + + const started = await looms.start( + { kind: 'relay', name: 'tracing-relay', version: 'v1' }, + {}, + { traceparent, tracestate }, + ) + + const run = await looms.getRun(started.runId) + expect(run.startIdentity?.traceparent).toBe(traceparent) + expect(run.startIdentity?.tracestate).toBe(tracestate) + + expect(capturedEffectContext?.traceparent).toBeDefined() + + expect( + capturedEffectContext?.traceparent?.startsWith('00-4bf92f3577b34da6a3ce929d0e0e4736-'), + ).toBe(true) + + expect(capturedEffectContext?.tracestate).toBe(tracestate) + } finally { + await looms.stop() + } + }) }) diff --git a/packages/runtime/src/local-effect-executor.ts b/packages/runtime/src/local-effect-executor.ts index e611713..43cb3d9 100644 --- a/packages/runtime/src/local-effect-executor.ts +++ b/packages/runtime/src/local-effect-executor.ts @@ -1,6 +1,7 @@ import { Effect, Option, Predicate, type Context, type ManagedRuntime } from 'effect' import { + childTraceparent, createEvent, createRunId, DEFAULT_DEFINITION_VERSION, @@ -43,6 +44,8 @@ export interface ExecuteLocalEffectsInput { readonly runId: string /** Inherited by runs the effects launch. */ readonly correlationId: string + readonly traceparent?: string | null + readonly tracestate?: string | null readonly outstanding: ReadonlyArray readonly attempts: ReadonlyMap readonly now: number @@ -58,6 +61,8 @@ function launcherFor( correlationId: string, item: OutstandingEffect, launched: EventEnvelope[], + traceparent?: string | null, + tracestate?: string | null, ): RunLauncher { const launchedBy: RunLaunchSource = { runId, @@ -71,28 +76,47 @@ function launcherFor( launch: (args) => { const targetRunId = args.runId ?? createRunId() - return launcher.launch({ ...args, runId: targetRunId, launchedBy, correlationId }).pipe( - Effect.tap(() => - Effect.sync(() => { - launched.push( - createEvent(runId, { - type: 'runtime.run.launched', - payload: { - runId: targetRunId, - kind: args.kind, - definitionName: args.definitionName, - definitionVersion: args.definitionVersion ?? DEFAULT_DEFINITION_VERSION, - effectId: item.effectId, - }, - threadId: item.threadId, - causationId: item.causingEventId, - correlationId, - origin: { type: 'thread', threadId: item.threadId }, - }), - ) - }), - ), - ) + const childTrace = + args.traceparent ?? (traceparent ? childTraceparent(traceparent) : undefined) + + const childState = args.tracestate ?? tracestate ?? undefined + + return launcher + .launch({ + ...args, + runId: targetRunId, + launchedBy, + correlationId, + traceparent: childTrace, + tracestate: childState, + }) + .pipe( + Effect.tap(() => + Effect.sync(() => { + const base = { + runId: targetRunId, + kind: args.kind, + definitionName: args.definitionName, + definitionVersion: args.definitionVersion ?? DEFAULT_DEFINITION_VERSION, + effectId: item.effectId, + } + + const withTrace = childTrace ? { ...base, traceparent: childTrace } : base + const payload = childState ? { ...withTrace, tracestate: childState } : withTrace + + launched.push( + createEvent(runId, { + type: 'runtime.run.launched', + payload, + threadId: item.threadId, + causationId: item.causingEventId, + correlationId, + origin: { type: 'thread', threadId: item.threadId }, + }), + ) + }), + ), + ) }, status: launcher.status, } @@ -179,11 +203,23 @@ export function executeLocalEffects( const retry = registry.effects.get(item.effect.type)?.retry const launched: EventEnvelope[] = [] + const effectTraceparent = input.traceparent + ? childTraceparent(input.traceparent) + : undefined + const dispatched = dispatchEffect( registry, services, definitions, - launcherFor(launcher, input.runId, input.correlationId, item, launched), + launcherFor( + launcher, + input.runId, + input.correlationId, + item, + launched, + input.traceparent, + input.tracestate, + ), item.effect, { effectId: item.effectId, @@ -191,6 +227,8 @@ export function executeLocalEffects( threadId: item.threadId, causingEventId: item.causingEventId, signal: controller.signal, + traceparent: effectTraceparent, + tracestate: input.tracestate ?? undefined, emit: appendLive, }, ).pipe( diff --git a/packages/runtime/src/looms.ts b/packages/runtime/src/looms.ts index a1fc06d..550e26a 100644 --- a/packages/runtime/src/looms.ts +++ b/packages/runtime/src/looms.ts @@ -95,7 +95,13 @@ export interface Looms { start( definition: TDef, input?: DefinitionInput, - options?: { runId?: string; threadId?: string; idempotencyKey?: string }, + options?: { + runId?: string + threadId?: string + idempotencyKey?: string + traceparent?: string + tracestate?: string + }, ): Promise /** Start a run by `{ kind, definitionName }` when you only have names (HTTP bodies, CLIs). */ startRun(args: { @@ -106,6 +112,8 @@ export interface Looms { runId?: string threadId?: string idempotencyKey?: string + traceparent?: string + tracestate?: string }): Promise /** Commit a run's start and schedule a wake without waiting for it. */ launchRun(args: { @@ -116,6 +124,8 @@ export interface Looms { runId?: string threadId?: string idempotencyKey?: string + traceparent?: string + tracestate?: string }): Promise getRun(runId: string): Promise getEvents(runId: string, options?: { fromSeq?: number; limit?: number }): Promise @@ -262,6 +272,8 @@ export function createLooms(options: CreateLoomsOptions = {}): Looms { runId: startOpts?.runId, threadId: startOpts?.threadId, idempotencyKey: startOpts?.idempotencyKey, + traceparent: startOpts?.traceparent, + tracestate: startOpts?.tracestate, }), ), getRun: (runId) => runEffect((i) => i.runtime.getRun(runId)), diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index ac0b33b..ab103f0 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -257,13 +257,15 @@ export function createRuntime = new Set([ @@ -63,6 +67,7 @@ export function isWorkerEffect(input: { readonly execution: EffectExecutionRecord | undefined readonly registry: ComposedRegistry readonly worker?: EffectWorker + readonly state?: RunState }): boolean { const { item, execution, worker } = input @@ -84,7 +89,12 @@ export function isWorkerEffect(input: { return definition.execution === 'worker' } - const claim = worker ? workerClaims(worker, input.runId, item) : undefined + const traceparent = input.state?.startIdentity?.traceparent + const tracestate = input.state?.startIdentity?.tracestate + + const claim = worker + ? workerClaims(worker, input.runId, item, traceparent, tracestate) + : undefined if (claim) { return true @@ -147,7 +157,13 @@ export function planWorkerEffects(input: { causationId: item.causingEventId, origin: { type: 'system' }, }), - task: workerTask(input.runId, item, attempt), + task: workerTask( + input.runId, + item, + attempt, + input.state.startIdentity?.traceparent, + input.state.startIdentity?.tracestate, + ), scheduleToStartDeadlineAt: retry?.scheduleToStartTimeoutMs ? input.now + retry.scheduleToStartTimeoutMs : null, From 774d17f02cf00fb703c231a85601bfa06fa6a485 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 13:43:56 +0200 Subject: [PATCH 10/23] fix(core): set run trace context without conditional spreads The fold assigns traceparent and tracestate after building the start identity. The runtime no longer copies trace context into the requested identity, because the retry check never compares it. Co-authored-by: Cursor --- packages/core/src/fold.ts | 18 +++++++++++++++--- packages/runtime/src/runtime.ts | 2 -- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/core/src/fold.ts b/packages/core/src/fold.ts index fc6fa6d..925e964 100644 --- a/packages/core/src/fold.ts +++ b/packages/core/src/fold.ts @@ -5,7 +5,13 @@ import type { EventEnvelope } from './envelope' import { createEffectId } from './ids' import { readRunLaunchSource } from './launcher' import { DEFAULT_DEFINITION_VERSION } from './module' -import type { ThreadRecord, OutstandingEffect, RunState, WaitRecord } from './state' +import type { + ThreadRecord, + OutstandingEffect, + RunStartIdentity, + RunState, + WaitRecord, +} from './state' import { completedRunStatus, emptyRunState } from './state' import { isTerminalStatus, type ThreadContext, type ThreadDefinition } from './thread' import type { JsonValue } from './types' @@ -130,11 +136,17 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi rootThreadId, launchedBy: readRunLaunchSource(payload.launchedBy), correlationId: event.correlationId ?? null, - ...(traceparent ? { traceparent } : {}), - ...(tracestate ? { tracestate } : {}), } : null + if (startIdentity && traceparent) { + startIdentity.traceparent = traceparent + } + + if (startIdentity && tracestate) { + startIdentity.tracestate = tracestate + } + return { ...state, runId: event.runId, diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index ab103f0..c0c818f 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -257,8 +257,6 @@ export function createRuntime Date: Fri, 23 Oct 2026 13:51:23 +0200 Subject: [PATCH 11/23] fix(runtime): authorize run routes from the start event instead of a full fold Run attributes are immutable and recorded on runtime.run.started, so authorization reads seq 1 and only falls back to loading the run when the log is trimmed. Store failures now fail the request instead of authorizing without attributes. Also fixes the start-body attribute decode, which did not type-check. Co-authored-by: Cursor --- packages/runtime/src/run-context.test.ts | 42 ++++++++++++++ packages/runtime/src/server/fetch-handler.ts | 59 ++++++++++++++++---- 2 files changed, 89 insertions(+), 12 deletions(-) diff --git a/packages/runtime/src/run-context.test.ts b/packages/runtime/src/run-context.test.ts index 2fb87db..37ec4a6 100644 --- a/packages/runtime/src/run-context.test.ts +++ b/packages/runtime/src/run-context.test.ts @@ -10,6 +10,8 @@ import { defineThread, hasEntries, invoke, + makeMemoryEventStore, + type EventStore, } from '@looms/core' import { SecretContextConflictError, StartRunConflictError } from './errors' @@ -223,6 +225,46 @@ describe('run attributes and secret context', () => { await looms.stop() }) + test('authorization reads only the start event to find attributes', async () => { + const memory = await Effect.runPromise(makeMemoryEventStore) + let fullReads = 0 + + const store: EventStore = { + ...memory, + read: (runId, options) => { + if (options?.limit !== 1) { + fullReads += 1 + } + + return memory.read(runId, options) + }, + } + + const readsAtDecision: number[] = [] + + const looms = createLooms({ + modules: [contextModule], + store, + authorize: (_request, route) => + Effect.sync(() => { + if (route.name === 'runs.get') { + readsAtDecision.push(fullReads) + } + + return route.attributes?.tenant === 'acme' ? authAllowed : authDenied(403, 'Forbidden') + }), + }) + + await looms.start(context, null, { runId: 'run_cheap_auth', attributes: { tenant: 'acme' } }) + fullReads = 0 + + const read = await looms.fetch(new Request('http://looms.test/runs/run_cheap_auth')) + + expect(read?.status).toBe(200) + expect(readsAtDecision).toEqual([0]) + await looms.stop() + }) + test('a launched run and a fork inherit attributes and secrets without logging them', async () => { const looms = createLooms({ modules: [contextModule] }) diff --git a/packages/runtime/src/server/fetch-handler.ts b/packages/runtime/src/server/fetch-handler.ts index c24a65b..924445a 100644 --- a/packages/runtime/src/server/fetch-handler.ts +++ b/packages/runtime/src/server/fetch-handler.ts @@ -3,6 +3,8 @@ import { Data, Effect, Option, Predicate, Schema } from 'effect' import { EventStoreTag, hasEntries, + isJsonObject, + readAttributes, RunAttributesSchema, type EventStore, type RunAttributes, @@ -47,16 +49,50 @@ class UnreadableStartBody extends Data.TaggedError('UnreadableStartBody')<{ readonly cause: unknown }> {} +const noAttributes: RunAttributes = {} + function readRequestAttributes(req: Request): Effect.Effect { return Effect.tryPromise({ try: () => req.clone().json(), catch: (cause) => new UnreadableStartBody({ cause }), }).pipe( - Effect.map((body) => { - const decoded = Schema.decodeUnknownOption(StartAttributesSchema)(body) - return Option.getOrElse(decoded, () => ({})).attributes ?? {} + Effect.map((body) => + Option.match(Schema.decodeUnknownOption(StartAttributesSchema)(body), { + onNone: () => noAttributes, + onSome: (decoded) => decoded.attributes ?? noAttributes, + }), + ), + Effect.catchTag('UnreadableStartBody', () => Effect.succeed(noAttributes)), + ) +} + +/** + * Attributes are immutable and recorded on `runtime.run.started`, which is seq 1 of every + * untrimmed log, forks included. Only a trimmed log needs the snapshot-backed fold. + */ +function readStoredAttributes( + runtime: LoomsRuntime, + store: EventStore, + runId: string, +): Effect.Effect { + const folded = runtime.getRun(runId).pipe( + Effect.provideService(EventStoreTag, store), + Effect.map((state) => state.attributes), + ) + + return store.read(runId, { fromSeq: 1, limit: 1 }).pipe( + Effect.flatMap(([first]) => { + if (first === undefined) { + return Effect.succeed(noAttributes) + } + + if (first.type === 'runtime.run.started' && isJsonObject(first.payload)) { + return Effect.succeed(readAttributes(first.payload.attributes)) + } + + return folded }), - Effect.catchTag('UnreadableStartBody', () => Effect.succeed({})), + Effect.catch(() => folded), ) } @@ -66,7 +102,7 @@ function authorizeRoute( authorize: Authorize, runtime: LoomsRuntime, store: EventStore, -): Effect.Effect { +): Effect.Effect { const { access, name } = matched.route if (access === 'public') { @@ -77,14 +113,13 @@ function authorizeRoute( return Effect.gen(function* () { const stored = Predicate.isString(runId) - ? yield* runtime.getRun(runId).pipe( - Effect.provideService(EventStoreTag, store), - Effect.orElseSucceed(() => undefined), - ) - : undefined + ? yield* readStoredAttributes(runtime, store, runId) + : noAttributes + + const requested = + matched.route.name === 'runs.start' ? yield* readRequestAttributes(req) : noAttributes - const requested = matched.route.name === 'runs.start' ? yield* readRequestAttributes(req) : {} - const attributes = stored && hasEntries(stored.attributes) ? stored.attributes : requested + const attributes = hasEntries(stored) ? stored : requested const route = Predicate.isString(runId) ? hasEntries(attributes) From 95da24b8e9aa5c589b1ac41e8cfcc8aca36ebb6f Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 13:52:06 +0200 Subject: [PATCH 12/23] style: format definition resolver sources Co-authored-by: Cursor --- packages/core/src/definition-resolver.ts | 2 +- packages/core/src/definition-store.ts | 12 +++++++----- packages/runtime/src/types.ts | 20 ++++---------------- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/packages/core/src/definition-resolver.ts b/packages/core/src/definition-resolver.ts index c10e9d9..5c46b08 100644 --- a/packages/core/src/definition-resolver.ts +++ b/packages/core/src/definition-resolver.ts @@ -2,8 +2,8 @@ import { Data, Effect, Predicate } from 'effect' import type { VersionedDefinition } from './definition-store' import type { EventEnvelope } from './envelope' -import type { JsonValue } from './types' import { DEFAULT_DEFINITION_VERSION, definitionKey } from './module' +import type { JsonValue } from './types' export class DefinitionResolveError extends Data.TaggedError('DefinitionResolveError')<{ readonly kind: string diff --git a/packages/core/src/definition-store.ts b/packages/core/src/definition-store.ts index dae99ee..e617e75 100644 --- a/packages/core/src/definition-store.ts +++ b/packages/core/src/definition-store.ts @@ -69,11 +69,13 @@ export function makeDefinitionStore( return Effect.succeed(undefined) } - return resolver.resolve(kind, name, version).pipe( - Effect.map((found) => - found && isStoredDefinition(kind, found) ? found : undefined, - ), - ) + return resolver + .resolve(kind, name, version) + .pipe( + Effect.map((found) => + found && isStoredDefinition(kind, found) ? found : undefined, + ), + ) }, } } diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 092a0ff..d643b42 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -172,10 +172,7 @@ export interface LoomsRuntime< runId: string, ): Effect.Effect< RunState, - | EventStoreTruncationError - | EventStoreTrimmedError - | EventStoreError - | DefinitionResolveError, + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError, EventStoreTag > getEvents( @@ -194,10 +191,7 @@ export interface LoomsRuntime< runId: string, ): Effect.Effect< RunOperationalStatus, - | EventStoreError - | EventStoreTrimmedError - | EventStoreTruncationError - | DefinitionResolveError, + EventStoreError | EventStoreTrimmedError | EventStoreTruncationError | DefinitionResolveError, EventStoreTag > retryEffect( @@ -235,19 +229,13 @@ export interface LoomsRuntime< > readonly rescanTimers: Effect.Effect< number, - | EventStoreTruncationError - | EventStoreTrimmedError - | EventStoreError - | DefinitionResolveError, + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError, EventStoreTag > /** Reschedule every durable timer and effect deadline after restart or repair. */ readonly recoverDeadlines: Effect.Effect< number, - | EventStoreTruncationError - | EventStoreTrimmedError - | EventStoreError - | DefinitionResolveError, + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError, EventStoreTag > readonly dispose: Effect.Effect From 6ebb530eb0731401dc2fe9afbeac22d942ab8ae7 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 13:52:47 +0200 Subject: [PATCH 13/23] style: format and lint websocket transport sources Co-authored-by: Cursor --- docs/protocol.md | 22 +++++----- packages/client/src/client.test.ts | 5 ++- packages/client/src/client.ts | 4 +- packages/cloudflare/src/durable-object.ts | 7 +++- packages/cloudflare/src/websocket.test.ts | 16 ++++--- packages/cloudflare/src/websocket.ts | 11 ++--- packages/core/src/subscription.test.ts | 51 ++++++++++++++++------- packages/core/src/subscription.ts | 10 +++-- packages/react/src/store.test.ts | 5 ++- packages/react/src/store.ts | 2 +- packages/runtime/src/sse.ts | 2 +- 11 files changed, 86 insertions(+), 49 deletions(-) diff --git a/docs/protocol.md b/docs/protocol.md index aef6a81..582915e 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -26,18 +26,18 @@ Matching is by event `type` plus an optional payload subset (`match: { approvalI ## HTTP -| Method | Path | Purpose | -| ------ | ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `GET` | `/health` | Liveness | -| `POST` | `/runs` | Start `{ kind, definitionName, input }` (`Accept: text/event-stream` or `?stream=true` streams the log live) | -| `GET` | `/runs` | List run ids (`createLooms` / shared store). Actor hosts (`createLocalActorHost`, Durable Objects, celld) return `501` — use a projector lake or client-side history instead | -| `GET` | `/runs/:id` | Current run state | +| Method | Path | Purpose | +| ------ | ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `GET` | `/health` | Liveness | +| `POST` | `/runs` | Start `{ kind, definitionName, input }` (`Accept: text/event-stream` or `?stream=true` streams the log live) | +| `GET` | `/runs` | List run ids (`createLooms` / shared store). Actor hosts (`createLocalActorHost`, Durable Objects, celld) return `501` — use a projector lake or client-side history instead | +| `GET` | `/runs/:id` | Current run state | | `GET` | `/runs/:id/events` | Event log (`?fromSeq=&limit=`). SSE with `live=true` / `Accept: text/event-stream`, or a WebSocket with `Upgrade: websocket` on a Durable Object. Resumes after `Last-Event-ID` or `fromSeq`. `threadId` and `includeEphemeral=false` filter the stream. | -| `POST` | `/runs/:id/events` | Signal the run | -| `GET` | `/runs/:id/threads` | Child threads | -| `POST` | `/runs/:id/wake` | Resume processing | -| `GET` | `/runs/:id/replay?seq=` | State before / after an event | -| `GET` | `/runs/:id/projections/:name` | Named read model | +| `POST` | `/runs/:id/events` | Signal the run | +| `GET` | `/runs/:id/threads` | Child threads | +| `POST` | `/runs/:id/wake` | Resume processing | +| `GET` | `/runs/:id/replay?seq=` | State before / after an event | +| `GET` | `/runs/:id/projections/:name` | Named read model | `looms.fetch(request)` handles these paths. It returns `null` for everything else so you can mount Looms next to your own UI. diff --git a/packages/client/src/client.test.ts b/packages/client/src/client.test.ts index c407f7b..525f497 100644 --- a/packages/client/src/client.test.ts +++ b/packages/client/src/client.test.ts @@ -237,7 +237,10 @@ describe('createLoomsClient', () => { class FakeSocket { static latest: FakeSocket | undefined readonly url: string - private readonly listeners = new Map void>>() + private readonly listeners = new Map< + string, + Array<(event: { readonly data?: unknown }) => void> + >() constructor(url: string) { this.url = url diff --git a/packages/client/src/client.ts b/packages/client/src/client.ts index 211a235..f245941 100644 --- a/packages/client/src/client.ts +++ b/packages/client/src/client.ts @@ -450,7 +450,9 @@ function globalWebSocket(): SubscriptionSocketConstructor | undefined { return globalThis.WebSocket } -function consumeWebSocket(subscription: WebSocketSubscription): Effect.Effect { +function consumeWebSocket( + subscription: WebSocketSubscription, +): Effect.Effect { const Socket = subscription.settings.webSocket ?? globalWebSocket() if (!Socket) { diff --git a/packages/cloudflare/src/durable-object.ts b/packages/cloudflare/src/durable-object.ts index 7f64add..916b0c0 100644 --- a/packages/cloudflare/src/durable-object.ts +++ b/packages/cloudflare/src/durable-object.ts @@ -13,7 +13,12 @@ import { type RunStatusSnapshot, } from '@looms/core' import type { Projector } from '@looms/projectors' -import { defaultAuthorize, type Authorize, type StartResult, type StartRunArgs } from '@looms/runtime' +import { + defaultAuthorize, + type Authorize, + type StartResult, + type StartRunArgs, +} from '@looms/runtime' import { durableObjectAlarms } from './alarm-scheduler' import { durableObjectEventStore, type DurableObjectEventStore } from './event-store' diff --git a/packages/cloudflare/src/websocket.test.ts b/packages/cloudflare/src/websocket.test.ts index 167df24..f5e17fd 100644 --- a/packages/cloudflare/src/websocket.test.ts +++ b/packages/cloudflare/src/websocket.test.ts @@ -2,7 +2,12 @@ import { describe, expect, test } from 'bun:test' import { Effect } from 'effect' -import { createEvent, makeMemoryEventStore, type EventStore, type SubscriptionAttachment } from '@looms/core' +import { + createEvent, + makeMemoryEventStore, + type EventStore, + type SubscriptionAttachment, +} from '@looms/core' import { attachSocketFanout, openRunSocket, type HibernatingSocketState } from './websocket' @@ -62,14 +67,7 @@ async function seededStore(): Promise { const store = await Effect.runPromise(makeMemoryEventStore) await Effect.runPromise( - store.append( - 'run_1', - [ - threadEvent('thr_a'), - threadEvent('thr_b'), - ], - { expectedTail: 0 }, - ), + store.append('run_1', [threadEvent('thr_a'), threadEvent('thr_b')], { expectedTail: 0 }), ) return store diff --git a/packages/cloudflare/src/websocket.ts b/packages/cloudflare/src/websocket.ts index 2736c3e..e3746d8 100644 --- a/packages/cloudflare/src/websocket.ts +++ b/packages/cloudflare/src/websocket.ts @@ -117,9 +117,11 @@ export function openRunSocket(options: OpenRunSocketOptions): Effect.Effect { const batch = decodeSubscriptionFrame('{"batch":[{"type":"runtime.run.started"}]}') expect(batch?.type).toBe('batch') - expect(encodeSubscriptionFrame({ type: 'batch', batch: [{ type: 'runtime.run.started' }] })).toBe( - '{"batch":[{"type":"runtime.run.started"}]}', - ) + expect( + encodeSubscriptionFrame({ type: 'batch', batch: [{ type: 'runtime.run.started' }] }), + ).toBe('{"batch":[{"type":"runtime.run.started"}]}') expect(decodeSubscriptionFrame('{"type":"resync","head":3,"reason":"trimmed"}')).toEqual({ type: 'resync', @@ -93,7 +93,9 @@ describe('subscription protocol', () => { }) test('cursor and filter follow the query string', () => { - const url = new URL('http://looms.test/runs/run_1/events?fromSeq=4&threadId=thr_a&includeEphemeral=0') + const url = new URL( + 'http://looms.test/runs/run_1/events?fromSeq=4&threadId=thr_a&includeEphemeral=0', + ) expect(subscriptionFromSeq(url, null)).toEqual({ ok: true, fromSeq: 4 }) expect(subscriptionFromSeq(url, '2')).toEqual({ ok: true, fromSeq: 3 }) @@ -114,11 +116,7 @@ describe('subscription protocol', () => { const next = openSubscription(socket, socket.attachment()!, { kind: 'open', hello: { type: 'hello', head: 1, tail: 3 }, - events: [ - envelope(1, 'thr_a'), - envelope(2, 'thr_b'), - envelope(3, 'thr_a', true), - ], + events: [envelope(1, 'thr_a'), envelope(2, 'thr_b'), envelope(3, 'thr_a', true)], }) expect(socket.sent.map((line) => JSON.parse(line))).toEqual([ @@ -131,7 +129,9 @@ describe('subscription protocol', () => { }) test('a trimmed cursor closes with resync', () => { - const socket = new FakeSocket(attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, 'ada')) + const socket = new FakeSocket( + attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, 'ada'), + ) const next = openSubscription(socket, socket.attachment()!, { kind: 'resync', head: 8 }) @@ -147,14 +147,26 @@ describe('subscription protocol', () => { }) test('backpressure closes the socket and a later append does not write to it', () => { - const slow = new FakeSocket(attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, null)) + const slow = new FakeSocket( + attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, null), + ) + slow.bufferedAmount = 32 - const live = new FakeSocket(attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, null)) + + const live = new FakeSocket( + attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, null), + ) fanoutSubscription([slow, live], [envelope(1, 'thr_a')], { bufferLimitBytes: 8, head: 4 }) expect(slow.closed?.reason).toBe('backpressure') - expect(JSON.parse(slow.sent[0] ?? '{}')).toMatchObject({ type: 'resync', reason: 'backpressure', head: 4 }) + + expect(JSON.parse(slow.sent[0] ?? '{}')).toMatchObject({ + type: 'resync', + reason: 'backpressure', + head: 4, + }) + expect(live.closed).toBeUndefined() expect(live.attachment()?.cursor).toBe(1) @@ -190,9 +202,18 @@ describe('subscription protocol', () => { }) test('push does not resend a seq the attachment already passed', () => { - const attachment = { ...attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, null), cursor: 1 } + const attachment = { + ...attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, null), + cursor: 1, + } + const socket = new FakeSocket(attachment) - const next = pushSubscriptionEvents(socket, [envelope(1, 'thr_a'), envelope(2, 'thr_a')], attachment) + + const next = pushSubscriptionEvents( + socket, + [envelope(1, 'thr_a'), envelope(2, 'thr_a')], + attachment, + ) expect(socket.sent).toHaveLength(1) expect(next?.cursor).toBe(2) diff --git a/packages/core/src/subscription.ts b/packages/core/src/subscription.ts index aaff364..7cc5c0d 100644 --- a/packages/core/src/subscription.ts +++ b/packages/core/src/subscription.ts @@ -1,7 +1,7 @@ import { Option, Predicate, Schema } from 'effect' -import { decodeLoomsEvent, encodeLoomsEvent } from './event-codec' import type { EventEnvelope } from './envelope' +import { decodeLoomsEvent, encodeLoomsEvent } from './event-codec' import { stringifyJson, type JsonValue } from './types' /** A socket is closed with this code when the client must resume from `head`. */ @@ -161,7 +161,9 @@ export function decodeSubscriptionFrame(data: string): SubscriptionFrame | undef export function subscriptionFromSeq( url: URL, lastEventId: string | null, -): { readonly ok: true; readonly fromSeq: number } | { readonly ok: false; readonly message: string } { +): + | { readonly ok: true; readonly fromSeq: number } + | { readonly ok: false; readonly message: string } { if (lastEventId !== null && lastEventId.trim() !== '') { const value = Number(lastEventId) @@ -200,7 +202,9 @@ export function subscriptionFilter(url: URL): SubscriptionFilter { export function subscriptionReplay( head: number, fromSeq: number, -): { readonly kind: 'resync'; readonly head: number } | { readonly kind: 'play'; readonly fromSeq: number } { +): + | { readonly kind: 'resync'; readonly head: number } + | { readonly kind: 'play'; readonly fromSeq: number } { if (fromSeq < head) { return { kind: 'resync', head } } diff --git a/packages/react/src/store.test.ts b/packages/react/src/store.test.ts index eab144b..d977fc4 100644 --- a/packages/react/src/store.test.ts +++ b/packages/react/src/store.test.ts @@ -291,7 +291,10 @@ describe('createLoomsStore websocket', () => { class FakeSocket { static latest: FakeSocket | undefined readonly url: string - private readonly listeners = new Map void>>() + private readonly listeners = new Map< + string, + Array<(event: { readonly data?: unknown }) => void> + >() constructor(url: string) { this.url = url diff --git a/packages/react/src/store.ts b/packages/react/src/store.ts index 393c377..0e9d375 100644 --- a/packages/react/src/store.ts +++ b/packages/react/src/store.ts @@ -1,5 +1,6 @@ import { Data, Effect, Fiber, Option, Predicate, Schema } from 'effect' +import type { SubscriptionSocketConstructor } from '@looms/client' import { decodeLoomsEvent, readSubscriptionData, @@ -11,7 +12,6 @@ import { type RegisteredEvent, type SubscriptionTransport, } from '@looms/core' -import type { SubscriptionSocketConstructor } from '@looms/client' import { EventIndex, ProjectionCache, type LiveFoldDefinition } from './derived' import { materializeEvents } from './materialize' diff --git a/packages/runtime/src/sse.ts b/packages/runtime/src/sse.ts index db9ff5d..a48b292 100644 --- a/packages/runtime/src/sse.ts +++ b/packages/runtime/src/sse.ts @@ -124,7 +124,7 @@ export function createEventStreamResponse(options: EventStreamOptions): Response type: 'resync', head: bounds.head, reason: 'trimmed', - } )}\n\n`, + })}\n\n`, ) close() From 7e93e514092d1c6e4c75109fcad82c9c4581774b Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 14:04:36 +0200 Subject: [PATCH 14/23] fix(core): track blob references per digest so trim never scans other runs Dropping a reference used to list and read every run's manifest, so each trim cost grew with the number of runs in the bucket. Each run now writes refs// markers, and collecting a body lists only that digest's markers. The old scheme also lost bodies when two Durable Objects shared a bucket. retain now writes markers before checking that each new body exists and fails the append if one was deleted. collect lists markers again after deleting and restores the body if a reference appeared. BlobBytes gains exists, and R2BucketLike gains head, so the check does not download the body. Co-authored-by: Cursor --- docs/production.md | 6 +- packages/cloudflare/src/r2-blob-store.test.ts | 7 +- packages/cloudflare/src/r2-blob-store.ts | 13 +- packages/core/src/blob-store.test.ts | 118 ++++++++++++++++ packages/core/src/blob-store.ts | 131 +++++++++++------- packages/runtime/src/filesystem-blob-store.ts | 16 ++- 6 files changed, 231 insertions(+), 60 deletions(-) create mode 100644 packages/core/src/blob-store.test.ts diff --git a/docs/production.md b/docs/production.md index a873644..856d15d 100644 --- a/docs/production.md +++ b/docs/production.md @@ -32,8 +32,10 @@ are still rejected. `memoryBlobStore` is the in-memory implementation. `filesystemBlobStore(directory)` is the Bun and Node implementation. `r2BlobStore(bucket)` writes an R2 bucket -under `looms/blobs/`. One blob store should have one writer. Two Durable Objects -sharing a bucket can delete a hash the other still references. +under `looms/blobs/`. Every run's Durable Object can share one bucket. Each run +writes a marker per body it references, so trimming one run checks only that +body's markers, never other runs' logs. If a body is deleted while another run +is referencing it, that run's append fails and its retry uploads the body again. ## Worker effect protocol diff --git a/packages/cloudflare/src/r2-blob-store.test.ts b/packages/cloudflare/src/r2-blob-store.test.ts index ef916dc..b2b4207 100644 --- a/packages/cloudflare/src/r2-blob-store.test.ts +++ b/packages/cloudflare/src/r2-blob-store.test.ts @@ -32,6 +32,8 @@ function fakeBucket(): R2BucketLike & { readonly keys: () => string[] } { }) }, + head: (key) => Promise.resolve(objects.has(key) ? { key } : null), + delete: (key) => { objects.delete(key) return Promise.resolve(undefined) @@ -78,6 +80,9 @@ describe('r2BlobStore', () => { const resolved = await Effect.runPromise(resolveJson(stored, blobs)) expect(resolved).toEqual(value) - expect(bucket.keys().length).toBeGreaterThan(1) + + await Effect.runPromise(blobs.replaceRefs('run-b', new Set())) + + expect(bucket.keys().filter((key) => key.includes(text.sha256))).toEqual([]) }) }) diff --git a/packages/cloudflare/src/r2-blob-store.ts b/packages/cloudflare/src/r2-blob-store.ts index 5adc31e..af6fb1f 100644 --- a/packages/cloudflare/src/r2-blob-store.ts +++ b/packages/cloudflare/src/r2-blob-store.ts @@ -24,14 +24,15 @@ export interface R2StoredObject { export interface R2BucketLike { put(key: string, value: Uint8Array): Promise get(key: string): Promise + head(key: string): Promise delete(key: string): Promise list(options?: { prefix?: string; cursor?: string }): Promise } /** - * Content-addressed blob store in R2 under `looms/blobs/`. - * Reference counting is local to this bucket. Give one store one writer: two Durable - * Objects deleting the same hash at the same time can remove a body the other still needs. + * Content-addressed blob store in R2 under `looms/blobs/`. Every run's Durable Object can + * share one bucket: each run writes its own reference markers, and a body is deleted only + * after no marker for it remains. */ export function r2BlobStore(bucket: R2BucketLike): BlobStore { return blobStoreFromBytes(r2BlobBytes(bucket)) @@ -70,6 +71,12 @@ function r2BlobBytes(bucket: R2BucketLike): BlobBytes { catch: (cause) => new BlobStoreError(`Unable to delete blob key ${key}`, cause), }).pipe(Effect.asVoid), + exists: (key) => + Effect.tryPromise({ + try: () => bucket.head(keyFor(key)), + catch: (cause) => new BlobStoreError(`Unable to check blob key ${key}`, cause), + }).pipe(Effect.map((object) => object !== null)), + list: (prefix) => listPrefixed(bucket, `${PREFIX}${prefix}`), } } diff --git a/packages/core/src/blob-store.test.ts b/packages/core/src/blob-store.test.ts new file mode 100644 index 0000000..f18eedd --- /dev/null +++ b/packages/core/src/blob-store.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { blobStoreFromBytes, sha256Hex, type BlobBytes } from './blob-store' + +interface Recorded { + readonly bytes: BlobBytes + readonly objects: Map + readonly reads: string[] + onRemove?: (key: string) => void +} + +function recordedBytes(): Recorded { + const objects = new Map() + const reads: string[] = [] + + const recorded: Recorded = { + objects, + reads, + bytes: { + read: (key) => + Effect.sync(() => { + reads.push(key) + return objects.get(key) ?? null + }), + write: (key, body) => + Effect.sync(() => { + objects.set(key, body) + }), + remove: (key) => + Effect.sync(() => { + objects.delete(key) + recorded.onRemove?.(key) + }), + exists: (key) => Effect.sync(() => objects.has(key)), + list: (prefix) => + Effect.sync(() => [...objects.keys()].filter((key) => key.startsWith(prefix))), + }, + } + + return recorded +} + +const body = new TextEncoder().encode('"a large string"') +const digest = await Effect.runPromise(sha256Hex(body)) +const objectKey = `objects/${digest}` + +describe('blob references', () => { + test('a shared body survives until the last run drops it', async () => { + const recorded = recordedBytes() + const blobs = blobStoreFromBytes(recorded.bytes) + + await Effect.runPromise(blobs.put(digest, body)) + await Effect.runPromise(blobs.retain('run-a', [digest])) + await Effect.runPromise(blobs.retain('run-b', [digest])) + await Effect.runPromise(blobs.replaceRefs('run-a', new Set())) + + expect(recorded.objects.has(objectKey)).toBe(true) + + await Effect.runPromise(blobs.replaceRefs('run-b', new Set())) + + expect([...recorded.objects.keys()].filter((key) => key.includes(digest))).toEqual([]) + }) + + test('dropping a reference never reads another run', async () => { + const recorded = recordedBytes() + const blobs = blobStoreFromBytes(recorded.bytes) + + await Effect.runPromise(blobs.put(digest, body)) + + for (const runId of ['run-a', 'run-b', 'run-c', 'run-d']) { + await Effect.runPromise(blobs.retain(runId, [digest])) + } + + recorded.reads.length = 0 + await Effect.runPromise(blobs.replaceRefs('run-a', new Set())) + + expect(recorded.reads).toEqual(['runs/run-a']) + }) + + test('a reference recorded after the body was deleted fails so the append can retry', async () => { + const recorded = recordedBytes() + const blobs = blobStoreFromBytes(recorded.bytes) + + await Effect.runPromise(blobs.put(digest, body)) + await Effect.runPromise(blobs.retain('run-a', [digest])) + await Effect.runPromise(blobs.put(digest, body)) + await Effect.runPromise(blobs.replaceRefs('run-a', new Set())) + + const failed = await Effect.runPromise(Effect.flip(blobs.retain('run-b', [digest]))) + + expect(failed.message).toContain('retry the append') + + await Effect.runPromise(blobs.put(digest, body)) + await Effect.runPromise(blobs.retain('run-b', [digest])) + + expect(recorded.objects.has(objectKey)).toBe(true) + }) + + test('a body deleted while another run references it is put back', async () => { + const recorded = recordedBytes() + const blobs = blobStoreFromBytes(recorded.bytes) + + await Effect.runPromise(blobs.put(digest, body)) + await Effect.runPromise(blobs.retain('run-a', [digest])) + + recorded.onRemove = (key) => { + if (key === objectKey) { + recorded.objects.set(`refs/${digest}/run-b`, new Uint8Array(0)) + } + } + + await Effect.runPromise(blobs.replaceRefs('run-a', new Set())) + + expect(recorded.objects.get(objectKey)).toEqual(body) + }) +}) diff --git a/packages/core/src/blob-store.ts b/packages/core/src/blob-store.ts index 0289a66..f9c54ec 100644 --- a/packages/core/src/blob-store.ts +++ b/packages/core/src/blob-store.ts @@ -33,8 +33,8 @@ export type BlobStoreFailure = BlobStoreError | BlobNotFoundError /** * Content-addressed bytes. `put` is idempotent: the same digest stores one body. - * `retain` and `replaceRefs` record which digests a run still needs. A body is - * deleted only when no run's reference set mentions it. One store should have one writer. + * `retain` and `replaceRefs` record which digests a run still needs. A body is deleted only + * when no run references it. Many runs may share one store, each writing its own references. */ export interface BlobStore { readonly put: (digest: string, body: Uint8Array) => Effect.Effect @@ -58,9 +58,12 @@ export interface BlobBytes { readonly read: (key: string) => Effect.Effect readonly write: (key: string, body: Uint8Array) => Effect.Effect readonly remove: (key: string) => Effect.Effect + readonly exists: (key: string) => Effect.Effect readonly list: (prefix: string) => Effect.Effect, BlobStoreError> } +const EMPTY = new Uint8Array(0) + export function memoryBlobStore(): BlobStore { return blobStoreFromBytes(memoryBlobBytes()) } @@ -105,45 +108,92 @@ export function blobStoreFromBytes(bytes: BlobBytes): BlobStore { return body }), + // Markers go down before the existence check, and `collect` lists markers again after + // deleting. Whichever side runs second sees the other, so a referenced body is never lost: + // either `collect` restores it or this append fails and its retry uploads the body again. retain: (runId, digests) => Effect.gen(function* () { - const current = yield* readRunDigests(bytes, runId) - const next = new Set(current) + const current = new Set(yield* readRunDigests(bytes, runId)) + const added = [...new Set(digests)].filter((digest) => !current.has(digest)) - for (const digest of digests) { - next.add(digest) + if (added.length === 0) { + return yield* Effect.void } - if (next.size === current.length) { - return + yield* Effect.forEach(added, assertDigest, { discard: true }) + + yield* Effect.forEach(added, (digest) => bytes.write(refKey(digest, runId), EMPTY), { + concurrency: REF_CONCURRENCY, + discard: true, + }) + + yield* writeRunDigests(bytes, runId, [...current, ...added]) + + const present = yield* Effect.forEach(added, (digest) => bytes.exists(objectKey(digest)), { + concurrency: REF_CONCURRENCY, + }) + + const missing = added.filter((_, index) => !present[index]) + + if (missing.length > 0) { + return yield* new BlobStoreError( + `blobs ${missing.join(', ')} were deleted before run ${runId} referenced them; retry the append`, + ) } - yield* writeRunDigests(bytes, runId, [...next]) + return yield* Effect.void }), replaceRefs: (runId, keep) => Effect.gen(function* () { const previous = yield* readRunDigests(bytes, runId) - - yield* writeRunDigests(bytes, runId, [...keep]) - const dropped = previous.filter((digest) => !keep.has(digest)) if (dropped.length === 0) { return } - const referenced = yield* referencedDigests(bytes) - - for (const digest of dropped) { - if (!referenced.has(digest)) { - yield* bytes.remove(objectKey(digest)) - } - } + yield* Effect.forEach( + dropped, + (digest) => + bytes.remove(refKey(digest, runId)).pipe(Effect.andThen(collect(bytes, digest))), + { concurrency: REF_CONCURRENCY, discard: true }, + ) + + yield* writeRunDigests( + bytes, + runId, + previous.filter((digest) => keep.has(digest)), + ) }), } } +const REF_CONCURRENCY = 8 + +/** Deletes a body no run references, and puts it back if a run referenced it meanwhile. */ +function collect(bytes: BlobBytes, digest: string): Effect.Effect { + const referenced = bytes.list(refPrefix(digest)).pipe(Effect.map((keys) => keys.length > 0)) + + return Effect.gen(function* () { + if (yield* referenced) { + return + } + + const body = yield* bytes.read(objectKey(digest)) + + if (!body) { + return + } + + yield* bytes.remove(objectKey(digest)) + + if (yield* referenced) { + yield* bytes.write(objectKey(digest), body) + } + }) +} + function memoryBlobBytes(): BlobBytes { const objects = new Map() @@ -165,6 +215,8 @@ function memoryBlobBytes(): BlobBytes { objects.delete(key) }), + exists: (key) => Effect.sync(() => objects.has(key)), + list: (prefix) => Effect.sync(() => [...objects.keys()].filter((key) => key.startsWith(prefix))), } @@ -178,6 +230,14 @@ function runKey(runId: string): string { return `runs/${encodeURIComponent(runId)}` } +function refPrefix(digest: string): string { + return `refs/${digest}/` +} + +function refKey(digest: string, runId: string): string { + return `${refPrefix(digest)}${encodeURIComponent(runId)}` +} + function assertDigest(digest: string): Effect.Effect { if (isSha256Digest(digest)) { return Effect.void @@ -232,36 +292,3 @@ function writeRunDigests( ): Effect.Effect { return bytes.write(runKey(runId), new TextEncoder().encode(JSON.stringify(digests))) } - -function referencedDigests(bytes: BlobBytes): Effect.Effect, BlobStoreError> { - return bytes.list('runs/').pipe( - Effect.flatMap((keys) => - Effect.forEach(keys, (key) => - bytes.read(key).pipe( - Effect.flatMap((body) => { - if (!body) { - return Effect.succeed([]) - } - - return decodeDigestList(new TextDecoder().decode(body)).pipe( - Effect.mapError( - (cause) => new BlobStoreError('blob references are unreadable', cause), - ), - ) - }), - ), - ), - ), - Effect.map((lists) => { - const digests = new Set() - - for (const list of lists) { - for (const digest of list) { - digests.add(digest) - } - } - - return digests - }), - ) -} diff --git a/packages/runtime/src/filesystem-blob-store.ts b/packages/runtime/src/filesystem-blob-store.ts index f971768..6fd7119 100644 --- a/packages/runtime/src/filesystem-blob-store.ts +++ b/packages/runtime/src/filesystem-blob-store.ts @@ -1,4 +1,4 @@ -import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises' +import { mkdir, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises' import { Data, Effect, Predicate } from 'effect' @@ -8,7 +8,7 @@ class MissingBlobFile extends Data.TaggedError('MissingBlobFile') {} /** * Content-addressed blob store rooted at `directory`. For Bun and Node hosts. - * A Durable Object should use `r2BlobStore` instead. One directory should have one writer. + * A Durable Object should use `r2BlobStore` instead. */ export function filesystemBlobStore(directory: string): BlobStore { return blobStoreFromBytes(filesystemBlobBytes(directory)) @@ -38,6 +38,18 @@ function filesystemBlobBytes(directory: string): BlobBytes { catch: (cause) => new BlobStoreError(`Unable to delete blob key ${key}`, cause), }), + exists: (key) => + Effect.tryPromise({ + try: () => stat(`${directory}/${key}`), + catch: (cause): BlobStoreError | MissingBlobFile => + isMissingFile(cause) + ? new MissingBlobFile() + : new BlobStoreError(`Unable to check blob key ${key}`, cause), + }).pipe( + Effect.as(true), + Effect.catchTag('MissingBlobFile', () => Effect.succeed(false)), + ), + list: (prefix) => listFolder(directory, prefix), } } From f43390ea24166b19f477a677fa7b267f77fd353b Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 14:13:13 +0200 Subject: [PATCH 15/23] refactor(runtime): build effect failure payloads from the protocol type Co-authored-by: Cursor --- packages/runtime/src/dispatch-effect.ts | 29 ++++++------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/packages/runtime/src/dispatch-effect.ts b/packages/runtime/src/dispatch-effect.ts index db43c8f..21a8b2e 100644 --- a/packages/runtime/src/dispatch-effect.ts +++ b/packages/runtime/src/dispatch-effect.ts @@ -1,4 +1,4 @@ -import type { Context, ManagedRuntime } from 'effect' +import type { Context, ManagedRuntime, Types } from 'effect' import { Effect, Predicate } from 'effect' import { @@ -8,6 +8,7 @@ import { readEffectFailure, type ComposedRegistry, type EffectContext, + type EffectFailedPayload, type EventInput, RunLauncherTag, type RegisteredDefinition, @@ -32,33 +33,17 @@ function causeMessage(cause: unknown): string { function failedAttempt(effectId: string, threadId: string, cause: unknown): EventInput { const failure = readEffectFailure(cause) const error = failure?.error ?? causeMessage(cause) - const retryAfterMs = failure?.retryAfterMs - - if (failure?.nonRetryable && retryAfterMs !== undefined) { - return { - type: 'runtime.effect.failed', - payload: { effectId, error, nonRetryable: true, retryAfterMs }, - threadId, - } - } + const payload: Types.Mutable = { effectId, error } if (failure?.nonRetryable) { - return { - type: 'runtime.effect.failed', - payload: { effectId, error, nonRetryable: true }, - threadId, - } + payload.nonRetryable = true } - if (retryAfterMs !== undefined) { - return { - type: 'runtime.effect.failed', - payload: { effectId, error, retryAfterMs }, - threadId, - } + if (failure?.retryAfterMs !== undefined) { + payload.retryAfterMs = failure.retryAfterMs } - return { type: 'runtime.effect.failed', payload: { effectId, error }, threadId } + return { type: 'runtime.effect.failed', payload, threadId } } export function dispatchEffect( From 8b628d3dac096707d134bfae70af8f37c7ea50f5 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 14:17:37 +0200 Subject: [PATCH 16/23] feat(runtime): let a running worker attempt read the run's secret context A transport that persists tasks cannot carry secrets, so its worker needs another way to read them. POST /runs/:runId/effects/:effectId/:attempt/secrets returns the secret context only while that attempt is started and unsettled, and stale or cancelled otherwise. It uses worker access, so it stays closed until the host configures worker auth, and responses are not cacheable. client.workerSecrets wraps it. Co-authored-by: Cursor --- .changeset/run-context.md | 2 +- apps/docs/src/docs-manifest.json | 2 +- docs/production.md | 10 +++- packages/client/src/client.test.ts | 24 +++++++++ packages/client/src/client.ts | 41 +++++++++++---- packages/core/src/worker-callback.ts | 6 +++ packages/runtime/src/run-context.test.ts | 51 ++++++++++++++++++- packages/runtime/src/server/routes/effects.ts | 19 +++++++ packages/runtime/src/types.ts | 14 +++++ packages/runtime/src/worker-callbacks.ts | 26 +++++++++- 10 files changed, 178 insertions(+), 17 deletions(-) diff --git a/.changeset/run-context.md b/.changeset/run-context.md index b6bf62d..d38fe7a 100644 --- a/.changeset/run-context.md +++ b/.changeset/run-context.md @@ -2,4 +2,4 @@ '@swirls/looms': minor --- -Runs accept immutable `attributes` and a `secrets` context. Attributes are recorded on `runtime.run.started`, the run state, and the run summary, and are passed to `authorize` when present. Secrets stay in a side store: they are available to effect handlers and in-process worker dispatch, and they are not written to events, snapshots, or HTTP responses. +Runs accept immutable `attributes` and a `secrets` context. Attributes are recorded on `runtime.run.started`, the run state, and the run summary, and are passed to `authorize` when present. Secrets stay in a side store: they are available to effect handlers and in-process worker dispatch, and they are not written to events, snapshots, or run responses. A worker whose transport persists tasks reads them with `POST /runs/:runId/effects/:effectId/:attempt/secrets` (`client.workerSecrets`), which only answers the attempt that is currently running. diff --git a/apps/docs/src/docs-manifest.json b/apps/docs/src/docs-manifest.json index 4b05ea8..1fdc569 100644 --- a/apps/docs/src/docs-manifest.json +++ b/apps/docs/src/docs-manifest.json @@ -830,7 +830,7 @@ "level": 2 } ], - "text": "Reliability and recovery Durability depends on the host and storage you choose. In-memory examples lose history on exit. SQLite persists locally; production actor hosts must preserve one writer per run and provide wake scheduling. Replay and recovery are different operations Historical replay folds recorded events to reconstruct state and does not dispatch effects. Recovery loads a snapshot and later events, then resumes unfinished work. An effect that lacks a recorded outcome may execute again. Looms does not promise exactly-once actions across arbitrary external services. A crash after an external action Looms requests an external action. The service performs it. The host stops before recording success. Recovery cannot infer success from the local log alone. Pass ctx.effectId as an idempotency key where the external API supports it. Check that the service retains the key for as long as you may need to retry the action. For unsupported APIs, store a durable operation ID, query the provider for the outcome, or reconcile before retrying. Never assume adding an arbitrary header makes an API idempotent. Retries and errors Effect retry policies count total attempts: maxAttempts: 3 means one initial attempt and up to two retries. Attempt starts and retry deadlines are recorded before the runtime waits, so a restart preserves the retry budget and backoff. Configure start, heartbeat, and execution timeouts for work delivered to external workers. Worker callbacks include the effect ID and attempt number; stale callbacks cannot complete a newer attempt. Exhaustion records runtime.effect.failed . Signals and webhook delivery Use the embed start/signal idempotencyKey option with a stable runId for retries of the same logical request. Verify webhook signatures and transform allowed webhook payloads into typed events on the server. Deduplicate by the provider delivery ID and correlate by run, thread, and domain identifiers. Retrying with a fresh key defeats deduplication. Interrupted model streams A model call can produce partial deltas before failing. Those deltas may already be recorded; resuming work may incur another model call and different output. Render partial content as provisional, inspect completion events, and decide how your application handles abandoned partial turns. The adapter does not guarantee exactly-once billing or identical regenerated text. Snapshots, retention, and projections A compatible snapshot bounds replay work; it is not a backup or a schema migration. Trimming events removes older audit and time-travel history and can prevent rebuilding new projections from the beginning. Define a retention and archive policy before enabling trimAfterSnapshot. Projectors run after local commit. Durable delivery stores a cursor per projector name and version, retries without advancing that cursor, and dead-letters exhausted batches. A run can finish before an external index or webhook catches up, so monitor lag and keep projector writes idempotent. Test recovery Test a restart while parked, an interruption during an external action, duplicate webhook delivery, and a deployment with existing runs. The testing guide provides the local restart test. Review versioning before changing definitions. ) }" + "text": "Reliability and recovery Durability depends on the host and storage you choose. In-memory examples lose history on exit. SQLite persists locally; production actor hosts must preserve one writer per run and provide wake scheduling. Replay and recovery are different operations Historical replay folds recorded events to reconstruct state and does not dispatch effects. Recovery loads a snapshot and later events, then resumes unfinished work. An effect that lacks a recorded outcome may execute again. Looms does not promise exactly-once actions across arbitrary external services. A crash after an external action Looms requests an external action. The service performs it. The host stops before recording success. Recovery cannot infer success from the local log alone. Pass ctx.effectId as an idempotency key where the external API supports it. Check that the service retains the key for as long as you may need to retry the action. For unsupported APIs, store a durable operation ID, query the provider for the outcome, or reconcile before retrying. Never assume adding an arbitrary header makes an API idempotent. Retries and errors Effect retry policies count total attempts: maxAttempts: 3 means one initial attempt and up to two retries. Attempt starts and retry deadlines are recorded before the runtime waits, so a restart preserves the retry budget and backoff. Configure start, heartbeat, and execution timeouts for work delivered to external workers. Worker callbacks include the effect ID and attempt number; stale callbacks cannot complete a newer attempt. Exhaustion records runtime.effect.failed . A handler can stop that budget early with failEffect and nonRetryable: true , or ask the next attempt to wait at least retryAfterMs without shortening the policy backoff. A worker heartbeat can carry details ; the next dispatch of that effect includes the latest details so the attempt can resume. The callback response is {'{ status, cancelRequested }'} , not the run state. cancelRequested tells an attempt that is already running to stop. Signals and webhook delivery Use the embed start/signal idempotencyKey option with a stable runId for retries of the same logical request. Verify webhook signatures and transform allowed webhook payloads into typed events on the server. Deduplicate by the provider delivery ID and correlate by run, thread, and domain identifiers. Retrying with a fresh key defeats deduplication. Interrupted model streams A model call can produce partial deltas before failing. Those deltas may already be recorded; resuming work may incur another model call and different output. Render partial content as provisional, inspect completion events, and decide how your application handles abandoned partial turns. The adapter does not guarantee exactly-once billing or identical regenerated text. Snapshots, retention, and projections A compatible snapshot bounds replay work; it is not a backup or a schema migration. Trimming events removes older audit and time-travel history and can prevent rebuilding new projections from the beginning. Define a retention and archive policy before enabling trimAfterSnapshot. Projectors run after local commit. Durable delivery stores a cursor per projector name and version, retries without advancing that cursor, and dead-letters exhausted batches. A run can finish before an external index or webhook catches up, so monitor lag and keep projector writes idempotent. Test recovery Test a restart while parked, an interruption during an external action, duplicate webhook delivery, and a deployment with existing runs. The testing guide provides the local restart test. Review versioning before changing definitions. ) }" }, { "path": "/docs/schedules", diff --git a/docs/production.md b/docs/production.md index a79f4a0..57cc144 100644 --- a/docs/production.md +++ b/docs/production.md @@ -28,8 +28,14 @@ any. They must be safe for every reader of the run. them in a side table, `looms_run_secrets`, which is not part of the event log or a snapshot. Effect handlers read them on `ctx.secrets`. An in-process worker receives them on the dispatch task. A transport that persists that task must -omit `secrets` and have the worker fetch them through an authenticated callback -for the current attempt. A handler must not copy secrets into an emitted event. +omit `secrets`. Its worker reads them with +`POST /runs/:runId/effects/:effectId/:attempt/secrets` (`client.workerSecrets`) +after its `started` callback is accepted. The route uses `worker` access, so it +returns 503 until the host configures worker auth. It answers +`{ status: 'accepted', secrets }` only while that attempt is started and +unsettled, and `{ status: 'stale' }` or `{ status: 'cancelled' }` otherwise. +Responses are sent with `cache-control: no-store`. A handler must not copy +secrets into an emitted event. A retry of the same start may omit `secrets` and keeps the stored context. Sending a different context conflicts, and the error does not include the secret values. diff --git a/packages/client/src/client.test.ts b/packages/client/src/client.test.ts index f454244..63ddbd5 100644 --- a/packages/client/src/client.test.ts +++ b/packages/client/src/client.test.ts @@ -66,6 +66,30 @@ describe('createLoomsClient', () => { ]) }) + test('workerSecrets posts to the attempt with the worker token', async () => { + const calls: { href: string; authorization?: string | null }[] = [] + + const client = createLoomsClient({ + workerCallbackToken: 'worker-token', + fetch: async (input, init) => { + calls.push({ + href: hrefOf(input), + authorization: new Headers(init?.headers).get('authorization'), + }) + + return Response.json({ status: 'accepted', secrets: { token: 'abc' } }) + }, + }) + + const result = await client.workerSecrets('run 1', { effectId: 'eff/1', attempt: 2 }) + + expect(result).toEqual({ status: 'accepted', secrets: { token: 'abc' } }) + + expect(calls).toEqual([ + { href: '/runs/run%201/effects/eff%2F1/2/secrets', authorization: 'Bearer worker-token' }, + ]) + }) + test('listDefinitions reads the definitions route', async () => { const calls: string[] = [] diff --git a/packages/client/src/client.ts b/packages/client/src/client.ts index bfd0d48..87d6efd 100644 --- a/packages/client/src/client.ts +++ b/packages/client/src/client.ts @@ -7,6 +7,7 @@ import { ReplayStepSchema, RunStateSchema, RunSummariesSchema, + SecretContextSchema, stringifyJson, type DefinitionInput, type DefinitionRef, @@ -16,8 +17,10 @@ import { type RunAttributes, type RunState, type SecretContext, + type WorkerCallback, type WorkerCallbackInput, type WorkerCallbackResult, + type WorkerSecretsResult, } from '@looms/core' import { consumeSseStreamEffect, eventsFromSseData, sseFrames } from './sse' @@ -50,6 +53,10 @@ interface WorkerFailRequestBody { retryAfterMs?: number } +function attemptPath(runId: string, attempt: WorkerCallback): string { + return `/runs/${encodeURIComponent(runId)}/effects/${encodeURIComponent(attempt.effectId)}/${attempt.attempt}` +} + function workerCallbackBody(input: WorkerCallbackInput) { switch (input.kind) { case 'complete': @@ -91,6 +98,11 @@ const WorkerCallbackResultSchema = Schema.Struct({ cancelRequested: Schema.Boolean, }) +const WorkerSecretsResultSchema = Schema.Union([ + Schema.Struct({ status: Schema.Literal('accepted'), secrets: SecretContextSchema }), + Schema.Struct({ status: Schema.Union([Schema.Literal('stale'), Schema.Literal('cancelled')]) }), +]) + /** `appended` counts the events the run gained while the wake ran. */ const WakeResultSchema = Schema.Struct({ runId: Schema.String, @@ -233,25 +245,31 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { secrets?: SecretContext }) => request('/runs', StartResultSchema, { method: 'POST', body: stringifyJson(args) }) + const workerHeaders = options.workerCallbackToken + ? { authorization: `Bearer ${options.workerCallbackToken}` } + : undefined + const workerCallback = ( runId: string, input: WorkerCallbackInput, ): Promise => { const body = workerCallbackBody(input) - return request( - `/runs/${encodeURIComponent(runId)}/effects/${encodeURIComponent(input.effectId)}/${input.attempt}/${input.kind}`, - WorkerCallbackResultSchema, - { - method: 'POST', - headers: options.workerCallbackToken - ? { authorization: `Bearer ${options.workerCallbackToken}` } - : undefined, - body: stringifyJson(body), - }, - ) + return request(`${attemptPath(runId, input)}/${input.kind}`, WorkerCallbackResultSchema, { + method: 'POST', + headers: workerHeaders, + body: stringifyJson(body), + }) } + /** Read the run's secret context. Only the attempt that is currently running gets it. */ + const workerSecrets = (runId: string, attempt: WorkerCallback): Promise => + request(`${attemptPath(runId, attempt)}/secrets`, WorkerSecretsResultSchema, { + method: 'POST', + headers: workerHeaders, + body: '{}', + }) + const streamRun = (args: { kind: string definitionName: string @@ -350,6 +368,7 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { body: stringifyJson(forkOptions?.runId === undefined ? {} : { runId: forkOptions.runId }), }), workerCallback, + workerSecrets, listRunSummaries: () => request('/runs/summaries', RunSummariesSchema), listDefinitions: () => request('/definitions', DefinitionCatalogSchema), replayTo: (runId: string, seq: number) => diff --git a/packages/core/src/worker-callback.ts b/packages/core/src/worker-callback.ts index 2badf51..cb9a60d 100644 --- a/packages/core/src/worker-callback.ts +++ b/packages/core/src/worker-callback.ts @@ -1,4 +1,5 @@ import type { EventInput } from './envelope' +import type { SecretContext } from './run-context' import type { JsonValue } from './types' /** Identifies one attempt of an effect executed by an external worker. */ @@ -31,3 +32,8 @@ export interface WorkerCallbackResult { readonly status: 'accepted' | 'stale' | 'cancelled' readonly cancelRequested: boolean } + +/** The run's secret context, released only to the attempt that is currently running. */ +export type WorkerSecretsResult = + | { readonly status: 'accepted'; readonly secrets: SecretContext } + | { readonly status: 'stale' | 'cancelled' } diff --git a/packages/runtime/src/run-context.test.ts b/packages/runtime/src/run-context.test.ts index 37ec4a6..7fcb831 100644 --- a/packages/runtime/src/run-context.test.ts +++ b/packages/runtime/src/run-context.test.ts @@ -16,7 +16,7 @@ import { import { SecretContextConflictError, StartRunConflictError } from './errors' import { createLooms } from './looms' -import { authDenied, authAllowed } from './server' +import { authDenied, authAllowed, bearerAuth } from './server' const SECRET = 'super-secret-token' @@ -135,6 +135,55 @@ describe('run attributes and secret context', () => { await looms.stop() }) + test('a worker reads the secret context only while its attempt is running', async () => { + const looms = createLooms({ + modules: [contextModule], + worker: { handles: ['context.seen'], dispatch: () => undefined }, + authorize: bearerAuth({ worker: 'worker-token' }), + }) + + await looms.start(context, null, { runId: 'run_worker', secrets: { token: SECRET } }) + const effectId = (await looms.getRun('run_worker')).outstandingEffects[0]?.effectId ?? '' + + const call = async (path: string, init: { token?: string; body?: string } = {}) => { + const response = await looms.fetch( + new Request( + `http://looms.test/runs/run_worker/effects/${encodeURIComponent(effectId)}/${path}`, + { + method: 'POST', + headers: { authorization: `Bearer ${init.token ?? 'worker-token'}` }, + body: init.body ?? '{}', + }, + ), + ) + + return { status: response?.status, headers: response?.headers, body: await response?.json() } + } + + expect((await call('1/secrets')).body).toEqual({ status: 'stale' }) + expect((await call('1/started')).body).toEqual({ status: 'accepted', cancelRequested: false }) + + const accepted = await call('1/secrets') + expect(accepted.body).toEqual({ status: 'accepted', secrets: { token: SECRET } }) + expect(accepted.headers?.get('cache-control')).toBe('no-store') + + expect((await call('1/secrets', { token: 'wrong' })).status).toBe(401) + expect((await call('2/secrets')).body).toEqual({ status: 'stale' }) + + const observed = { + type: 'context.observed', + payload: { tenant: '', hasSecret: true }, + threadId: (await looms.getRun('run_worker')).outstandingEffects[0]?.threadId, + } + + await call('1/complete', { body: JSON.stringify({ events: [observed] }) }) + + expect((await looms.getRun('run_worker')).status).toBe('completed') + expect((await call('1/secrets')).body).toEqual({ status: 'stale' }) + expect(containsSecret(JSON.stringify(await looms.getEvents('run_worker')))).toBe(false) + await looms.stop() + }) + test('a different attribute or secret context conflicts without revealing the secret', async () => { const looms = createLooms({ modules: [contextModule] }) diff --git a/packages/runtime/src/server/routes/effects.ts b/packages/runtime/src/server/routes/effects.ts index 402f65b..5d7f2fe 100644 --- a/packages/runtime/src/server/routes/effects.ts +++ b/packages/runtime/src/server/routes/effects.ts @@ -123,6 +123,25 @@ export function effectRoutes({ runtime, store }: RouteOptions): readonly Route[] ), ), }), + route({ + method: 'POST', + path: '/runs/:runId/effects/:effectId/:attempt/secrets', + name: 'effects.workerSecrets', + access: 'worker', + params: { attempt: integer }, + handle: ({ params }) => + provideStore( + runtime.workerSecrets(params.runId, { + effectId: params.effectId, + attempt: params.attempt, + }), + store, + ).pipe( + Effect.map((result) => + Response.json(result, { headers: { 'cache-control': 'no-store' } }), + ), + ), + }), route({ method: 'POST', path: '/runs/:runId/effects/:effectId/:attempt/:action', diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 9e554fa..07e5279 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -5,8 +5,10 @@ import type { ComposedRegistry, EventEnvelope, EventInput, + WorkerCallback, WorkerCallbackInput, WorkerCallbackResult, + WorkerSecretsResult, EventStore, EventStoreAppendError, EventStoreError, @@ -232,6 +234,18 @@ export interface LoomsRuntime< Error | EventStoreAppendError | EventStoreError, EventStoreTag > + /** + * The run's secret context for a worker attempt that has started and not settled. Any other + * attempt is `stale`, so a redelivered or superseded message never sees the secrets. + */ + workerSecrets( + runId: string, + callback: WorkerCallback, + ): Effect.Effect< + WorkerSecretsResult, + Error | EventStoreTruncationError | EventStoreTrimmedError | EventStoreError, + EventStoreTag + > readonly listRuns: Effect.Effect /** Indexed run rows. Folds a run only when its header has not been written yet. */ readonly listRunSummaries: Effect.Effect< diff --git a/packages/runtime/src/worker-callbacks.ts b/packages/runtime/src/worker-callbacks.ts index 882618e..e49e247 100644 --- a/packages/runtime/src/worker-callbacks.ts +++ b/packages/runtime/src/worker-callbacks.ts @@ -9,7 +9,9 @@ import { type OutstandingEffect, type RetryPolicy, type RunState, + type WorkerCallback, type WorkerCallbackInput, + type WorkerSecretsResult, type WorkerCallbackResult, } from '@looms/core' @@ -302,5 +304,27 @@ export function createWorkerCallbacks(options: WorkerCallbacksOptions) { }), ).pipe(Effect.tap((result) => (result.status === 'stale' ? Effect.void : options.wake(runId)))) - return { workerCallback: applyWorkerCallback } + const workerSecrets = (runId: string, callback: WorkerCallback) => + Effect.gen(function* () { + const store = yield* EventStoreTag + const { state } = yield* repository.loadCursor(store, runId) + const execution = state.effectExecutions[callback.effectId] + + if (execution?.attempt !== callback.attempt || isRunTerminal(state)) { + return { status: 'stale' } satisfies WorkerSecretsResult + } + + if (execution.status === 'cancel_requested') { + return { status: 'cancelled' } satisfies WorkerSecretsResult + } + + if (execution.status !== 'started' && execution.status !== 'heartbeat') { + return { status: 'stale' } satisfies WorkerSecretsResult + } + + const secrets = store.runSecrets ? yield* store.runSecrets.get(runId) : {} + return { status: 'accepted', secrets } satisfies WorkerSecretsResult + }) + + return { workerCallback: applyWorkerCallback, workerSecrets } } From cdbb170fbded791d9cb325bee5b4115048ed551d Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 14:20:27 +0200 Subject: [PATCH 17/23] test(runtime): check child runs inherit attributes and secrets Co-authored-by: Cursor --- packages/runtime/src/child-run.test.ts | 36 +++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/packages/runtime/src/child-run.test.ts b/packages/runtime/src/child-run.test.ts index a1bdeda..85340b7 100644 --- a/packages/runtime/src/child-run.test.ts +++ b/packages/runtime/src/child-run.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'bun:test' -import { Predicate } from 'effect' +import { Effect, Predicate } from 'effect' import type { RunState } from '@looms/core' import { defineWorkflow, workflow } from '@looms/workflow' @@ -83,6 +83,40 @@ describe('child runs', () => { await looms.stop() }) + test('a child run inherits the parent run attributes and secrets', async () => { + const looms = createLooms({ modules: [workflow({ definitions: [child, parent] })] }) + + const started = await looms.start( + parent, + {}, + { + attributes: { tenant: 'acme' }, + secrets: { token: 'parent-secret' }, + }, + ) + + await waitForStatus(looms, started.runId, 'completed') + + const launched = (await looms.getEvents(started.runId)).find( + (event) => event.type === 'runtime.run.launched', + ) + + const childRunId = + launched && Predicate.isObject(launched.payload) && Predicate.isString(launched.payload.runId) + ? launched.payload.runId + : '' + + const store = await looms.store + + expect((await looms.getRun(childRunId)).attributes).toEqual({ tenant: 'acme' }) + + expect( + await Effect.runPromise(store.runSecrets?.get(childRunId) ?? Effect.succeed({})), + ).toEqual({ token: 'parent-secret' }) + + await looms.stop() + }) + test('collects mapped child runs in item order', async () => { const looms = createLooms({ modules: [workflow({ definitions: [mappedChild, mappedParent] })], From 3f5702c132a0eb2a4af32dbc22e1266f8a6416d7 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 14:27:17 +0200 Subject: [PATCH 18/23] feat(runtime): carry the caller's W3C trace context through a run POST /runs reads the traceparent and tracestate headers, and start, startRun, and client.startRun accept trace. A valid version-00 context is recorded on runtime.run.started and startIdentity.trace. Invalid values are dropped rather than stored. The context reaches effect handlers as ctx.trace, worker tasks as task.trace, and launched and child runs unless the launch sets its own. It is passed unchanged: Looms records no spans, so minting span ids would point at parents that never exist. The object's keys are the W3C header names, so it works directly as an OpenTelemetry carrier. The local executor now takes one RunContext (correlation id, attributes, secrets, trace) in place of separate positional arguments. Co-authored-by: Cursor --- .changeset/otel-trace.md | 5 + docs/production.md | 19 ++ packages/client/src/client.test.ts | 30 +++ packages/client/src/client.ts | 21 +- packages/core/src/effects.ts | 4 + packages/core/src/fold.test.ts | 1 + packages/core/src/fold.ts | 2 + packages/core/src/index.ts | 1 + packages/core/src/launcher.ts | 3 + packages/core/src/protocol.ts | 2 + packages/core/src/state.ts | 6 + packages/core/src/trace.test.ts | 39 +++ packages/core/src/trace.ts | 63 +++++ packages/runtime/src/helpers.ts | 6 + packages/runtime/src/local-effect-executor.ts | 40 ++- packages/runtime/src/looms.ts | 4 + packages/runtime/src/runtime.ts | 26 +- packages/runtime/src/server/routes/runs.ts | 2 + packages/runtime/src/trace.test.ts | 243 ++++++++++++++++++ packages/runtime/src/types.ts | 8 + packages/runtime/src/wake-engine.ts | 9 +- .../runtime/src/worker-effect-dispatch.ts | 1 + 22 files changed, 502 insertions(+), 33 deletions(-) create mode 100644 .changeset/otel-trace.md create mode 100644 packages/core/src/trace.test.ts create mode 100644 packages/core/src/trace.ts create mode 100644 packages/runtime/src/trace.test.ts diff --git a/.changeset/otel-trace.md b/.changeset/otel-trace.md new file mode 100644 index 0000000..3580c53 --- /dev/null +++ b/.changeset/otel-trace.md @@ -0,0 +1,5 @@ +--- +'@swirls/looms': minor +--- + +Runs carry the caller's W3C trace context. `POST /runs` reads the `traceparent` and `tracestate` headers, and `start`, `startRun`, and `client.startRun` accept `trace`. A valid context is recorded on `runtime.run.started` and `startIdentity.trace`, then passed unchanged to effect handlers (`ctx.trace`), worker tasks (`task.trace`), and every launched or child run unless the launch sets its own. diff --git a/docs/production.md b/docs/production.md index 568c2d8..6add06e 100644 --- a/docs/production.md +++ b/docs/production.md @@ -64,6 +64,25 @@ a different context conflicts, and the error does not include the secret values. Forks and effect-launched runs inherit the parent's attributes and secrets. A store without `runSecrets` rejects a start that supplies secrets. +## Trace context + +`POST /runs` reads the W3C `traceparent` and `tracestate` headers. In process, +pass `trace: { traceparent, tracestate }` to `start` or `startRun`, and +`client.startRun({ trace })` sends it as those headers. A valid version-00 +`traceparent` is recorded on `runtime.run.started` and folded into +`startIdentity.trace`. Anything else is dropped, and so is a `tracestate` over +512 characters. It is not part of the start identity, so a retry from another +trace returns the same run. + +The run's trace context is passed unchanged to every effect handler as +`ctx.trace`, to every worker dispatch as `task.trace`, and to every run an +effect launches, including workflow child runs. A launch can set its own +`trace`, for example to parent the child under a span the handler started. +Forks keep the source's context. Looms creates no spans itself. The object's +keys are the W3C header names, so with OpenTelemetry, +`propagation.extract(context.active(), ctx.trace ?? {})` gives the parent +context for a handler's span. + ## Worker effect protocol 1. The runtime dispatches `{ runId, effectId, attempt, type, input, details? }`. diff --git a/packages/client/src/client.test.ts b/packages/client/src/client.test.ts index 63ddbd5..aff03d9 100644 --- a/packages/client/src/client.test.ts +++ b/packages/client/src/client.test.ts @@ -66,6 +66,36 @@ describe('createLoomsClient', () => { ]) }) + test('startRun sends the trace context as W3C headers, not in the body', async () => { + const calls: { traceparent: string | null; tracestate: string | null; body: unknown }[] = [] + + const client = createLoomsClient({ + fetch: async (_input, init) => { + const headers = new Headers(init?.headers) + + calls.push({ + traceparent: headers.get('traceparent'), + tracestate: headers.get('tracestate'), + body: Predicate.isString(init?.body) ? JSON.parse(init.body) : null, + }) + + return Response.json({ runId: 'run_1', threadId: 'thr_1', state: emptyRunState('run_1') }) + }, + }) + + const traceparent = '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01' + + await client.startRun({ + kind: 'agent', + definitionName: 'echo', + trace: { traceparent, tracestate: 'k=v' }, + }) + + expect(calls).toEqual([ + { traceparent, tracestate: 'k=v', body: { kind: 'agent', definitionName: 'echo' } }, + ]) + }) + test('workerSecrets posts to the attempt with the worker token', async () => { const calls: { href: string; authorization?: string | null }[] = [] diff --git a/packages/client/src/client.ts b/packages/client/src/client.ts index 87d6efd..cd642ba 100644 --- a/packages/client/src/client.ts +++ b/packages/client/src/client.ts @@ -9,6 +9,7 @@ import { RunSummariesSchema, SecretContextSchema, stringifyJson, + traceHeaders, type DefinitionInput, type DefinitionRef, type EventEnvelope, @@ -17,6 +18,7 @@ import { type RunAttributes, type RunState, type SecretContext, + type TraceContext, type WorkerCallback, type WorkerCallbackInput, type WorkerCallbackResult, @@ -243,7 +245,17 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { runId?: string attributes?: RunAttributes secrets?: SecretContext - }) => request('/runs', StartResultSchema, { method: 'POST', body: stringifyJson(args) }) + /** Sent as the W3C `traceparent` and `tracestate` headers. */ + trace?: TraceContext + }) => { + const { trace, ...body } = args + + return request('/runs', StartResultSchema, { + method: 'POST', + headers: traceHeaders(trace), + body: stringifyJson(body), + }) + } const workerHeaders = options.workerCallbackToken ? { authorization: `Bearer ${options.workerCallbackToken}` } @@ -276,8 +288,10 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { definitionVersion?: string input?: JsonValue runId?: string + trace?: TraceContext }): StreamHandle => { - const runId = args.runId ?? createRunId() + const { trace, ...body } = args + const runId = body.runId ?? createRunId() const stream = Stream.unwrap( Effect.tryPromise({ @@ -285,10 +299,11 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { fetchImpl(`${baseUrl}/runs`, { method: 'POST', headers: { + ...traceHeaders(trace), 'content-type': 'application/json', accept: 'text/event-stream', }, - body: stringifyJson({ ...args, runId }), + body: stringifyJson({ ...body, runId }), signal, }), catch: (cause) => new LoomsClientError('Failed to start event stream', cause), diff --git a/packages/core/src/effects.ts b/packages/core/src/effects.ts index fb029d5..28636ff 100644 --- a/packages/core/src/effects.ts +++ b/packages/core/src/effects.ts @@ -9,6 +9,7 @@ import { type InvalidInputError, type SchemaInput, } from './schema' +import type { TraceContext } from './trace' import { asJson, type JsonValue } from './types' export type WaitOnEvent = { @@ -232,6 +233,8 @@ export interface EffectContext { readonly attributes: RunAttributes /** Bearer material for this run. Never copy it into an emitted event. */ readonly secrets: SecretContext + /** The W3C trace context of the caller that started the run, to parent spans the handler starts. */ + readonly trace: TraceContext | null emit(event: EventInput): Promise } @@ -243,6 +246,7 @@ export interface ScopedEffectContext { readonly signal: AbortSignal readonly attributes: RunAttributes readonly secrets: SecretContext + readonly trace: TraceContext | null emit(event: E): Promise } diff --git a/packages/core/src/fold.test.ts b/packages/core/src/fold.test.ts index 697d952..86f00e8 100644 --- a/packages/core/src/fold.test.ts +++ b/packages/core/src/fold.test.ts @@ -102,6 +102,7 @@ describe('foldRun', () => { rootThreadId: threadId, launchedBy: null, correlationId: null, + trace: null, }) expect(state.threads[threadId]?.status).toBe('running') diff --git a/packages/core/src/fold.ts b/packages/core/src/fold.ts index 8073d32..55e9fd6 100644 --- a/packages/core/src/fold.ts +++ b/packages/core/src/fold.ts @@ -9,6 +9,7 @@ import { readAttributes } from './run-context' import type { ThreadRecord, OutstandingEffect, RunState, WaitRecord } from './state' import { completedRunStatus, emptyRunState } from './state' import { isTerminalStatus, type ThreadContext, type ThreadDefinition } from './thread' +import { readStoredTrace } from './trace' import type { JsonValue } from './types' export interface FoldRegistry { @@ -155,6 +156,7 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi launchedBy: readRunLaunchSource(payload.launchedBy), correlationId: event.correlationId ?? null, attributes: readAttributes(payload.attributes), + trace: readStoredTrace(payload.trace), } : null diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 533edd3..772bbb0 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -44,6 +44,7 @@ export * from './blob-offload' export * from './definition-store' export * from './definition-resolver' export * from './launcher' +export * from './trace' export * from './run-context' export * from './published' export type { DebuggerHost, DebuggerRequest } from './debugger-host' diff --git a/packages/core/src/launcher.ts b/packages/core/src/launcher.ts index 4ed3e49..f64fd48 100644 --- a/packages/core/src/launcher.ts +++ b/packages/core/src/launcher.ts @@ -2,6 +2,7 @@ import { Context, Data, Effect, Option, Schema } from 'effect' import type { EventInput } from './envelope' import type { RunAttributes, SecretContext } from './run-context' +import type { TraceContext } from './trace' import type { JsonValue, RunStatus } from './types' export const ChildCloseSchema = Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]) @@ -49,6 +50,8 @@ export interface LaunchRunArgs { readonly close?: ChildClose readonly attributes?: RunAttributes readonly secrets?: SecretContext + /** Defaults to the launching run's trace context. */ + readonly trace?: TraceContext } /** Enough of a run to decide whether a previous fire is still in progress. */ diff --git a/packages/core/src/protocol.ts b/packages/core/src/protocol.ts index 7354fe9..5bbf8b3 100644 --- a/packages/core/src/protocol.ts +++ b/packages/core/src/protocol.ts @@ -6,6 +6,7 @@ import { JsonValueSchema } from './envelope' import { RunLaunchSourceSchema } from './launcher' import { DEFAULT_DEFINITION_VERSION } from './module' import { RunAttributesSchema } from './run-context' +import { TraceContextSchema } from './trace' const DefinitionVersionSchema = Schema.String.pipe( Schema.withDecodingDefaultKey(Effect.succeed(DEFAULT_DEFINITION_VERSION)), @@ -21,6 +22,7 @@ export const protocolCatalog = defineEventCatalog('runtime', { requestedThreadId: Schema.optional(Schema.NullOr(Schema.String)), launchedBy: Schema.optional(RunLaunchSourceSchema), attributes: Schema.optional(RunAttributesSchema), + trace: Schema.optional(TraceContextSchema), }), 'run.completed': Schema.Struct({ output: Schema.NullOr(JsonValueSchema), diff --git a/packages/core/src/state.ts b/packages/core/src/state.ts index 165c839..c86e658 100644 --- a/packages/core/src/state.ts +++ b/packages/core/src/state.ts @@ -11,6 +11,7 @@ import { RunLaunchSourceSchema, type ChildClose, type RunLaunchSource } from './ import { DEFAULT_DEFINITION_VERSION } from './module' import { RunAttributesSchema, type RunAttributes } from './run-context' import { isTerminalStatus } from './thread' +import { TraceContextSchema, type TraceContext } from './trace' import type { JsonValue, ThreadStatus, RunStatus } from './types' export interface ThreadRecord { @@ -75,6 +76,8 @@ export interface RunStartIdentity { launchedBy: RunLaunchSource | null correlationId: string | null attributes: RunAttributes + /** The caller's W3C trace context. Effects, workers, and launched runs inherit it. */ + trace: TraceContext | null } /** The run and last sequence a fork copied its history from. */ @@ -135,6 +138,9 @@ export const RunStateSchema = Schema.Struct({ Schema.withDecodingDefaultKey(Effect.succeed(null)), ), attributes: RunAttributesSchema.pipe(Schema.withDecodingDefaultKey(Effect.succeed({}))), + trace: Schema.NullOr(TraceContextSchema).pipe( + Schema.withDecodingDefaultKey(Effect.succeed(null)), + ), }), ).pipe(Schema.withDecodingDefaultKey(Effect.succeed(null))), attributes: RunAttributesSchema.pipe(Schema.withDecodingDefaultKey(Effect.succeed({}))), diff --git a/packages/core/src/trace.test.ts b/packages/core/src/trace.test.ts new file mode 100644 index 0000000..072b252 --- /dev/null +++ b/packages/core/src/trace.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, test } from 'bun:test' + +import { readTraceContext, readTraceHeaders, traceHeaders } from './trace' + +const TRACEPARENT = '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01' + +describe('trace context', () => { + test('reads a version-00 traceparent and its tracestate', () => { + expect(readTraceContext(` ${TRACEPARENT} `, 'congo=t61rcWkgMzE')).toEqual({ + traceparent: TRACEPARENT, + tracestate: 'congo=t61rcWkgMzE', + }) + }) + + test('drops values that are not a W3C traceparent', () => { + expect(readTraceContext(undefined)).toBeUndefined() + expect(readTraceContext('not-a-trace')).toBeUndefined() + expect(readTraceContext(TRACEPARENT.toUpperCase())).toBeUndefined() + expect(readTraceContext(TRACEPARENT.replace(/^00/, '01'))).toBeUndefined() + expect(readTraceContext(`00-${'0'.repeat(32)}-00f067aa0ba902b7-01`)).toBeUndefined() + + expect(readTraceContext(`00-4bf92f3577b34da6a3ce929d0e0e4736-${'0'.repeat(16)}-01`)).toBe( + undefined, + ) + }) + + test('drops a tracestate over the W3C length limit but keeps the traceparent', () => { + expect(readTraceContext(TRACEPARENT, `a=${'b'.repeat(600)}`)).toEqual({ + traceparent: TRACEPARENT, + }) + }) + + test('round-trips through request headers', () => { + const trace = { traceparent: TRACEPARENT, tracestate: 'k=v' } + + expect(readTraceHeaders(new Headers(traceHeaders(trace)))).toEqual(trace) + expect(traceHeaders(undefined)).toEqual({}) + }) +}) diff --git a/packages/core/src/trace.ts b/packages/core/src/trace.ts new file mode 100644 index 0000000..152e81a --- /dev/null +++ b/packages/core/src/trace.ts @@ -0,0 +1,63 @@ +import { Option, Schema } from 'effect' + +import type { JsonValue } from './types' + +/** + * W3C trace context from the caller that started a run. The field names are the W3C header names, + * so the object is also a carrier for OpenTelemetry `propagation.extract`. + */ +export const TraceContextSchema = Schema.Struct({ + traceparent: Schema.String, + tracestate: Schema.optional(Schema.String), +}) + +export type TraceContext = Schema.Schema.Type + +const decodeTraceContext = Schema.decodeUnknownOption(TraceContextSchema) + +const TRACEPARENT = /^00-([0-9a-f]{32})-([0-9a-f]{16})-[0-9a-f]{2}$/ +const INVALID_TRACE_ID = '0'.repeat(32) +const INVALID_PARENT_ID = '0'.repeat(16) +const MAX_TRACESTATE_LENGTH = 512 + +/** + * A version-00 `traceparent`, with its `tracestate` when that is within the W3C length limit. + * Anything else is not a trace context, so callers drop it rather than store it. + */ +export function readTraceContext( + traceparent: string | null | undefined, + tracestate?: string | null, +): TraceContext | undefined { + const match = traceparent ? TRACEPARENT.exec(traceparent.trim()) : null + + if (!match || match[1] === INVALID_TRACE_ID || match[2] === INVALID_PARENT_ID) { + return undefined + } + + const parent = match[0] + const state = tracestate?.trim() + + return state && state.length <= MAX_TRACESTATE_LENGTH + ? { traceparent: parent, tracestate: state } + : { traceparent: parent } +} + +/** The trace context recorded on `runtime.run.started`, or null when the run has none. */ +export function readStoredTrace(value: JsonValue | undefined): TraceContext | null { + return value === undefined ? null : Option.getOrNull(decodeTraceContext(value)) +} + +export function readTraceHeaders(headers: Headers): TraceContext | undefined { + return readTraceContext(headers.get('traceparent'), headers.get('tracestate')) +} + +/** The W3C headers for an outgoing request. */ +export function traceHeaders(trace: TraceContext | undefined): Record { + if (!trace) { + return {} + } + + return trace.tracestate + ? { traceparent: trace.traceparent, tracestate: trace.tracestate } + : { traceparent: trace.traceparent } +} diff --git a/packages/runtime/src/helpers.ts b/packages/runtime/src/helpers.ts index 491fb6e..0b406a1 100644 --- a/packages/runtime/src/helpers.ts +++ b/packages/runtime/src/helpers.ts @@ -27,6 +27,7 @@ import { type JsonValue, type OutstandingEffect, type SecretContext, + type TraceContext, type RunCursor, type SchemaInput, type VersionedDefinition, @@ -41,6 +42,7 @@ export interface WorkerTaskExtras { /** Progress from the previous attempt's last heartbeat. */ readonly details?: JsonValue | null readonly secrets?: SecretContext + readonly trace?: TraceContext | null } export function workerTask( @@ -65,6 +67,10 @@ export function workerTask( task.secrets = extras.secrets } + if (extras.trace) { + task.trace = extras.trace + } + return task } diff --git a/packages/runtime/src/local-effect-executor.ts b/packages/runtime/src/local-effect-executor.ts index 5466470..d9eb578 100644 --- a/packages/runtime/src/local-effect-executor.ts +++ b/packages/runtime/src/local-effect-executor.ts @@ -16,6 +16,7 @@ import { type RunLauncher, type RunLaunchSource, type SecretContext, + type TraceContext, } from '@looms/core' import { dispatchEffect } from './dispatch-effect' @@ -42,12 +43,17 @@ export interface LocalEffectExecutorOptions { readonly launcher: RunLauncher } -export interface ExecuteLocalEffectsInput { - readonly runId: string - /** Inherited by runs the effects launch. */ +/** Run-level values every effect sees. Runs the effects launch inherit them unless the launch overrides them. */ +export interface RunContext { readonly correlationId: string readonly attributes: RunAttributes readonly secrets: SecretContext + readonly trace: TraceContext | null +} + +export interface ExecuteLocalEffectsInput { + readonly runId: string + readonly context: RunContext readonly outstanding: ReadonlyArray readonly attempts: ReadonlyMap readonly now: number @@ -78,9 +84,7 @@ function runLaunchedPayload(args: LaunchRunArgs, targetRunId: string, effectId: function launcherFor( launcher: RunLauncher, runId: string, - correlationId: string, - attributes: RunAttributes, - secrets: SecretContext, + context: RunContext, item: OutstandingEffect, launched: EventEnvelope[], ): RunLauncher { @@ -101,9 +105,10 @@ function launcherFor( ...args, runId: targetRunId, launchedBy, - correlationId, - attributes: args.attributes ?? attributes, - secrets: args.secrets ?? secrets, + correlationId: context.correlationId, + attributes: args.attributes ?? context.attributes, + secrets: args.secrets ?? context.secrets, + trace: args.trace ?? context.trace ?? undefined, }) .pipe( Effect.tap(() => @@ -114,7 +119,7 @@ function launcherFor( payload: runLaunchedPayload(args, targetRunId, item.effectId), threadId: item.threadId, causationId: item.causingEventId, - correlationId, + correlationId: context.correlationId, origin: { type: 'thread', threadId: item.threadId }, }), ) @@ -213,23 +218,16 @@ export function executeLocalEffects( registry, services, definitionResolver, - launcherFor( - launcher, - input.runId, - input.correlationId, - input.attributes, - input.secrets, - item, - launched, - ), + launcherFor(launcher, input.runId, input.context, item, launched), item.effect, { effectId: item.effectId, runId: input.runId, threadId: item.threadId, causingEventId: item.causingEventId, - attributes: input.attributes, - secrets: input.secrets, + attributes: input.context.attributes, + secrets: input.context.secrets, + trace: input.context.trace, signal: controller.signal, emit: appendLive, }, diff --git a/packages/runtime/src/looms.ts b/packages/runtime/src/looms.ts index efe2927..88e462f 100644 --- a/packages/runtime/src/looms.ts +++ b/packages/runtime/src/looms.ts @@ -117,6 +117,7 @@ export interface Looms { idempotencyKey?: string attributes?: StartRunArgs['attributes'] secrets?: StartRunArgs['secrets'] + trace?: StartRunArgs['trace'] }, ): Promise /** Start a run by `{ kind, definitionName }` when you only have names (HTTP bodies, CLIs). */ @@ -130,6 +131,7 @@ export interface Looms { idempotencyKey?: string attributes?: StartRunArgs['attributes'] secrets?: StartRunArgs['secrets'] + trace?: StartRunArgs['trace'] }): Promise /** Commit a run's start and schedule a wake without waiting for it. */ launchRun(args: { @@ -142,6 +144,7 @@ export interface Looms { idempotencyKey?: string attributes?: StartRunArgs['attributes'] secrets?: StartRunArgs['secrets'] + trace?: StartRunArgs['trace'] }): Promise getRun(runId: string): Promise getEvents(runId: string, options?: { fromSeq?: number; limit?: number }): Promise @@ -300,6 +303,7 @@ export function createLooms(options: CreateLoomsOptions = {}): Looms { idempotencyKey: startOpts?.idempotencyKey, attributes: startOpts?.attributes, secrets: startOpts?.secrets, + trace: startOpts?.trace, }), ), getRun: (runId) => runEffect((i) => i.runtime.getRun(runId)), diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index aa02aea..a96964b 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -1,4 +1,4 @@ -import { Clock, Effect, ManagedRuntime, Predicate } from 'effect' +import { Clock, Effect, ManagedRuntime, Predicate, type Types } from 'effect' import { cachedDefinitionResolver, @@ -18,6 +18,7 @@ import { unavailableRunLauncher, planFork, project, + readTraceContext, replayTo, sameStringRecord, summarizeRun, @@ -27,8 +28,10 @@ import { type EventInput, type EventOrigin, type RunLauncher, + type RunStartedPayload, type RunState, type SignalInterruptRule, + type TraceContext, type InvalidEventError, } from '@looms/core' @@ -127,10 +130,11 @@ interface StartedPayloadInput { readonly requestedThreadId: string | null readonly launchedBy: StartRunArgs['launchedBy'] readonly attributes: StartRunArgs['attributes'] + readonly trace: TraceContext | undefined } -function startedPayload(input: StartedPayloadInput) { - const payload = { +function startedPayload(input: StartedPayloadInput): RunStartedPayload { + const payload: Types.Mutable = { rootThreadId: input.threadId, kind: input.kind, definitionName: input.definitionName, @@ -139,10 +143,19 @@ function startedPayload(input: StartedPayloadInput) { requestedThreadId: input.requestedThreadId, } - const launched = input.launchedBy ? { ...payload, launchedBy: input.launchedBy } : payload - const attributes = input.attributes ?? {} + if (input.launchedBy) { + payload.launchedBy = input.launchedBy + } + + if (input.attributes && hasEntries(input.attributes)) { + payload.attributes = input.attributes + } + + if (input.trace) { + payload.trace = input.trace + } - return hasEntries(attributes) ? { ...launched, attributes } : launched + return payload } function reconcileRunSecrets( @@ -369,6 +382,7 @@ export function createRuntime() + +const observe = defineEffect({ + type: 'traced.observe', + execute: (_input, ctx) => { + seen.set(ctx.runId, ctx.trace) + return [{ type: 'traced.observed', payload: {}, threadId: ctx.threadId }] + }, +}) + +const remote = defineEffect({ type: 'traced.remote', execute: () => [] }) + +const launch = defineEffect({ + type: 'traced.launch', + input: TracedInput, + execute: (input, ctx) => + Effect.gen(function* () { + const launcher = yield* RunLauncherTag + + yield* launcher.launch({ + runId: `${ctx.runId}-child`, + kind: 'traced', + definitionName: 'traced', + input: { launch: false, override: false }, + trace: input.override ? OVERRIDE : undefined, + }) + + return [{ type: 'traced.launched', payload: {}, threadId: ctx.threadId }] + }), +}) + +const tracedThread = defineThread({ + kind: 'traced', + input: TracedInput, + initialState: (ctx) => ({ + launch: ctx.input?.launch ?? false, + override: ctx.input?.override ?? false, + observed: false, + launched: false, + }), + step: (state, event) => { + switch (event.type) { + case 'traced.observed': + return { ...state, observed: true } + case 'traced.launched': + return { ...state, launched: true } + default: + return state + } + }, + effects: (state) => { + if (!state.observed) { + return [invoke(observe, {}, 'observe'), invoke(remote, {}, 'remote')] + } + + if (state.launch && !state.launched) { + return [invoke(launch, { launch: false, override: state.override }, 'launch')] + } + + return [complete({ ok: true })] + }, +}) + +const tracedModule = defineRuntimeModule({ + namespace: 'traced', + protocolVersion: '1.0.0', + events: catalog, + threads: { traced: tracedThread }, + effects: { observe, remote, launch }, + definitions: [{ kind: 'traced', name: 'traced', version: 'v1', input: TracedInput }], +}) + +const traced = { kind: 'traced', name: 'traced', version: 'v1' } + +function tracedLooms(dispatched: EffectWorkerTask[] = []) { + return createLooms({ + modules: [tracedModule], + worker: { handles: ['traced.remote'], dispatch: (task) => void dispatched.push(task) }, + }) +} + +describe('trace propagation', () => { + test('carries the caller trace from the start request to effects, workers, and launched runs', async () => { + const dispatched: EffectWorkerTask[] = [] + const looms = tracedLooms(dispatched) + + const response = await looms.fetch( + new Request('http://looms.test/runs', { + method: 'POST', + headers: { 'content-type': 'application/json', ...CALLER }, + body: JSON.stringify({ + kind: 'traced', + definitionName: 'traced', + runId: 'run_trace', + input: { launch: true, override: false }, + }), + }), + ) + + expect(response?.status).toBe(200) + + const started = (await looms.getEvents('run_trace')).find( + (event) => event.type === 'runtime.run.started', + ) + + expect(started?.payload).toMatchObject({ trace: CALLER }) + expect(seen.get('run_trace')).toEqual(CALLER) + expect(dispatched.find((task) => task.runId === 'run_trace')?.trace).toEqual(CALLER) + expect((await looms.getRun('run_trace-child')).startIdentity?.trace).toEqual(CALLER) + + const deadline = Date.now() + 2_000 + + while (!seen.has('run_trace-child') && Date.now() < deadline) { + await Bun.sleep(5) + } + + expect(seen.get('run_trace-child')).toEqual(CALLER) + + await looms.stop() + }) + + test('a launch can name its own trace context', async () => { + const looms = tracedLooms() + + await looms.start( + traced, + { launch: true, override: true }, + { + runId: 'run_override', + trace: CALLER, + }, + ) + + expect((await looms.getRun('run_override-child')).startIdentity?.trace).toEqual(OVERRIDE) + await looms.stop() + }) + + test('drops an invalid traceparent instead of storing it', async () => { + const looms = tracedLooms() + + await looms.start( + traced, + { launch: false, override: false }, + { + runId: 'run_invalid', + trace: { traceparent: 'not-a-trace' }, + }, + ) + + const started = (await looms.getEvents('run_invalid')).find( + (event) => event.type === 'runtime.run.started', + ) + + expect(Predicate.isObject(started?.payload) && 'trace' in started.payload).toBe(false) + expect(seen.get('run_invalid')).toBeNull() + await looms.stop() + }) + + test('a retried start from another trace is the same run', async () => { + const looms = tracedLooms() + const input = { launch: false, override: false } + + await looms.start(traced, input, { runId: 'run_retry', idempotencyKey: 'k', trace: CALLER }) + + const retried = await looms.start(traced, input, { + runId: 'run_retry', + idempotencyKey: 'k', + trace: OVERRIDE, + }) + + expect(retried.runId).toBe('run_retry') + expect((await looms.getRun('run_retry')).startIdentity?.trace).toEqual(CALLER) + await looms.stop() + }) + + test('a workflow child run inherits the parent trace', async () => { + const child = defineWorkflow({ name: 'trace-child', nodes: [{ id: 'out', run: () => 1 }] }) + + const parent = defineWorkflow({ + name: 'trace-parent', + childExecution: 'run', + nodes: [{ id: 'spawn', run: (ctx) => ctx.spawn(child, {}) }], + }) + + const looms = createLooms({ modules: [workflow({ definitions: [child, parent] })] }) + const started = await looms.start(parent, {}, { trace: CALLER }) + const deadline = Date.now() + 2_000 + let state = await looms.getRun(started.runId) + + while (state.status !== 'completed' && Date.now() < deadline) { + await Bun.sleep(5) + state = await looms.getRun(started.runId) + } + + const launched = (await looms.getEvents(started.runId)).find( + (event) => event.type === 'runtime.run.launched', + ) + + const childRunId = + launched && Predicate.isObject(launched.payload) && Predicate.isString(launched.payload.runId) + ? launched.payload.runId + : '' + + expect(state.status).toBe('completed') + expect((await looms.getRun(childRunId)).startIdentity?.trace).toEqual(CALLER) + await looms.stop() + }) +}) diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index b43750b..7ffb3f8 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -21,6 +21,7 @@ import type { RunLaunchSource, RunAttributes, SecretContext, + TraceContext, EventStoreTruncationError, JsonValue, ProjectionDefinition, @@ -107,6 +108,8 @@ export interface EffectWorkerTask { readonly input: JsonValue /** Progress from the previous attempt's last heartbeat, when it reported any. */ readonly details?: JsonValue + /** The run's W3C trace context, for parenting the worker's spans. */ + readonly trace?: TraceContext /** * Present only on the in-memory dispatch. A transport that persists the task must leave this out * and let the worker read the secret context through an authenticated callback. @@ -142,6 +145,11 @@ export interface StartRunArgs { * Omitting it on a retry keeps the context already stored. */ secrets?: SecretContext + /** + * The caller's W3C trace context, recorded on `runtime.run.started` when it is a valid version-00 + * `traceparent`. It is not part of the start identity, so a retry from another trace is not a conflict. + */ + trace?: TraceContext } export interface StartResult { diff --git a/packages/runtime/src/wake-engine.ts b/packages/runtime/src/wake-engine.ts index c5105e4..98dce44 100644 --- a/packages/runtime/src/wake-engine.ts +++ b/packages/runtime/src/wake-engine.ts @@ -435,9 +435,12 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake }, { runId, - correlationId: cursor.state.startIdentity?.correlationId ?? runId, - attributes: cursor.state.attributes ?? {}, - secrets, + context: { + correlationId: cursor.state.startIdentity?.correlationId ?? runId, + attributes: cursor.state.attributes ?? {}, + secrets, + trace: cursor.state.startIdentity?.trace ?? null, + }, outstanding, attempts, now, diff --git a/packages/runtime/src/worker-effect-dispatch.ts b/packages/runtime/src/worker-effect-dispatch.ts index 9db4d0e..8652ef6 100644 --- a/packages/runtime/src/worker-effect-dispatch.ts +++ b/packages/runtime/src/worker-effect-dispatch.ts @@ -152,6 +152,7 @@ export function planWorkerEffects(input: { task: workerTask(input.runId, item, attempt, { details: existing?.heartbeatDetails, secrets: input.secrets, + trace: input.state.startIdentity?.trace, }), scheduleToStartDeadlineAt: retry?.scheduleToStartTimeoutMs ? input.now + retry.scheduleToStartTimeoutMs From 6c874b7ef7ee5fbf8a714a1829e09676751eb017 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 14:32:06 +0200 Subject: [PATCH 19/23] fix(cloudflare): authorize socket upgrades with the run's attributes The Durable Object accepts run WebSockets outside the fetch handler, so it called authorize without the attributes every other run route passes. An authorizer that checks the tenant saw none. runRouteInfo now builds RouteInfo for both paths from the same start-event read. Co-authored-by: Cursor --- packages/cloudflare/src/durable-object.ts | 6 +- packages/cloudflare/src/websocket.test.ts | 67 ++++++++++++++++++++ packages/cloudflare/src/websocket.ts | 13 +++- packages/runtime/src/index.ts | 1 + packages/runtime/src/server.ts | 1 + packages/runtime/src/server/fetch-handler.ts | 41 +----------- packages/runtime/src/server/run-route.ts | 59 +++++++++++++++++ 7 files changed, 145 insertions(+), 43 deletions(-) create mode 100644 packages/runtime/src/server/run-route.ts diff --git a/packages/cloudflare/src/durable-object.ts b/packages/cloudflare/src/durable-object.ts index 8b85764..b865a35 100644 --- a/packages/cloudflare/src/durable-object.ts +++ b/packages/cloudflare/src/durable-object.ts @@ -18,6 +18,7 @@ import type { Projector } from '@looms/projectors' import { defaultAuthorize, type Authorize, + type LoomsRuntime, type StartResult, type StartRunArgs, } from '@looms/runtime' @@ -132,7 +133,7 @@ export abstract class LoomsDurableObject extends DurableObject { if (isRunSocketUpgrade(req)) { - return this.getCell().then(() => this.acceptRunSocket(req)) + return this.getCell().then((cell) => this.acceptRunSocket(req, cell.runtime)) } return this.getCell().then((cell) => cell.fetch(req)) @@ -151,7 +152,7 @@ export abstract class LoomsDurableObject extends DurableObject { + private acceptRunSocket(req: Request, runtime: LoomsRuntime): Promise { const store = this.store const runId = this.ctx.id.name @@ -162,6 +163,7 @@ export abstract class LoomsDurableObject extends DurableObject { const response = await Effect.runPromise( openRunSocket({ + runtime, ctx, store, request: upgrade('http://looms.test/runs/run_1/events'), @@ -134,6 +140,7 @@ describe('hibernating run sockets', () => { await Effect.runPromise( openRunSocket({ + runtime, ctx: new FakeState(), store, request: upgrade('http://looms.test/runs/run_1/events?fromSeq=2&threadId=thr_b'), @@ -156,11 +163,24 @@ describe('hibernating run sockets', () => { throw new Error('memory store does not trim') } + const snapshotStore = await Effect.runPromise(makeMemorySnapshotStore) + + await Effect.runPromise( + snapshotStore.save({ + runId: 'run_1', + cursor: 2, + stateHash: 'test', + takenAt: 0, + state: emptyRunState('run_1'), + }), + ) + await Effect.runPromise(store.trim('run_1', 2)) const server = new FakeSocket() await Effect.runPromise( openRunSocket({ + runtime: createRuntime({ modules: [], snapshotStore }), ctx: new FakeState(), store, request: upgrade('http://looms.test/runs/run_1/events?fromSeq=1'), @@ -185,6 +205,7 @@ describe('hibernating run sockets', () => { await Effect.runPromise( openRunSocket({ + runtime, ctx: new FakeState(), store, request: upgrade('http://looms.test/runs/run_1/events'), @@ -208,6 +229,7 @@ describe('hibernating run sockets', () => { const response = await Effect.runPromise( openRunSocket({ + runtime, ctx, store, request: upgrade('http://looms.test/runs/run_1/events'), @@ -220,11 +242,55 @@ describe('hibernating run sockets', () => { expect(ctx.accepted).toHaveLength(0) }) + test('authorizes with the run attributes like every other run route', async () => { + const store = await Effect.runPromise(makeMemoryEventStore) + const routes: RouteInfo[] = [] + + const started = createEvent('run_1', { + type: 'runtime.run.started', + payload: { + rootThreadId: 'thr_a', + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v1', + input: null, + requestedThreadId: null, + attributes: { tenant: 'acme' }, + }, + threadId: 'thr_a', + origin: { type: 'system' }, + }) + + await Effect.runPromise(store.append('run_1', [started], { expectedTail: 0 })) + + const response = await Effect.runPromise( + openRunSocket({ + runtime, + ctx: new FakeState(), + store, + request: upgrade('http://looms.test/runs/run_1/events'), + runId: 'run_1', + createPair: pairFrom(new FakeSocket(), new FakeSocket()), + authorize: (_request, route) => { + routes.push(route) + return Effect.succeed(authAllowed) + }, + }), + ) + + expect(response.status).toBe(101) + + expect(routes).toEqual([ + { access: 'read', name: 'runs.events', runId: 'run_1', attributes: { tenant: 'acme' } }, + ]) + }) + test('rejects an invalid Last-Event-ID and a mismatched run', async () => { const store = await seededStore() const invalid = await Effect.runPromise( openRunSocket({ + runtime, ctx: new FakeState(), store, request: new Request('http://looms.test/runs/run_1/events', { @@ -238,6 +304,7 @@ describe('hibernating run sockets', () => { const missing = await Effect.runPromise( openRunSocket({ + runtime, ctx: new FakeState(), store, request: upgrade('http://looms.test/runs/other/events'), diff --git a/packages/cloudflare/src/websocket.ts b/packages/cloudflare/src/websocket.ts index e3746d8..50a490a 100644 --- a/packages/cloudflare/src/websocket.ts +++ b/packages/cloudflare/src/websocket.ts @@ -16,7 +16,9 @@ import { defaultAuthorize, loadSubscriptionOpening, readSubscriptionBounds, + runRouteInfo, type Authorize, + type LoomsRuntime, } from '@looms/runtime' const EVENTS_PATH = /^\/runs\/([^/]+)\/events\/?$/ @@ -29,6 +31,7 @@ export interface HibernatingSocketState { export interface OpenRunSocketOptions { readonly ctx: HibernatingSocketState + readonly runtime: LoomsRuntime readonly store: EventStore readonly request: Request readonly runId: string @@ -84,11 +87,17 @@ export function openRunSocket(options: OpenRunSocketOptions): Effect.Effect { ) } -/** - * Attributes are immutable and recorded on `runtime.run.started`, which is seq 1 of every - * untrimmed log, forks included. Only a trimmed log needs the snapshot-backed fold. - */ -function readStoredAttributes( - runtime: LoomsRuntime, - store: EventStore, - runId: string, -): Effect.Effect { - const folded = runtime.getRun(runId).pipe( - Effect.provideService(EventStoreTag, store), - Effect.map((state) => state.attributes), - ) - - return store.read(runId, { fromSeq: 1, limit: 1 }).pipe( - Effect.flatMap(([first]) => { - if (first === undefined) { - return Effect.succeed(noAttributes) - } - - if (first.type === 'runtime.run.started' && isJsonObject(first.payload)) { - return Effect.succeed(readAttributes(first.payload.attributes)) - } - - return folded - }), - Effect.catch(() => folded), - ) -} - function authorizeRoute( req: Request, matched: MatchedRoute, diff --git a/packages/runtime/src/server/run-route.ts b/packages/runtime/src/server/run-route.ts new file mode 100644 index 0000000..ae04618 --- /dev/null +++ b/packages/runtime/src/server/run-route.ts @@ -0,0 +1,59 @@ +import { Effect } from 'effect' + +import { + EventStoreTag, + hasEntries, + isJsonObject, + readAttributes, + type EventStore, + type RunAttributes, +} from '@looms/core' + +import type { LoomsRuntime } from '../runtime' +import type { AuthorizedAccess, RouteInfo } from './auth' + +const noAttributes: RunAttributes = {} + +/** + * Attributes are immutable and recorded on `runtime.run.started`, which is seq 1 of every + * untrimmed log, forks included. Only a trimmed log needs the snapshot-backed fold. + */ +export function readStoredAttributes( + runtime: LoomsRuntime, + store: EventStore, + runId: string, +): Effect.Effect { + const folded = runtime.getRun(runId).pipe( + Effect.provideService(EventStoreTag, store), + Effect.map((state) => state.attributes), + ) + + return store.read(runId, { fromSeq: 1, limit: 1 }).pipe( + Effect.flatMap(([first]) => { + if (first === undefined) { + return Effect.succeed(noAttributes) + } + + if (first.type === 'runtime.run.started' && isJsonObject(first.payload)) { + return Effect.succeed(readAttributes(first.payload.attributes)) + } + + return folded + }), + Effect.catch(() => folded), + ) +} + +/** + * What `authorize` sees for a request about one run: the run's attributes are included when it has + * any. Every path that serves a run, including a WebSocket upgrade, authorizes through this. + */ +export function runRouteInfo( + runtime: LoomsRuntime, + store: EventStore, + route: { readonly access: AuthorizedAccess; readonly name: string; readonly runId: string }, +): Effect.Effect { + return readStoredAttributes(runtime, store, route.runId).pipe( + Effect.map((attributes) => (hasEntries(attributes) ? { ...route, attributes } : route)), + ) +} From d77e5b8b8f232e3cca13e1061e947604aef3cc3f Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 14:33:19 +0200 Subject: [PATCH 20/23] feat(cloudflare): echo a subprotocol grant on the socket upgrade A browser that offers a grant in Sec-WebSocket-Protocol fails the handshake unless the response selects it. grantSubprotocol returns the offered v1. value so the Durable Object and other hosts can echo it. Co-authored-by: Cursor --- .changeset/run-grants.md | 2 +- docs/production.md | 7 ++++--- packages/cloudflare/src/websocket.test.ts | 20 ++++++++++++++++++++ packages/cloudflare/src/websocket.ts | 9 ++++++++- packages/runtime/src/index.ts | 1 + packages/runtime/src/server.ts | 1 + packages/runtime/src/server/grant.ts | 10 +++++++++- 7 files changed, 44 insertions(+), 6 deletions(-) diff --git a/.changeset/run-grants.md b/.changeset/run-grants.md index 0d308fe..4e1f007 100644 --- a/.changeset/run-grants.md +++ b/.changeset/run-grants.md @@ -4,4 +4,4 @@ feat(runtime): authorize browser event streams with a signed run grant -`grantAuth` checks a short-lived HMAC grant from the `grant` query parameter or a `v1.` WebSocket subprotocol. The grant allows one principal to read one run, which is the credential a browser can send on EventSource or WebSocket. +`grantAuth` checks a short-lived HMAC grant from the `grant` query parameter or a `v1.` WebSocket subprotocol. The grant allows one principal to read one run, which is the credential a browser can send on EventSource or WebSocket. The Durable Object socket echoes a subprotocol grant on the upgrade, and `grantSubprotocol` gives other hosts the value to echo. diff --git a/docs/production.md b/docs/production.md index e94163d..55a24a7 100644 --- a/docs/production.md +++ b/docs/production.md @@ -13,7 +13,7 @@ coalesced. A deployment, eviction, or alarm retry may repeat a wake; event preconditions make that safe. Cross-object queries and transactions are not provided. Build global run lists with a projector. -Live subscribers on a Durable Object should open a WebSocket (`Upgrade: websocket` on `GET /runs/:id/events`). The object hibernates the socket, answers `ping` with `pong` without waking, and fans out events when the log is appended. SSE stays available, including on `looms.serve`, but its keepalive keeps a Durable Object awake. Writes stay on HTTP. A browser still cannot send `Authorization` on the upgrade; grant tokens are a separate change. An idle-socket billed-duration check on a real Cloudflare account has not been run. +Live subscribers on a Durable Object should open a WebSocket (`Upgrade: websocket` on `GET /runs/:id/events`). The object hibernates the socket, answers `ping` with `pong` without waking, and fans out events when the log is appended. SSE stays available, including on `looms.serve`, but its keepalive keeps a Durable Object awake. Writes stay on HTTP. A browser cannot send `Authorization` on the upgrade, so pass a read grant (see `grantAuth` below); `@looms/client` puts `grant` on the socket URL. An idle-socket billed-duration check on a real Cloudflare account has not been run. A workflow can launch children with `childExecution: 'run'`. Each child is its own run, addressed by a deterministic id, and reports `runtime.child.settled` back to the parent. Cancelling the parent cancels those children. `childClose: 'wait'` records `runtime.child.close.requested` and cancels the parent only after the children reach a terminal status. `childClose: 'terminate'` cancels them as the parent closes. Children stay threads in the parent run unless `childExecution` is set. @@ -145,8 +145,9 @@ in `Sec-WebSocket-Protocol` when the query parameter is absent. The grant names one run, one principal, and `read` access. It does not authorize writes, worker callbacks, operations, the debugger, or a different run. `exp` is an exclusive Unix millisecond timestamp. A host that accepts the grant as a WebSocket -subprotocol must echo that protocol on the upgrade response; the query parameter -needs no echo. +subprotocol must echo that protocol on the upgrade response (`grantSubprotocol` +returns it); the Durable Object socket already does. The query parameter needs no +echo. ```ts import { Effect } from 'effect' diff --git a/packages/cloudflare/src/websocket.test.ts b/packages/cloudflare/src/websocket.test.ts index c04f123..dc7a538 100644 --- a/packages/cloudflare/src/websocket.test.ts +++ b/packages/cloudflare/src/websocket.test.ts @@ -285,6 +285,26 @@ describe('hibernating run sockets', () => { ]) }) + test('echoes a grant offered as a WebSocket subprotocol', async () => { + const store = await seededStore() + + const response = await Effect.runPromise( + openRunSocket({ + runtime, + ctx: new FakeState(), + store, + request: new Request('http://looms.test/runs/run_1/events', { + headers: { upgrade: 'websocket', 'sec-websocket-protocol': 'chat, v1.abc.def' }, + }), + runId: 'run_1', + createPair: pairFrom(new FakeSocket(), new FakeSocket()), + }), + ) + + expect(response.status).toBe(101) + expect(response.headers.get('sec-websocket-protocol')).toBe('v1.abc.def') + }) + test('rejects an invalid Last-Event-ID and a mismatched run', async () => { const store = await seededStore() diff --git a/packages/cloudflare/src/websocket.ts b/packages/cloudflare/src/websocket.ts index 50a490a..83e0537 100644 --- a/packages/cloudflare/src/websocket.ts +++ b/packages/cloudflare/src/websocket.ts @@ -14,6 +14,7 @@ import { import { authResponse, defaultAuthorize, + grantSubprotocol, loadSubscriptionOpening, readSubscriptionBounds, runRouteInfo, @@ -143,7 +144,13 @@ export function openRunSocket(options: OpenRunSocketOptions): Effect.Effect Date: Fri, 23 Oct 2026 14:39:37 +0200 Subject: [PATCH 21/23] test(runtime): measure retryAfterMs from before the run starts The retry is scheduled from the wake's clock, which is read before the event timestamp, so comparing against the event could fail by a millisecond. The assertion also no longer skips when the payload is missing. Co-authored-by: Cursor --- packages/runtime/src/worker-protocol.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/runtime/src/worker-protocol.test.ts b/packages/runtime/src/worker-protocol.test.ts index 3f3f8dc..7bf30c8 100644 --- a/packages/runtime/src/worker-protocol.test.ts +++ b/packages/runtime/src/worker-protocol.test.ts @@ -737,19 +737,19 @@ describe('durable external effect workers', () => { ], }) + const before = Date.now() const started = await looms.start(workerDefinition, {}) - const scheduled = (await looms.getEvents(started.runId)).find( + const payload = (await looms.getEvents(started.runId)).find( (event) => event.type === 'runtime.effect.retry.scheduled', - ) - - const payload = scheduled?.payload + )?.payload - expect(scheduled).toBeDefined() + const nextAttemptAt = + Predicate.isReadonlyObject(payload) && !Array.isArray(payload) + ? payload.nextAttemptAt + : undefined - if (payload && !Array.isArray(payload) && Predicate.isReadonlyObject(payload)) { - expect(payload.nextAttemptAt).toBeGreaterThanOrEqual((scheduled?.ts ?? 0) + 60_000) - } + expect(nextAttemptAt).toBeGreaterThanOrEqual(before + 60_000) await looms.stop() }) From 1e1ece32ebcbca5e388875c0de867a56cda52525 Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Fri, 23 Oct 2026 15:04:19 +0200 Subject: [PATCH 22/23] feat(cloudflare): rebuild the queue transport on the worker callback API The producer now sends trace context and attempt deadlines, and offloads inputs over 100 KB to the blob store as inputRef. Consumers use the worker callback client, read secrets through the secrets callback, ack after started, and report complete, fail, or cancelled once. A pull consumer with a Cloudflare HTTP source batches acks, backs off when idle, reports the backlog, and drains on stop. The in-memory queue models leases, retries, and dead letters. In the runtime, an acknowledged dispatch rejection goes through the retry policy. Creating a runtime fails when an acknowledged worker claims an effect without a schedule-to-start timeout. The in-flight cap counts only outstanding worker attempts. Co-authored-by: Cursor --- .changeset/queue-worker.md | 6 +- bun.lock | 1 + docs/production.md | 66 +- packages/cloudflare/package.json | 1 + packages/cloudflare/src/index.ts | 40 +- packages/cloudflare/src/memory-queue.ts | 105 +++ packages/cloudflare/src/queue-consumer.ts | 240 +++++++ packages/cloudflare/src/queue-message.ts | 74 +++ packages/cloudflare/src/queue-pull.ts | 281 ++++++++ packages/cloudflare/src/queue-worker.test.ts | 599 ++++++++++++------ packages/cloudflare/src/queue-worker.ts | 418 +++--------- packages/runtime/src/helpers.ts | 29 +- packages/runtime/src/index.ts | 1 + packages/runtime/src/planners.test.ts | 1 + packages/runtime/src/runtime.ts | 4 + packages/runtime/src/types.ts | 18 +- packages/runtime/src/wake-engine.ts | 7 +- .../runtime/src/worker-effect-dispatch.ts | 79 ++- packages/runtime/src/worker-protocol.test.ts | 85 ++- 19 files changed, 1494 insertions(+), 561 deletions(-) create mode 100644 packages/cloudflare/src/memory-queue.ts create mode 100644 packages/cloudflare/src/queue-consumer.ts create mode 100644 packages/cloudflare/src/queue-message.ts create mode 100644 packages/cloudflare/src/queue-pull.ts diff --git a/.changeset/queue-worker.md b/.changeset/queue-worker.md index ea61b99..d3b2f78 100644 --- a/.changeset/queue-worker.md +++ b/.changeset/queue-worker.md @@ -2,6 +2,8 @@ '@swirls/looms': minor --- -feat(cloudflare): add native Cloudflare Queues effect worker and push/pull consumers +feat(cloudflare): run worker effects over Cloudflare Queues -Adds `cloudflareQueueWorker` producer implementing `EffectWorker` with `dispatchSemantics: 'acknowledged'`, queue push/pull consumers with single-retry authority and mid-flight cancellation abort, and `maxInFlightWorkerEffects` fairness guard per run. +`cloudflareQueueWorker` sends each attempt to a queue as JSON, with the run's trace context and the attempt's deadlines; inputs over 100 KB travel as a blob reference and secrets stay off the queue. `queueConsumer` (push) and `createQueuePullConsumer` with `cloudflareQueuePullSource` (HTTP pull) run handlers that heartbeat, read secrets through the worker callback API, and stop when the run is cancelled. The consumer acks after `started`, so Looms owns retries; duplicate and stale deliveries are dropped, and dead-lettered messages are recovered by the schedule-to-start timeout. The pull consumer batches acks, backs off when idle, reports the backlog, and drains on `stop()`. `createMemoryQueue` gives tests the same lease and dead-letter behavior. + +In the runtime, `EffectWorker.dispatchSemantics: 'acknowledged'` fails a rejected dispatch under the retry policy instead of parking it as ambiguous, and creating a runtime fails if such a worker claims an effect without `retry.scheduleToStartTimeoutMs`. Worker tasks carry `deadlines`, and `maxInFlightWorkerEffects` caps the worker attempts one run has in flight. diff --git a/bun.lock b/bun.lock index d5649cb..3601304 100644 --- a/bun.lock +++ b/bun.lock @@ -250,6 +250,7 @@ }, "devDependencies": { "@cloudflare/workers-types": "^5.20260914.1", + "@looms/client": "workspace:*", "@looms/schedule": "workspace:*", "@looms/workflow": "workspace:*", "@types/bun": "^1.3.9", diff --git a/docs/production.md b/docs/production.md index c2b6004..5ebbeb4 100644 --- a/docs/production.md +++ b/docs/production.md @@ -87,8 +87,10 @@ context for a handler's span. ## Worker effect protocol -1. The runtime dispatches `{ runId, effectId, attempt, type, input, details? }`. +1. The runtime dispatches `{ runId, effectId, attempt, type, input, details?, trace?, deadlines? }`. `details` is the progress carried by the previous attempt's last heartbeat. + `deadlines` holds the schedule-to-start time and the start-to-close and + heartbeat timeouts from the effect's retry policy. 2. The worker calls `started` before the schedule-to-start deadline. 3. Long work calls `heartbeat`. Each accepted heartbeat extends the heartbeat deadline without exceeding start-to-close. A heartbeat may include `details`; @@ -106,27 +108,75 @@ that an attempt already in progress should stop: a queued dispatch cannot be rec and the host recorded nothing. Callbacks include the same `effectId` and `attempt`. Stale callbacks are ignored. -A dispatch transport failure is recorded as `ambiguous`: an operator must first +A worker with `dispatchSemantics: 'acknowledged'` promises that a rejected +`dispatch` sent nothing, so the attempt fails under the retry policy. Because its +transport can still lose a message it accepted, creating a runtime fails unless +every effect that worker claims sets `retry.scheduleToStartTimeoutMs`. For other +workers, a dispatch transport failure is recorded as `ambiguous`: an operator must first verify the destination's idempotency record, then use `retryEffect` or `POST /runs/:runId/effects/:effectId/retry`. Never blindly retry a non-idempotent effect. ### Cloudflare Queues transport -`cloudflareQueueWorker` connects Durable Objects to external workers or background fleets over Cloudflare Queues: +`cloudflareQueueWorker` sends each attempt to a Cloudflare Queue as JSON. Every +routed effect needs `retry.scheduleToStartTimeoutMs`. An input that would push the +message past 100 KB is stored in `blobs` and sent as `inputRef`; without a blob +store that dispatch fails. Secrets never go on the queue. ```ts import { cloudflareQueueWorker } from '@swirls/looms/cloudflare' const worker = cloudflareQueueWorker({ - routes: [{ handles: ['sandbox.exec', 'batch.convert'], queue: env.WORKER_QUEUE }], + routes: [{ handles: ['sandbox.exec'], queue: env.SANDBOX_QUEUE }], + blobs: r2BlobStore(env.BLOBS), }) + +createLooms({ modules, worker, maxInFlightWorkerEffects: 16 }) +``` + +A consumer runs handlers of the form `(task, ctx) => events`. `ctx.heartbeat(details)` +reports progress, `ctx.secrets()` reads the run's secret context through the +authenticated callback, and `ctx.signal` aborts when the run is cancelled. Throw a +`QueueTaskError` with `nonRetryable` or `retryAfterMs` to control the retry. + +```ts +import { createLoomsClient } from '@swirls/looms/client' +import { + cloudflareQueuePullSource, + createQueuePullConsumer, + queueConsumer, +} from '@swirls/looms/cloudflare' + +const callbacks = createLoomsClient({ baseUrl, workerCallbackToken }) +const handlers = { 'sandbox.exec': runSandbox } + +// In a Worker: export default { queue: queueConsumer({ callbacks, handlers, blobs }) } + +const consumer = createQueuePullConsumer({ + source: cloudflareQueuePullSource({ accountId, queueId, token }), + callbacks, + handlers, + blobs, + concurrency: 8, +}) + +process.once('SIGTERM', () => consumer.stop()) ``` -- **Acknowledged dispatch**: `dispatchSemantics: 'acknowledged'` treats a thrown send (including 429 rate limits) as an un-enqueued failure retried with backoff, eliminating ambiguous dispatch operator interventions. -- **One retry authority**: Consumers acknowledge the queue message immediately once the `started` callback succeeds. From that point on, Looms heartbeats and start-to-close deadlines govern execution liveness. -- **Fairness guard**: `maxInFlightWorkerEffects` caps concurrent in-flight dispatches per run, preventing a single fanout from saturating a shared queue. -- **In-flight cancellation**: Because enqueued messages cannot be recalled from Cloudflare Queues, in-flight attempts learn of cancellation from heartbeat responses and abort the task signal. +- The consumer acks a message once Looms accepts `started`. After that Looms owns + retries through heartbeat and start-to-close timeouts, so a long effect does not + depend on the queue's lease. A duplicate or stale delivery is acked without running. +- A message that cannot start yet is retried by the queue with backoff. One that + never starts lands in the dead-letter queue, and the schedule-to-start timeout + schedules a new attempt. +- A queue send that fails, such as a 429, fails the attempt under the retry policy. +- The pull consumer batches acks, backs off while the queue is empty or the API + errors, and reports `backlog()` from Cloudflare's `message_backlog_count`. + `stop()` stops pulling, waits for running attempts, and sends the last acks. +- `maxInFlightWorkerEffects` caps how many worker attempts one run has in flight, + so a wide fan-out cannot fill a shared queue. +- `createMemoryQueue()` has Cloudflare's lease, retry, and dead-letter behavior for tests. ## Inspection and repair diff --git a/packages/cloudflare/package.json b/packages/cloudflare/package.json index f2093ba..3e890c2 100644 --- a/packages/cloudflare/package.json +++ b/packages/cloudflare/package.json @@ -37,6 +37,7 @@ }, "devDependencies": { "@cloudflare/workers-types": "^5.20260914.1", + "@looms/client": "workspace:*", "@looms/schedule": "workspace:*", "@looms/workflow": "workspace:*", "@types/bun": "^1.3.9", diff --git a/packages/cloudflare/src/index.ts b/packages/cloudflare/src/index.ts index 711521a..eea4d2b 100644 --- a/packages/cloudflare/src/index.ts +++ b/packages/cloudflare/src/index.ts @@ -23,20 +23,40 @@ export { type DurableObjectNamespaceLike, type DurableObjectStubLike, } from './router' +export { createMemoryQueue, type MemoryQueue, type MemoryQueueOptions } from './memory-queue' export { - cloudflareQueueWorker, - createMemoryQueue, processQueueMessage, - queuePushConsumer, - type CloudflareQueueWorkerOptions, - type MemoryQueue, - type MessageBatchLike, + queueConsumer, + QueueTaskError, + type QueueBatchLike, type QueueConsumerOptions, + type QueueTask, + type QueueTaskContext, + type QueueTaskHandler, + type WorkerCallbackClient, +} from './queue-consumer' +export { + readQueueEffectMessage, type QueueEffectMessage, - type QueueLike, type QueueMessageLike, +} from './queue-message' +export { + cloudflareQueuePullSource, + createQueuePullConsumer, + QueuePullError, + type CloudflareQueuePullSourceOptions, + type PulledQueueMessage, + type QueueLeaseRetry, + type QueuePullConsumer, + type QueuePullConsumerOptions, + type QueuePullResult, + type QueuePullSource, +} from './queue-pull' +export { + cloudflareQueueWorker, + DEFAULT_MAX_QUEUE_MESSAGE_BYTES, + QueueDispatchError, + type CloudflareQueueWorkerOptions, + type QueueProducer, type QueueRoute, - type QueueSendOptions, - type QueueTaskContext, - type QueueTaskHandler, } from './queue-worker' diff --git a/packages/cloudflare/src/memory-queue.ts b/packages/cloudflare/src/memory-queue.ts new file mode 100644 index 0000000..16b3d5a --- /dev/null +++ b/packages/cloudflare/src/memory-queue.ts @@ -0,0 +1,105 @@ +import { Clock, Effect } from 'effect' + +import type { QueueBatchLike } from './queue-consumer' +import type { QueueEffectMessage } from './queue-message' +import type { PulledQueueMessage, QueuePullSource } from './queue-pull' +import type { QueueProducer } from './queue-worker' + +export interface MemoryQueueOptions { + /** Redeliveries before a message moves to the dead-letter list. Default 3, as on Cloudflare. */ + readonly maxRetries?: number +} + +/** A queue in process memory with Cloudflare's lease, retry, and dead-letter behavior. */ +export interface MemoryQueue extends QueueProducer, QueuePullSource { + /** Messages still held, leased or not. */ + size(): number + deadLetters(): ReadonlyArray + /** Lease visible messages as a push batch, the way a Worker's `queue` handler receives them. */ + batch(size?: number): Effect.Effect +} + +interface Entry { + readonly body: QueueEffectMessage + deliveries: number + visibleAt: number + leaseId: string | undefined +} + +export function createMemoryQueue(options: MemoryQueueOptions = {}): MemoryQueue { + const maxRetries = options.maxRetries ?? 3 + let entries: Entry[] = [] + const dead: QueueEffectMessage[] = [] + let nextLease = 0 + + const settleAt = ( + now: number, + acks: ReadonlyArray, + retries: ReadonlyArray<{ readonly leaseId: string; readonly delaySeconds: number }>, + ) => { + entries = entries.filter( + (entry) => entry.leaseId === undefined || !acks.includes(entry.leaseId), + ) + + for (const retry of retries) { + const entry = entries.find((candidate) => candidate.leaseId === retry.leaseId) + + if (entry) { + entry.leaseId = undefined + entry.visibleAt = now + retry.delaySeconds * 1_000 + } + } + } + + const pull = (batchSize: number, visibilityTimeoutMs: number) => + Effect.map(Clock.currentTimeMillis, (now) => { + const visible = entries.filter((entry) => entry.visibleAt <= now) + const exhausted = visible.filter((entry) => entry.deliveries > maxRetries) + + dead.push(...exhausted.map((entry) => entry.body)) + entries = entries.filter((entry) => !exhausted.includes(entry)) + + const leased = visible + .filter((entry) => entry.deliveries <= maxRetries) + .slice(0, batchSize) + .map((entry): PulledQueueMessage => { + nextLease += 1 + entry.deliveries += 1 + entry.visibleAt = now + visibilityTimeoutMs + entry.leaseId = `lease_${nextLease}` + + return { leaseId: entry.leaseId, body: entry.body, attempts: entry.deliveries } + }) + + return { messages: leased, backlog: entries.length } + }) + + const settle: QueuePullSource['settle'] = (acks, retries) => + Effect.map(Clock.currentTimeMillis, (now) => settleAt(now, acks, retries)) + + return { + send: (message) => { + entries.push({ body: message, deliveries: 0, visibleAt: 0, leaseId: undefined }) + + return Promise.resolve() + }, + pull, + settle, + size: () => entries.length, + deadLetters: () => [...dead], + batch: (size = 100) => + pull(size, 30_000).pipe( + Effect.map(({ messages }) => ({ + messages: messages.map((message) => ({ + body: message.body, + attempts: message.attempts, + ack: () => Effect.runSync(settle([message.leaseId], [])), + retry: (retry) => + Effect.runSync( + settle([], [{ leaseId: message.leaseId, delaySeconds: retry?.delaySeconds ?? 0 }]), + ), + })), + })), + ), + } +} diff --git a/packages/cloudflare/src/queue-consumer.ts b/packages/cloudflare/src/queue-consumer.ts new file mode 100644 index 0000000..74f62ae --- /dev/null +++ b/packages/cloudflare/src/queue-consumer.ts @@ -0,0 +1,240 @@ +import { Data, Effect } from 'effect' + +import { + resolveJson, + type BlobStore, + type EventInput, + type JsonValue, + type SecretContext, + type WorkerCallback, + type WorkerCallbackInput, + type WorkerCallbackResult, + type WorkerSecretsResult, +} from '@looms/core' + +import { + readQueueEffectMessage, + type QueueEffectMessage, + type QueueMessageLike, +} from './queue-message' + +/** The worker half of the Looms HTTP API. `createLoomsClient({ baseUrl, workerCallbackToken })` is one. */ +export interface WorkerCallbackClient { + workerCallback(runId: string, input: WorkerCallbackInput): Promise + workerSecrets(runId: string, attempt: WorkerCallback): Promise +} + +/** The attempt a handler runs, with its input read back from the blob store when it was offloaded. */ +export interface QueueTask extends Omit { + readonly input: JsonValue +} + +export interface QueueTaskContext { + /** Aborts once Looms asks the attempt to stop or no longer counts it as running. */ + readonly signal: AbortSignal + /** Report progress. A later attempt receives the last `details` as `task.details`. */ + heartbeat(details?: JsonValue): Promise + /** The run's secret context. Rejects once the attempt is no longer running. */ + secrets(): Promise +} + +export type QueueTaskHandler = ( + task: QueueTask, + ctx: QueueTaskContext, +) => Promise> | ReadonlyArray + +/** Throw from a handler to control how Looms treats the failed attempt. */ +export class QueueTaskError extends Data.TaggedError('QueueTaskError')<{ + readonly message: string + /** Fail the effect even when the retry policy has attempts left. */ + readonly nonRetryable?: boolean + /** Wait at least this long before the next attempt. */ + readonly retryAfterMs?: number +}> {} + +export interface QueueConsumerOptions { + readonly handlers: Readonly> + readonly callbacks: WorkerCallbackClient + /** Reads inputs the producer offloaded. Use the store the producer writes to. */ + readonly blobs?: BlobStore +} + +export interface QueueBatchLike { + readonly messages: ReadonlyArray +} + +class CallbackError extends Data.TaggedError('CallbackError')<{ readonly message: string }> {} + +/** + * Run one delivery. The message is acked as soon as Looms accepts `started`, and Looms owns every + * retry after that through heartbeat and start-to-close timeouts. A duplicate or stale delivery is + * acked without running. A message that cannot be started yet is retried by the queue, and one that + * never starts ends in the dead-letter queue, where the schedule-to-start timeout recovers it. + */ +export function processQueueMessage( + message: QueueMessageLike, + options: QueueConsumerOptions, +): Effect.Effect { + const task = readQueueEffectMessage(message) + const handler = task && options.handlers[task.type] + + const redeliver = Effect.sync(() => + message.retry({ delaySeconds: redeliveryDelay(message.attempts) }), + ) + + if (!task || !handler) { + return redeliver + } + + const attempt: WorkerCallback = { effectId: task.effectId, attempt: task.attempt } + + return callback(() => + options.callbacks.workerCallback(task.runId, { ...attempt, kind: 'started' }), + ).pipe( + Effect.matchEffect({ + onFailure: () => redeliver, + onSuccess: (started) => + Effect.sync(() => message.ack()).pipe( + Effect.andThen( + started.status === 'accepted' + ? runAttempt(task, attempt, handler, started.cancelRequested, options) + : Effect.void, + ), + ), + }), + ) +} + +/** A push consumer: call it from the Worker's `queue(batch)` handler. */ +export function queueConsumer( + options: QueueConsumerOptions, +): (batch: QueueBatchLike) => Promise { + return (batch) => + Effect.runPromise( + Effect.forEach(batch.messages, (message) => processQueueMessage(message, options), { + concurrency: 'unbounded', + discard: true, + }), + ) +} + +function runAttempt( + task: QueueEffectMessage, + attempt: WorkerCallback, + handler: QueueTaskHandler, + cancelRequested: boolean, + options: QueueConsumerOptions, +): Effect.Effect { + const controller = new AbortController() + + if (cancelRequested) { + controller.abort() + } + + const report = (input: WorkerCallbackInput) => + callback(() => options.callbacks.workerCallback(task.runId, input)).pipe(Effect.ignore) + + const ctx: QueueTaskContext = { + signal: controller.signal, + heartbeat: (details) => + Effect.runPromise( + callback(() => + options.callbacks.workerCallback( + task.runId, + details === undefined + ? { ...attempt, kind: 'heartbeat' } + : { ...attempt, kind: 'heartbeat', details }, + ), + ).pipe( + Effect.map((result) => { + if (result.status !== 'accepted' || result.cancelRequested) { + controller.abort() + } + }), + ), + ), + secrets: () => + Effect.runPromise( + callback(() => options.callbacks.workerSecrets(task.runId, attempt)).pipe( + Effect.flatMap((result) => { + if (result.status === 'accepted') { + return Effect.succeed(result.secrets) + } + + controller.abort() + + return Effect.fail(new CallbackError({ message: `the attempt is ${result.status}` })) + }), + ), + ), + } + + return taskInput(task, options.blobs).pipe( + Effect.flatMap((input) => + Effect.tryPromise({ + try: () => Promise.resolve(handler({ ...taskFields(task), input }, ctx)), + catch: (cause) => + cause instanceof QueueTaskError + ? cause + : new QueueTaskError({ + message: cause instanceof Error ? cause.message : String(cause), + }), + }), + ), + Effect.matchEffect({ + onSuccess: (events) => report({ ...attempt, kind: 'complete', events }), + onFailure: (error) => + report( + controller.signal.aborted + ? { ...attempt, kind: 'cancelled' } + : { + ...attempt, + kind: 'fail', + error: error.message, + nonRetryable: error.nonRetryable, + retryAfterMs: error.retryAfterMs, + }, + ), + }), + ) +} + +function taskFields(task: QueueEffectMessage): Omit { + const { v: _v, input: _input, inputRef: _inputRef, ...fields } = task + + return fields +} + +function taskInput( + task: QueueEffectMessage, + blobs: BlobStore | undefined, +): Effect.Effect { + if (!task.inputRef) { + return Effect.succeed(task.input ?? null) + } + + if (!blobs) { + return Effect.fail( + new QueueTaskError({ message: 'the input was offloaded and the consumer has no blob store' }), + ) + } + + return resolveJson(task.inputRef, blobs).pipe( + Effect.mapError( + (error) => new QueueTaskError({ message: `could not read the input: ${error.message}` }), + ), + ) +} + +function callback(call: () => Promise): Effect.Effect { + return Effect.tryPromise({ + try: call, + catch: (cause) => + new CallbackError({ message: cause instanceof Error ? cause.message : String(cause) }), + }) +} + +/** Seconds before the queue redelivers a message that could not start: 1, 2, 4, ... up to 60. */ +function redeliveryDelay(attempts: number): number { + return Math.min(60, 2 ** Math.max(0, attempts - 1)) +} diff --git a/packages/cloudflare/src/queue-message.ts b/packages/cloudflare/src/queue-message.ts new file mode 100644 index 0000000..7199b38 --- /dev/null +++ b/packages/cloudflare/src/queue-message.ts @@ -0,0 +1,74 @@ +import { Option, Schema } from 'effect' + +import { + isBlobRef, + TraceContextSchema, + type BlobRef, + type JsonValue, + type TraceContext, +} from '@looms/core' +import type { EffectWorkerDeadlines } from '@looms/runtime' + +const QueueEffectMessageSchema = Schema.Struct({ + v: Schema.Literal(1), + runId: Schema.String, + effectId: Schema.String, + attempt: Schema.Finite, + type: Schema.String, + input: Schema.optional(Schema.Json), + inputRef: Schema.optional(Schema.Json), + details: Schema.optional(Schema.Json), + trace: Schema.optional(TraceContextSchema), + deadlines: Schema.optional( + Schema.Struct({ + scheduleToStartAt: Schema.optional(Schema.Finite), + startToCloseTimeoutMs: Schema.optional(Schema.Finite), + heartbeatTimeoutMs: Schema.optional(Schema.Finite), + }), + ), +}) + +const decodeMessage = Schema.decodeUnknownOption(QueueEffectMessageSchema) + +/** + * One effect attempt on a Cloudflare Queue. It never carries the run's secrets; the consumer reads + * them through the worker callback API. Exactly one of `input` and `inputRef` is set. + */ +export interface QueueEffectMessage { + readonly v: 1 + readonly runId: string + readonly effectId: string + readonly attempt: number + readonly type: string + readonly input?: JsonValue + /** Set instead of `input` when the input would push the message over the queue's size limit. */ + readonly inputRef?: BlobRef + /** Progress from the previous attempt's last heartbeat. */ + readonly details?: JsonValue + readonly trace?: TraceContext + readonly deadlines?: EffectWorkerDeadlines +} + +/** A delivered message, shaped like a Cloudflare `Message`. */ +export interface QueueMessageLike { + readonly body: unknown + readonly attempts: number + ack(): void + retry(options?: { readonly delaySeconds?: number }): void +} + +/** `undefined` unless the body is a version 1 message with exactly one of `input` and `inputRef`. */ +export function readQueueEffectMessage(message: QueueMessageLike): QueueEffectMessage | undefined { + return Option.match(decodeMessage(message.body), { + onNone: () => undefined, + onSome: ({ inputRef, ...decoded }) => { + if (inputRef === undefined) { + return decoded.input === undefined ? undefined : decoded + } + + return decoded.input === undefined && isBlobRef(inputRef) + ? { ...decoded, inputRef } + : undefined + }, + }) +} diff --git a/packages/cloudflare/src/queue-pull.ts b/packages/cloudflare/src/queue-pull.ts new file mode 100644 index 0000000..b523bc9 --- /dev/null +++ b/packages/cloudflare/src/queue-pull.ts @@ -0,0 +1,281 @@ +import { + Data, + Deferred, + Effect, + Fiber, + FiberSet, + Option, + Predicate, + Schema, + Semaphore, +} from 'effect' + +import { stringifyJson, type JsonValue } from '@looms/core' + +import { processQueueMessage, type QueueConsumerOptions } from './queue-consumer' +import type { QueueMessageLike } from './queue-message' + +export interface PulledQueueMessage { + readonly leaseId: string + readonly body: unknown + /** Deliveries so far, including this one. */ + readonly attempts: number +} + +export interface QueuePullResult { + readonly messages: ReadonlyArray + /** Messages waiting in the queue, when the source reports it. */ + readonly backlog?: number +} + +export interface QueueLeaseRetry { + readonly leaseId: string + readonly delaySeconds: number +} + +export class QueuePullError extends Data.TaggedError('QueuePullError')<{ + readonly message: string + readonly status?: number +}> {} + +/** Where a pull consumer leases messages from. `cloudflareQueuePullSource` reads a Cloudflare Queue. */ +export interface QueuePullSource { + readonly pull: ( + batchSize: number, + visibilityTimeoutMs: number, + ) => Effect.Effect + readonly settle: ( + acks: ReadonlyArray, + retries: ReadonlyArray, + ) => Effect.Effect +} + +export interface QueuePullConsumerOptions extends QueueConsumerOptions { + readonly source: QueuePullSource + /** Attempts run at once. Default 8. */ + readonly concurrency?: number + /** How long a leased message stays hidden. Acks land long before; default 30 s. */ + readonly visibilityTimeoutMs?: number + /** The longest wait between pulls that come back empty or fail. Default 5 s. */ + readonly maxIdleMs?: number + /** How often buffered acks and retries are sent. Default 250 ms. */ + readonly ackIntervalMs?: number + /** A pull or ack request failed. The consumer backs off and keeps going. */ + readonly onError?: (error: QueuePullError) => void +} + +export interface QueuePullConsumer { + /** Messages waiting in the queue at the last pull, for autoscaling. */ + backlog(): number | undefined + inFlight(): number + /** Stop pulling, let running attempts finish, and send the last acks. Call it on SIGTERM. */ + stop(): Promise +} + +const MAX_PULL_BATCH = 100 +const MIN_IDLE_MS = 50 + +/** Lease messages from `source` and run them with the same handlers as {@link queueConsumer}. */ +export function createQueuePullConsumer(options: QueuePullConsumerOptions): QueuePullConsumer { + const concurrency = Math.max(1, options.concurrency ?? 8) + const visibilityTimeoutMs = options.visibilityTimeoutMs ?? 30_000 + const maxIdleMs = options.maxIdleMs ?? 5_000 + const slots = Semaphore.makeUnsafe(concurrency) + const stopping = Deferred.makeUnsafe() + let backlog: number | undefined + let inFlight = 0 + let acks: string[] = [] + let retries: QueueLeaseRetry[] = [] + + const reportError = (error: QueuePullError) => Effect.sync(() => options.onError?.(error)) + + const flush = Effect.suspend(() => { + if (acks.length === 0 && retries.length === 0) { + return Effect.void + } + + const settled = options.source.settle(acks, retries) + + acks = [] + retries = [] + + return settled.pipe(Effect.catchTag('QueuePullError', reportError)) + }) + + const lease = (message: PulledQueueMessage): QueueMessageLike => ({ + body: message.body, + attempts: message.attempts, + ack: () => { + acks.push(message.leaseId) + }, + retry: (retry) => { + retries.push({ leaseId: message.leaseId, delaySeconds: retry?.delaySeconds ?? 0 }) + }, + }) + + const untilStopped = (effect: Effect.Effect) => + Effect.raceFirst(effect, Deferred.await(stopping)) + + const run = Effect.gen(function* () { + const running = yield* FiberSet.make() + + const flusher = yield* Effect.forkScoped( + Effect.forever(Effect.sleep(options.ackIntervalMs ?? 250).pipe(Effect.andThen(flush))), + ) + + let idleMs = 0 + + while (!(yield* Deferred.isDone(stopping))) { + yield* untilStopped(Semaphore.withPermits(slots, 1)(Effect.void)) + + if (yield* Deferred.isDone(stopping)) { + break + } + + const pulled = yield* options.source + .pull(Math.min(concurrency - inFlight, MAX_PULL_BATCH), visibilityTimeoutMs) + .pipe( + Effect.catchTag('QueuePullError', (error) => + reportError(error).pipe(Effect.as({ messages: [] })), + ), + ) + + backlog = pulled.backlog ?? backlog + + for (const message of pulled.messages) { + inFlight += 1 + + yield* FiberSet.run( + running, + Semaphore.withPermits( + slots, + 1, + )(processQueueMessage(lease(message), options)).pipe( + Effect.ensuring( + Effect.sync(() => { + inFlight -= 1 + }), + ), + ), + ) + } + + idleMs = + pulled.messages.length > 0 ? 0 : Math.min(maxIdleMs, Math.max(MIN_IDLE_MS, idleMs * 2)) + + if (idleMs > 0) { + yield* untilStopped(Effect.sleep(idleMs)) + } + } + + yield* FiberSet.awaitEmpty(running) + yield* Fiber.interrupt(flusher) + yield* flush + }) + + const fiber = Effect.runFork(Effect.scoped(run)) + + return { + backlog: () => backlog, + inFlight: () => inFlight, + stop: () => + Effect.runPromise( + Deferred.succeed(stopping, undefined).pipe(Effect.andThen(Fiber.join(fiber))), + ), + } +} + +export interface CloudflareQueuePullSourceOptions { + readonly accountId: string + readonly queueId: string + /** An API token with Queues edit permission. */ + readonly token: string + readonly fetch?: typeof fetch +} + +const PullResponseSchema = Schema.Struct({ + result: Schema.Struct({ + message_backlog_count: Schema.optional(Schema.Finite), + messages: Schema.Array( + Schema.Struct({ + lease_id: Schema.String, + attempts: Schema.Finite, + body: Schema.Json, + }), + ), + }), +}) + +const decodePullResponse = Schema.decodeUnknownEffect(PullResponseSchema) +const decodeJsonText = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Json)) + +/** The Cloudflare Queues HTTP pull API. The producer must send `contentType: 'json'`. */ +export function cloudflareQueuePullSource( + options: CloudflareQueuePullSourceOptions, +): QueuePullSource { + const fetchImpl = options.fetch ?? fetch + + const endpoint = `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(options.accountId)}/queues/${encodeURIComponent(options.queueId)}/messages` + + const post = (path: string, body: JsonValue) => + Effect.tryPromise({ + try: () => + fetchImpl(`${endpoint}/${path}`, { + method: 'POST', + headers: { + authorization: `Bearer ${options.token}`, + 'content-type': 'application/json', + }, + body: stringifyJson(body), + }), + catch: (cause) => + new QueuePullError({ message: cause instanceof Error ? cause.message : String(cause) }), + }).pipe( + Effect.filterOrFail( + (response) => response.ok, + (response) => + new QueuePullError({ + message: `Cloudflare Queues ${path} returned ${response.status}`, + status: response.status, + }), + ), + ) + + return { + pull: (batchSize, visibilityTimeoutMs) => + post('pull', { batch_size: batchSize, visibility_timeout_ms: visibilityTimeoutMs }).pipe( + Effect.flatMap((response) => + Effect.tryPromise({ + try: () => response.json(), + catch: () => new QueuePullError({ message: 'the pull response is not JSON' }), + }), + ), + Effect.flatMap((json) => + decodePullResponse(json).pipe( + Effect.mapError(() => new QueuePullError({ message: 'unexpected pull response' })), + ), + ), + Effect.map(({ result }) => ({ + backlog: result.message_backlog_count, + messages: result.messages.map((message) => ({ + leaseId: message.lease_id, + attempts: message.attempts, + body: jsonBody(message.body), + })), + })), + ), + settle: (acks, retries) => + post('ack', { + acks: acks.map((leaseId) => ({ lease_id: leaseId })), + retries: retries.map((retry) => ({ + lease_id: retry.leaseId, + delay_seconds: retry.delaySeconds, + })), + }).pipe(Effect.asVoid), + } +} + +/** The pull API returns a JSON message's body as its JSON text. */ +function jsonBody(body: JsonValue): JsonValue { + return Predicate.isString(body) ? Option.getOrElse(decodeJsonText(body), () => body) : body +} diff --git a/packages/cloudflare/src/queue-worker.test.ts b/packages/cloudflare/src/queue-worker.test.ts index b75e4b8..ee6e938 100644 --- a/packages/cloudflare/src/queue-worker.test.ts +++ b/packages/cloudflare/src/queue-worker.test.ts @@ -1,250 +1,489 @@ import { describe, expect, test } from 'bun:test' -import { Predicate } from 'effect' +import { Effect } from 'effect' +import { createLoomsClient } from '@looms/client' import { - cloudflareQueueWorker, - createMemoryQueue, - processQueueMessage, - queuePushConsumer, - type QueueTaskHandler, -} from './queue-worker' - -function urlOf(input: RequestInfo | URL): string { - if (Predicate.isString(input)) { - return input + complete, + defineEffect, + defineRuntimeModule, + defineThread, + invoke, + memoryBlobStore, + utf8JsonBytes, + type BlobStore, + type JsonValue, + type RetryPolicy, +} from '@looms/core' +import { bearerAuth, createLooms, type Looms } from '@looms/runtime' + +import { createMemoryQueue, type MemoryQueue } from './memory-queue' +import { queueConsumer, QueueTaskError, type QueueTaskHandler } from './queue-consumer' +import type { QueueEffectMessage } from './queue-message' +import { cloudflareQueuePullSource, createQueuePullConsumer } from './queue-pull' +import { cloudflareQueueWorker, type QueueProducer } from './queue-worker' + +const WORKER_TOKEN = 'worker-token' +const SECRET = 'sk_live_never_logged' +const definition = { kind: 'sandbox', name: 'sandbox', version: 'v1' } + +const FAST: RetryPolicy = { + maxAttempts: 3, + backoffMs: 5, + scheduleToStartTimeoutMs: 60_000, +} + +interface SandboxState { + readonly input: JsonValue + readonly output: JsonValue | null +} + +function sandboxModule(retry: RetryPolicy) { + const exec = defineEffect({ type: 'sandbox.exec', retry }) + + return defineRuntimeModule({ + namespace: 'sandbox', + protocolVersion: '1.0.0', + effects: { exec }, + threads: { + sandbox: defineThread({ + kind: 'sandbox', + initialState: (ctx): SandboxState => ({ input: ctx.input ?? null, output: null }), + step: (state, event) => + event.type === 'sandbox.done' ? { ...state, output: event.payload ?? true } : state, + effects: (state) => + state.output === null ? [invoke(exec, state.input, 'exec')] : [complete(state.output)], + }), + }, + definitions: [definition], + }) +} + +interface Harness { + readonly looms: Looms + readonly queue: MemoryQueue + readonly sent: QueueEffectMessage[] + readonly callbacks: ReturnType +} + +function harness( + options: { + retry?: RetryPolicy + producer?: (queue: MemoryQueue, sent: QueueEffectMessage[]) => QueueProducer + blobs?: BlobStore + maxRetries?: number + } = {}, +): Harness { + const queue = createMemoryQueue({ maxRetries: options.maxRetries }) + const sent: QueueEffectMessage[] = [] + + const producer: QueueProducer = options.producer?.(queue, sent) ?? { + send: (message, sendOptions) => { + sent.push(message) + + return queue.send(message, sendOptions) + }, } - if (input instanceof URL) { - return input.href + const looms = createLooms({ + modules: [sandboxModule(options.retry ?? FAST)], + worker: cloudflareQueueWorker({ + routes: [{ handles: ['sandbox.exec'], queue: producer }], + blobs: options.blobs, + }), + authorize: bearerAuth({ worker: WORKER_TOKEN }), + }) + + const callbacks = createLoomsClient({ + baseUrl: 'http://looms.test', + workerCallbackToken: WORKER_TOKEN, + fetch: async (input, init) => + (await looms.fetch(new Request(input, init))) ?? new Response(null, { status: 404 }), + }) + + return { looms, queue, sent, callbacks } +} + +const done: QueueTaskHandler = (task) => [{ type: 'sandbox.done', payload: { ran: task.attempt } }] + +async function waitFor(condition: () => boolean | Promise, timeoutMs = 2_000) { + const deadline = Date.now() + timeoutMs + + while (!(await condition())) { + if (Date.now() > deadline) { + throw new Error('condition not met in time') + } + + await Bun.sleep(5) } +} - return input.url +async function status(looms: Looms, runId: string) { + return (await looms.getRun(runId)).status } -describe('cloudflare queues effect worker', () => { - test('producer enqueues formatted QueueEffectMessage', async () => { - const queue = createMemoryQueue() +async function eventTypes(looms: Looms, runId: string) { + return (await looms.getEvents(runId)).map((event) => event.type) +} - const worker = cloudflareQueueWorker({ - routes: [{ handles: ['sandbox.run', 'payment.charge'], queue }], - }) +describe('Cloudflare Queues effect transport', () => { + test('runs an attempt over the queue and reads secrets only through the callback', async () => { + const { looms, queue, sent, callbacks } = harness() + let secret: string | undefined + + const started = await looms.start( + definition, + { cmd: 'ls' }, + { + secrets: { token: SECRET }, + trace: { traceparent: '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01' }, + }, + ) - expect(worker.dispatchSemantics).toBe('acknowledged') - expect(Predicate.isFunction(worker.handles) && worker.handles('sandbox.run')).toBe(true) - expect(Predicate.isFunction(worker.handles) && worker.handles('unknown.task')).toBe(false) + const consumer = createQueuePullConsumer({ + source: queue, + callbacks, + handlers: { + 'sandbox.exec': async (task, ctx) => { + secret = (await ctx.secrets()).token - await worker.dispatch({ - runId: 'run_1', - effectId: 'eff_1', - attempt: 1, - type: 'sandbox.run', - input: { command: 'echo hello' }, + return done(task, ctx) + }, + }, }) - expect(queue.enqueued).toHaveLength(1) - const enqueued = queue.enqueued[0] + await waitFor(async () => (await status(looms, started.runId)) === 'completed') + await consumer.stop() - expect(enqueued).toMatchObject({ - v: 1, - runId: 'run_1', - effectId: 'eff_1', - attempt: 1, - type: 'sandbox.run', - input: { command: 'echo hello' }, - }) + expect(secret).toBe(SECRET) + expect(JSON.stringify(sent)).not.toContain(SECRET) + expect(sent[0]?.input).toEqual({ cmd: 'ls' }) + expect(sent[0]?.trace?.traceparent).toContain('4bf92f3577b34da6a3ce929d0e0e4736') + expect(sent[0]?.deadlines?.scheduleToStartAt).toBeGreaterThan(0) + expect(queue.size()).toBe(0) + await looms.stop() }) - test('producer throws on 429 for acknowledged retry', async () => { - const queue = createMemoryQueue() - queue.setSimulation({ rateLimit429: true }) + test('a duplicate delivery runs the handler once', async () => { + let runs = 0 - const worker = cloudflareQueueWorker({ - routes: [{ handles: ['sandbox.run'], queue }], + const { looms, queue, callbacks } = harness({ + producer: (memory) => ({ + send: async (message, sendOptions) => { + await memory.send(message, sendOptions) + await memory.send(message, sendOptions) + }, + }), }) - expect( - worker.dispatch({ - runId: 'run_1', - effectId: 'eff_1', - attempt: 1, - type: 'sandbox.run', - input: {}, - }), - ).rejects.toThrow('429 Too Many Requests') - }) + const started = await looms.start(definition, null) - test('producer offloads inputs exceeding threshold', async () => { - const queue = createMemoryQueue() - let offloaded = false + await queueConsumer({ + callbacks, + handlers: { + 'sandbox.exec': (task, ctx) => { + runs += 1 - const worker = cloudflareQueueWorker({ - routes: [{ handles: ['sandbox.run'], queue }], - offloadInput: async (input) => { - offloaded = true - return { $ref: 'blob_123', size: JSON.stringify(input).length } + return done(task, ctx) + }, }, - }) + })(await Effect.runPromise(queue.batch())) + + expect(runs).toBe(1) + expect(queue.size()).toBe(0) + expect(await status(looms, started.runId)).toBe('completed') + await looms.stop() + }) - const largeInput = { payload: 'x'.repeat(120_000) } + test('a consumer that dies before started leaves the message for redelivery', async () => { + const { looms, queue, callbacks } = harness() + const started = await looms.start(definition, null) - await worker.dispatch({ - runId: 'run_1', - effectId: 'eff_1', - attempt: 1, - type: 'sandbox.run', - input: largeInput, + await Effect.runPromise(queue.pull(10, 20)) + + const consumer = createQueuePullConsumer({ + source: queue, + callbacks, + handlers: { 'sandbox.exec': done }, + visibilityTimeoutMs: 20, }) - expect(offloaded).toBe(true) - expect(queue.enqueued[0]?.input).toEqual({ $ref: 'blob_123', size: 120_014 }) + await waitFor(async () => (await status(looms, started.runId)) === 'completed') + await consumer.stop() + + expect(queue.size()).toBe(0) + await looms.stop() }) - test('consumer acknowledges queue message immediately once started is accepted', async () => { - const queue = createMemoryQueue() - - await queue.send({ - v: 1, - runId: 'run_1', - effectId: 'eff_1', - attempt: 1, - type: 'sandbox.run', - input: { cmd: 'test' }, - deadlines: {}, + test('an attempt that dies after started is retried by Looms, and its stale message is dropped', async () => { + const { looms, queue, sent, callbacks } = harness({ + retry: { ...FAST, heartbeatTimeoutMs: 40 }, }) - const calls: string[] = [] + const ran: number[] = [] - const mockFetch: typeof fetch = async (input) => { - const url = urlOf(input) - calls.push(url) + const consume = queueConsumer({ + callbacks, + handlers: { + 'sandbox.exec': (task, ctx) => { + ran.push(task.attempt) - if (url.includes('/started')) { - return new Response(JSON.stringify({ status: 'accepted' }), { status: 200 }) - } + return task.attempt === 1 ? new Promise(() => undefined) : done(task, ctx) + }, + }, + }) - if (url.includes('/complete')) { - return new Response(JSON.stringify({ status: 'accepted' }), { status: 200 }) - } + const started = await looms.start(definition, null) - return new Response(null, { status: 200 }) - } + void consume(await Effect.runPromise(queue.batch())) + + await waitFor(() => sent.length === 2) + await consume(await Effect.runPromise(queue.batch())) - const handler: QueueTaskHandler = async () => { - // At this point in handler execution, the queue message must already be acknowledged - expect(queue.acknowledged).toHaveLength(1) - return [{ type: 'sandbox.finished', payload: { ok: true } }] + expect(await status(looms, started.runId)).toBe('completed') + + const first = sent[0] + + if (!first) { + throw new Error('missing first attempt') } - const consumer = queuePushConsumer({ - handlers: { 'sandbox.run': handler }, - callbackUrl: 'http://looms.internal', - fetch: mockFetch, + await queue.send(first) + await consume(await Effect.runPromise(queue.batch())) + + expect(ran).toEqual([1, 2]) + expect(queue.size()).toBe(0) + await looms.stop() + }) + + test('cancelling the run aborts the running attempt through its heartbeat', async () => { + const { looms, queue, callbacks } = harness() + let aborted = false + + const started = await looms.start(definition, null) + + const consumer = createQueuePullConsumer({ + source: queue, + callbacks, + handlers: { + 'sandbox.exec': async (_task, ctx) => { + while (!ctx.signal.aborted) { + await ctx.heartbeat({ at: Date.now() }) + await Bun.sleep(5) + } + + aborted = true + + throw new Error('stopped') + }, + }, }) - const batch = queue.receiveBatch() - await consumer.consumeBatch(batch) + await waitFor(async () => + (await eventTypes(looms, started.runId)).includes('runtime.effect.heartbeat'), + ) - expect(calls.some((c) => c.includes('/started'))).toBe(true) - expect(calls.some((c) => c.includes('/complete'))).toBe(true) - expect(queue.acknowledged).toHaveLength(1) + await looms.cancel(started.runId) + await waitFor(() => aborted) + await consumer.stop() + + expect(await eventTypes(looms, started.runId)).toContain('runtime.effect.cancelled') + expect(await status(looms, started.runId)).toBe('cancelled') + await looms.stop() }) - test('stale attempt is acknowledged on queue without running handler', async () => { - const queue = createMemoryQueue() - - await queue.send({ - v: 1, - runId: 'run_1', - effectId: 'eff_1', - attempt: 1, - type: 'sandbox.run', - input: {}, - deadlines: {}, + test('a send the queue rejects is retried with backoff', async () => { + let rejected = 0 + + const { looms, queue, sent } = harness({ + producer: (memory, attempts) => ({ + send: (message, sendOptions) => { + attempts.push(message) + + if (rejected === 0) { + rejected += 1 + + return Promise.reject(new Error('429 Too Many Requests')) + } + + return memory.send(message, sendOptions) + }, + }), }) - let handlerRan = false + const started = await looms.start(definition, null) - const mockFetch: typeof fetch = async (input) => { - const url = urlOf(input) + await waitFor(() => queue.size() === 1) - if (url.includes('/started')) { - return new Response(JSON.stringify({ status: 'stale' }), { status: 200 }) - } + const types = await eventTypes(looms, started.runId) - return new Response(null, { status: 200 }) - } + expect(types).toContain('runtime.effect.retry.scheduled') + expect(types).not.toContain('runtime.effect.ambiguous') + expect(sent.map((message) => message.attempt)).toEqual([1, 2]) + await looms.stop() + }) - const handler: QueueTaskHandler = async () => { - handlerRan = true - return [] - } + test('an input over the message limit travels as a blob reference', async () => { + const blobs = memoryBlobStore() + const { looms, queue, sent, callbacks } = harness({ blobs }) + const input = { file: 'x'.repeat(150_000) } + let length = 0 - const consumer = queuePushConsumer({ - handlers: { 'sandbox.run': handler }, - callbackUrl: 'http://looms.internal', - fetch: mockFetch, + const started = await looms.start(definition, input) + + await queueConsumer({ + callbacks, + blobs, + handlers: { + 'sandbox.exec': (task, ctx) => { + length = JSON.stringify(task.input).length + + return done(task, ctx) + }, + }, + })(await Effect.runPromise(queue.batch())) + + expect(sent[0]?.input).toBeUndefined() + expect(sent[0]?.inputRef?.bytes).toBe(utf8JsonBytes(input)) + expect(utf8JsonBytes(sent[0] ?? null)).toBeLessThan(128 * 1024) + expect(length).toBe(JSON.stringify(input).length) + expect(await status(looms, started.runId)).toBe('completed') + await looms.stop() + }) + + test('a dead-lettered message is recovered by the schedule-to-start timeout', async () => { + const { looms, queue, sent, callbacks } = harness({ + retry: { ...FAST, scheduleToStartTimeoutMs: 60 }, + maxRetries: 0, }) - const batch = queue.receiveBatch() - await consumer.consumeBatch(batch) + const started = await looms.start(definition, null) + + await Effect.runPromise(queue.pull(10, 1)) + await Bun.sleep(5) + await Effect.runPromise(queue.pull(10, 1)) + + expect(queue.deadLetters()).toHaveLength(1) + + await waitFor(() => sent.length === 2) + + await queueConsumer({ callbacks, handlers: { 'sandbox.exec': done } })( + await Effect.runPromise(queue.batch()), + ) + + expect(await status(looms, started.runId)).toBe('completed') + await looms.stop() + }) + + test('a non-retryable handler error fails the effect without another attempt', async () => { + const { looms, queue, sent, callbacks } = harness() + const started = await looms.start(definition, null) + + await queueConsumer({ + callbacks, + handlers: { + 'sandbox.exec': () => { + throw new QueueTaskError({ message: 'bad input', nonRetryable: true }) + }, + }, + })(await Effect.runPromise(queue.batch())) - expect(handlerRan).toBe(false) - expect(queue.acknowledged).toHaveLength(1) + expect(await status(looms, started.runId)).toBe('failed') + expect(sent).toHaveLength(1) + await looms.stop() }) - test('cancellation mid-flight signals handler abort', async () => { - const queue = createMemoryQueue() - - await queue.send({ - v: 1, - runId: 'run_1', - effectId: 'eff_1', - attempt: 1, - type: 'sandbox.run', - input: {}, - deadlines: {}, + test('stop lets running attempts finish, sends their acks, and pulls nothing more', async () => { + const { looms, queue, callbacks } = harness() + const gate = Promise.withResolvers() + + const first = await looms.start(definition, null) + + const consumer = createQueuePullConsumer({ + source: queue, + callbacks, + handlers: { 'sandbox.exec': async (task, ctx) => gate.promise.then(() => done(task, ctx)) }, }) - const mockFetch: typeof fetch = async (input) => { - const url = urlOf(input) + await waitFor(() => consumer.inFlight() === 1) + + const stopped = consumer.stop() - if (url.includes('/started')) { - return new Response(JSON.stringify({ status: 'accepted' }), { status: 200 }) - } + gate.resolve() + await stopped - if (url.includes('/heartbeat')) { - return new Response(JSON.stringify({ status: 'accepted', cancelRequested: true }), { - status: 200, + expect(await status(looms, first.runId)).toBe('completed') + expect(queue.size()).toBe(0) + + await looms.start(definition, null) + await Bun.sleep(20) + + expect(queue.size()).toBe(1) + await looms.stop() + }) +}) + +describe('cloudflareQueuePullSource', () => { + test('pulls and acks through the Cloudflare Queues HTTP API', async () => { + const requests: Array<{ url: string; auth: string | null; body: string }> = [] + + const source = cloudflareQueuePullSource({ + accountId: 'acct', + queueId: 'q1', + token: 'cf-token', + fetch: async (input, init) => { + const request = new Request(input, init) + + requests.push({ + url: request.url, + auth: request.headers.get('authorization'), + body: await request.text(), }) - } - return new Response(null, { status: 200 }) - } + return request.url.endsWith('/pull') + ? Response.json({ + result: { + message_backlog_count: 7, + messages: [{ lease_id: 'l1', attempts: 2, body: '{"v":1}' }], + }, + }) + : Response.json({ success: true }) + }, + }) - let aborted = false + const pulled = await Effect.runPromise(source.pull(10, 30_000)) - const handler: QueueTaskHandler = async (_task, ctx) => { - ctx.signal.addEventListener('abort', () => { - aborted = true - }) + await Effect.runPromise(source.settle(['l1'], [{ leaseId: 'l2', delaySeconds: 4 }])) - // Send a heartbeat which returns cancelRequested - await ctx.heartbeat({ progress: 50 }) - return [] - } + expect(pulled).toEqual({ + backlog: 7, + messages: [{ leaseId: 'l1', attempts: 2, body: { v: 1 } }], + }) - const batch = queue.receiveBatch() - const message = batch.messages[0] + expect(requests).toEqual([ + { + url: 'https://api.cloudflare.com/client/v4/accounts/acct/queues/q1/messages/pull', + auth: 'Bearer cf-token', + body: '{"batch_size":10,"visibility_timeout_ms":30000}', + }, + { + url: 'https://api.cloudflare.com/client/v4/accounts/acct/queues/q1/messages/ack', + auth: 'Bearer cf-token', + body: '{"acks":[{"lease_id":"l1"}],"retries":[{"lease_id":"l2","delay_seconds":4}]}', + }, + ]) + }) - if (message) { - await processQueueMessage(message, { - handlers: { 'sandbox.run': handler }, - callbackUrl: 'http://looms.internal', - fetch: mockFetch, - }) - } + test('a rate-limited pull fails with its status', async () => { + const source = cloudflareQueuePullSource({ + accountId: 'acct', + queueId: 'q1', + token: 'cf-token', + fetch: async () => new Response(null, { status: 429 }), + }) + + const error = await Effect.runPromise(Effect.flip(source.pull(10, 30_000))) - expect(aborted).toBe(true) + expect(error.status).toBe(429) }) }) diff --git a/packages/cloudflare/src/queue-worker.ts b/packages/cloudflare/src/queue-worker.ts index bb049ab..86baf2e 100644 --- a/packages/cloudflare/src/queue-worker.ts +++ b/packages/cloudflare/src/queue-worker.ts @@ -1,363 +1,123 @@ -import { DateTime, Predicate, Schema } from 'effect' +import { Data, Effect, Predicate, type Types } from 'effect' -import { stringifyJson, type EventInput, type JsonValue } from '@looms/core' +import { claimJson, utf8JsonBytes, type BlobStore } from '@looms/core' import type { EffectWorker, EffectWorkerTask } from '@looms/runtime' -const CallbackResponseSchema = Schema.Struct({ - status: Schema.Union([ - Schema.Literal('accepted'), - Schema.Literal('stale'), - Schema.Literal('cancelled'), - ]), - cancelRequested: Schema.optional(Schema.Boolean), -}) +import type { QueueEffectMessage } from './queue-message' -const decodeCallbackResponse = Schema.decodeUnknownOption(CallbackResponseSchema) +/** Leaves room under Cloudflare's 128 KB message limit. */ +export const DEFAULT_MAX_QUEUE_MESSAGE_BYTES = 100_000 -function utcNow(): Date { - return DateTime.toDateUtc(DateTime.nowUnsafe()) -} - -export interface QueueEffectMessage { - readonly v: 1 - readonly runId: string - readonly effectId: string - readonly attempt: number - readonly type: string - readonly input: JsonValue - readonly traceparent?: string - readonly deadlines: { - readonly scheduleToStartTimeoutMs?: number - readonly startToCloseTimeoutMs?: number - readonly heartbeatTimeoutMs?: number - } -} - -export interface QueueSendOptions { - readonly contentType?: 'json' | 'text' | 'bytes' | 'v8' -} - -export interface QueueLike { - send(message: Body, options?: QueueSendOptions): Promise +/** A Cloudflare Queue producer binding, or anything that sends the same way. */ +export interface QueueProducer { + send(message: QueueEffectMessage, options?: { readonly contentType?: 'json' }): Promise } export interface QueueRoute { readonly handles: ReadonlyArray | ((type: string) => boolean) - readonly queue: QueueLike + readonly queue: QueueProducer } export interface CloudflareQueueWorkerOptions { - readonly routes: readonly QueueRoute[] - /** Optional offload hook when input exceeds ~100 KB (Cloudflare Queues cap is 128 KB). */ - readonly offloadInput?: (input: JsonValue) => Promise -} - + /** The first route that handles an effect type gets its attempts. */ + readonly routes: ReadonlyArray + /** Holds an input that would push a message past `maxMessageBytes`. Without it, that dispatch fails. */ + readonly blobs?: BlobStore + readonly maxMessageBytes?: number +} + +export class QueueDispatchError extends Data.TaggedError('QueueDispatchError')<{ + readonly message: string +}> {} + +/** + * An {@link EffectWorker} that sends each attempt to a Cloudflare Queue. A send the queue rejects + * (a 429, for one) fails the attempt under the effect's retry policy. A message the queue accepts + * and later dead-letters is recovered by the effect's schedule-to-start timeout, which every routed + * effect must set. Cancellation reaches a running attempt through its callback responses, since a + * queued message cannot be recalled. + */ export function cloudflareQueueWorker(options: CloudflareQueueWorkerOptions): EffectWorker { - const isHandled = (type: string) => - options.routes.some((route) => + const routeFor = (type: string) => + options.routes.find((route) => Predicate.isFunction(route.handles) ? route.handles(type) : route.handles.includes(type), ) return { dispatchSemantics: 'acknowledged', - handles: isHandled, - dispatch: (task: EffectWorkerTask): Promise => { - const route = options.routes.find((candidate) => - Predicate.isFunction(candidate.handles) - ? candidate.handles(task.type) - : candidate.handles.includes(task.type), - ) - - if (!route) { - return Promise.reject( - new Error(`No Cloudflare Queue route handles effect type "${task.type}"`), - ) - } - - const input = task.input - const estimatedBytes = stringifyJson(input).length - - const offloadPromise = - estimatedBytes > 100_000 && options.offloadInput - ? options.offloadInput(task.input) - : Promise.resolve(input) - - return offloadPromise.then((resolvedInput) => { - const message: QueueEffectMessage = { - v: 1, - runId: task.runId, - effectId: task.effectId, - attempt: task.attempt, - type: task.type, - input: resolvedInput, - deadlines: {}, - } - - return route.queue.send(message, { contentType: 'json' }) - }) - }, - cancel: () => undefined, + handles: (type) => routeFor(type) !== undefined, + dispatch: (task) => Effect.runPromise(sendTask(task, routeFor(task.type), options)), } } -export interface QueueTaskContext { - readonly signal: AbortSignal - readonly heartbeat: (details?: JsonValue) => Promise - readonly secrets: () => Promise> -} - -export type QueueTaskHandler = ( - task: QueueEffectMessage, - ctx: QueueTaskContext, -) => Promise | readonly EventInput[] - -export interface QueueMessageLike { - readonly id: string - readonly timestamp: Date - readonly body: Body - readonly attempts?: number - ack(): void - retry(options?: { delaySeconds?: number }): void -} - -export interface MessageBatchLike { - readonly queue: string - readonly messages: readonly QueueMessageLike[] - ackAll(): void - retryAll(options?: { delaySeconds?: number }): void -} - -export interface QueueConsumerOptions { - readonly handlers: Record - readonly callbackUrl: string - readonly workerToken?: string - readonly fetch?: typeof fetch -} - -const emptySecrets = (): Promise> => Promise.resolve({}) - -export function processQueueMessage( - message: QueueMessageLike, - options: QueueConsumerOptions, -): Promise { - const fetchImpl = options.fetch ?? fetch - const task = message.body - const handler = options.handlers[task.type] - - if (!handler) { - message.retry() - return Promise.resolve() +function sendTask( + task: EffectWorkerTask, + route: QueueRoute | undefined, + options: CloudflareQueueWorkerOptions, +): Effect.Effect { + if (!route) { + return Effect.fail(new QueueDispatchError({ message: `no queue route handles "${task.type}"` })) } - const baseUrl = options.callbackUrl.replace(/\/$/, '') - - const headers = new Headers({ - 'content-type': 'application/json', - }) - - if (options.workerToken) { - headers.set('authorization', `Bearer ${options.workerToken}`) - } - - const startedUrl = `${baseUrl}/runs/${encodeURIComponent(task.runId)}/effects/${encodeURIComponent(task.effectId)}/${task.attempt}/started` - - return fetchImpl(startedUrl, { - method: 'POST', - headers, - body: stringifyJson({}), - }) - .then((startedRes) => { - if (!startedRes.ok) { - message.retry() - return undefined - } - - return startedRes.json().then((startedJson) => { - const startedData = decodeCallbackResponse(startedJson) - - if (startedData._tag === 'None') { - message.retry() - return undefined - } - - const { status, cancelRequested } = startedData.value - - if (status === 'stale' || status === 'cancelled') { - message.ack() - return undefined - } - - message.ack() - - const controller = new AbortController() - - if (cancelRequested) { - controller.abort() - } - - const heartbeatUrl = `${baseUrl}/runs/${encodeURIComponent(task.runId)}/effects/${encodeURIComponent(task.effectId)}/${task.attempt}/heartbeat` - - const heartbeat = (details?: JsonValue): Promise => - fetchImpl(heartbeatUrl, { - method: 'POST', - headers, - body: stringifyJson(details === undefined ? {} : { details }), - }).then((res) => { - if (res.ok) { - return res.json().then((dataJson) => { - const data = decodeCallbackResponse(dataJson) - - if (data._tag === 'Some' && data.value.cancelRequested) { - controller.abort() - } - }) - } - - return undefined - }) - - const handlerPromise = Promise.resolve( - handler(task, { - signal: controller.signal, - heartbeat, - secrets: emptySecrets, + return queueMessage(task, options).pipe( + Effect.flatMap((message) => + Effect.tryPromise({ + try: () => route.queue.send(message, { contentType: 'json' }), + catch: (cause) => + new QueueDispatchError({ + message: `queue send failed: ${cause instanceof Error ? cause.message : String(cause)}`, }), - ) - - return handlerPromise - .then((events) => { - const completeUrl = `${baseUrl}/runs/${encodeURIComponent(task.runId)}/effects/${encodeURIComponent(task.effectId)}/${task.attempt}/complete` - - return fetchImpl(completeUrl, { - method: 'POST', - headers, - body: stringifyJson({ events: [...events] }), - }).then(() => undefined) - }) - .catch((cause: unknown) => { - const error = cause instanceof Error ? cause.message : String(cause) - const failUrl = `${baseUrl}/runs/${encodeURIComponent(task.runId)}/effects/${encodeURIComponent(task.effectId)}/${task.attempt}/fail` - - return fetchImpl(failUrl, { - method: 'POST', - headers, - body: stringifyJson({ error }), - }).then(() => undefined) - }) - }) - }) - .catch(() => { - message.retry() - }) -} - -export function queuePushConsumer(options: QueueConsumerOptions) { - return { - consumeBatch: (batch: MessageBatchLike): Promise => - batch.messages.reduce( - (prev, msg) => prev.then(() => processQueueMessage(msg, options)), - Promise.resolve(), - ), + }), + ), + ) +} + +function queueMessage( + task: EffectWorkerTask, + options: CloudflareQueueWorkerOptions, +): Effect.Effect { + const message: Types.Mutable = { + v: 1, + runId: task.runId, + effectId: task.effectId, + attempt: task.attempt, + type: task.type, } -} - -export interface MemoryQueueSimulation { - duplicateDelivery?: boolean - crashBeforeStarted?: boolean - crashAfterStarted?: boolean - rateLimit429?: boolean -} - -export interface MemoryQueue extends QueueLike { - readonly messages: readonly QueueEffectMessage[] - readonly enqueued: readonly QueueEffectMessage[] - readonly acknowledged: readonly string[] - readonly retried: readonly string[] - receiveBatch(count?: number): MessageBatchLike - setSimulation(sim: MemoryQueueSimulation): void -} - -export function createMemoryQueue(): MemoryQueue { - const enqueuedList: QueueEffectMessage[] = [] - const acknowledgedList: string[] = [] - const retriedList: string[] = [] - let simulation: MemoryQueueSimulation = {} - return { - get messages() { - return [...enqueuedList] - }, - get enqueued() { - return [...enqueuedList] - }, - get acknowledged() { - return [...acknowledgedList] - }, - get retried() { - return [...retriedList] - }, - setSimulation(sim: MemoryQueueSimulation) { - simulation = { ...simulation, ...sim } - }, - send: (message: QueueEffectMessage): Promise => { - if (simulation.rateLimit429) { - return Promise.reject(new Error('429 Too Many Requests: Queue rate limit exceeded')) - } + if (task.details !== undefined) { + message.details = task.details + } - enqueuedList.push(message) - return Promise.resolve() - }, - receiveBatch: (count = 10): MessageBatchLike => { - const selected = enqueuedList.slice(0, count) + if (task.trace) { + message.trace = task.trace + } - const queueMessages: QueueMessageLike[] = selected.map((msg, idx) => { - const id = `msg_${idx}_${msg.effectId}` + if (task.deadlines) { + message.deadlines = task.deadlines + } - return { - id, - timestamp: utcNow(), - body: msg, - ack: () => { - acknowledgedList.push(id) - }, - retry: () => { - retriedList.push(id) - }, - } - }) + const inline: QueueEffectMessage = { ...message, input: task.input } + const limit = options.maxMessageBytes ?? DEFAULT_MAX_QUEUE_MESSAGE_BYTES - if (simulation.duplicateDelivery && queueMessages.length > 0) { - const first = queueMessages[0] + if (utf8JsonBytes(inline) <= limit) { + return Effect.succeed(inline) + } - if (first) { - queueMessages.push({ - id: `${first.id}_dup`, - timestamp: utcNow(), - body: first.body, - ack: () => { - acknowledgedList.push(`${first.id}_dup`) - }, - retry: () => { - retriedList.push(`${first.id}_dup`) - }, - }) - } - } + const { blobs } = options - return { - queue: 'looms-effects-queue', - messages: queueMessages, - ackAll: () => { - for (const m of queueMessages) { - m.ack() - } - }, - retryAll: () => { - for (const m of queueMessages) { - m.retry() - } - }, - } - }, + if (!blobs) { + return Effect.fail( + new QueueDispatchError({ + message: `the "${task.type}" message is over ${limit} bytes and no blob store is configured`, + }), + ) } + + return claimJson(task.input, blobs).pipe( + Effect.tap((ref) => blobs.retain(task.runId, [ref.sha256])), + Effect.map((inputRef): QueueEffectMessage => ({ ...message, inputRef })), + Effect.mapError( + (error) => new QueueDispatchError({ message: `could not store the input: ${error.message}` }), + ), + ) } diff --git a/packages/runtime/src/helpers.ts b/packages/runtime/src/helpers.ts index 0b406a1..06262b0 100644 --- a/packages/runtime/src/helpers.ts +++ b/packages/runtime/src/helpers.ts @@ -26,6 +26,7 @@ import { type DefinitionResolver, type JsonValue, type OutstandingEffect, + type RetryPolicy, type SecretContext, type TraceContext, type RunCursor, @@ -35,7 +36,7 @@ import { type SnapshotStore, } from '@looms/core' -import type { EffectWorkerTask } from './types' +import type { EffectWorkerDeadlines, EffectWorkerTask } from './types' /** What a dispatch carries beyond the effect itself. */ export interface WorkerTaskExtras { @@ -43,6 +44,8 @@ export interface WorkerTaskExtras { readonly details?: JsonValue | null readonly secrets?: SecretContext readonly trace?: TraceContext | null + readonly retry?: RetryPolicy + readonly scheduleToStartAt?: number | null } export function workerTask( @@ -71,9 +74,33 @@ export function workerTask( task.trace = extras.trace } + const deadlines = workerDeadlines(extras) + + if (deadlines) { + task.deadlines = deadlines + } + return task } +function workerDeadlines(extras: WorkerTaskExtras): EffectWorkerDeadlines | undefined { + const deadlines: Types.Mutable = {} + + if (extras.scheduleToStartAt != null) { + deadlines.scheduleToStartAt = extras.scheduleToStartAt + } + + if (extras.retry?.startToCloseTimeoutMs) { + deadlines.startToCloseTimeoutMs = extras.retry.startToCloseTimeoutMs + } + + if (extras.retry?.heartbeatTimeoutMs) { + deadlines.heartbeatTimeoutMs = extras.retry.heartbeatTimeoutMs + } + + return Object.keys(deadlines).length > 0 ? deadlines : undefined +} + export class UnknownDefinitionError extends Data.TaggedError('UnknownDefinitionError')<{ readonly kind: string readonly definitionName: string diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index f3cb34e..ac6ab09 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -11,6 +11,7 @@ export { StartRunConflictError, type CreateRuntimeOptions, type EffectWorker, + type EffectWorkerDeadlines, type EffectWorkerTask, type LoomsRuntime, type RunOperationalStatus, diff --git a/packages/runtime/src/planners.test.ts b/packages/runtime/src/planners.test.ts index 5b9fda8..0d34708 100644 --- a/packages/runtime/src/planners.test.ts +++ b/packages/runtime/src/planners.test.ts @@ -210,6 +210,7 @@ describe('worker effect planner', () => { attempt: 2, type: 'worker.send', input: { message: 'hello' }, + deadlines: { scheduleToStartAt: 1_500 }, }) expect(plan?.queuedEvent?.type).toBe('runtime.effect.queued') diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index 6d05aa6..39e6b8b 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -64,6 +64,7 @@ import { assertSameDurableStart } from './run-start' import { createWakeEngine } from './wake-engine' import { createTimeoutScheduler, type WakeScheduler } from './wake-scheduler' import { createWorkerCallbacks } from './worker-callbacks' +import { assertWorkerDeadlines } from './worker-effect-dispatch' export type { RegisteredDefinition } from '@looms/core' export type { WakeScheduler } from './wake-scheduler' @@ -82,6 +83,7 @@ export { RunOverloadedError } from './ingress' export type { CreateRuntimeOptions, EffectWorker, + EffectWorkerDeadlines, EffectWorkerTask, LoomsRuntime, RunOperationalStatus, @@ -199,6 +201,8 @@ export function createRuntime ({ ...definition.value, diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 7d98409..8a8b371 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -112,6 +112,8 @@ export interface EffectWorkerTask { readonly details?: JsonValue /** The run's W3C trace context, for parenting the worker's spans. */ readonly trace?: TraceContext + /** The attempt's deadlines from the effect's retry policy, when it sets any. */ + readonly deadlines?: EffectWorkerDeadlines /** * Present only on the in-memory dispatch. A transport that persists the task must leave this out * and let the worker read the secret context through an authenticated callback. @@ -119,6 +121,14 @@ export interface EffectWorkerTask { readonly secrets?: SecretContext } +/** Looms enforces these; a worker can use them to pace heartbeats or give up early. */ +export interface EffectWorkerDeadlines { + /** Epoch ms. A `started` callback after this is stale. */ + readonly scheduleToStartAt?: number + readonly startToCloseTimeoutMs?: number + readonly heartbeatTimeoutMs?: number +} + export interface EffectWorker { /** Effect types this worker claims, even when a local handler exists. */ readonly handles?: ReadonlyArray | ((type: string) => boolean) @@ -127,9 +137,11 @@ export interface EffectWorker { dispatch(task: EffectWorkerTask): void | Promise cancel?(task: EffectWorkerTask): void | Promise /** - * Dispatch semantics: - * - 'unacknowledged' (default): failed dispatch is treated as ambiguous (wait for schedule-to-start or operator). - * - 'acknowledged': resolved send means enqueued, thrown error means not enqueued (retryable dispatch with backoff). + * What a rejected `dispatch` means. With `'unacknowledged'` (the default) the task may still have + * arrived, so the attempt waits for its schedule-to-start timeout or an operator. With + * `'acknowledged'` a rejection means the transport refused it, so the attempt fails under the + * retry policy. A transport can still lose an accepted message, so every effect such a worker + * claims must set `retry.scheduleToStartTimeoutMs`. */ readonly dispatchSemantics?: 'unacknowledged' | 'acknowledged' } diff --git a/packages/runtime/src/wake-engine.ts b/packages/runtime/src/wake-engine.ts index f7b1763..26e481c 100644 --- a/packages/runtime/src/wake-engine.ts +++ b/packages/runtime/src/wake-engine.ts @@ -317,7 +317,12 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake ) } - const dispatchEvent = yield* dispatchWorkerEffect(runId, workerPlan, options.worker) + const dispatchEvent = yield* dispatchWorkerEffect( + runId, + workerPlan, + options.worker, + now, + ) const dispatchAppend = yield* appendObserved( store, diff --git a/packages/runtime/src/worker-effect-dispatch.ts b/packages/runtime/src/worker-effect-dispatch.ts index 03a77ed..b1bc907 100644 --- a/packages/runtime/src/worker-effect-dispatch.ts +++ b/packages/runtime/src/worker-effect-dispatch.ts @@ -5,15 +5,18 @@ import { deferredUntil, isPrimitiveEffect, type ComposedRegistry, + type EffectDefinition, type EffectExecutionRecord, type EventEnvelope, type OutstandingEffect, + type RetryPolicy, type RunState, type SecretContext, } from '@looms/core' import { RuntimeExecutionError } from './errors' import { workerTask } from './helpers' +import { planAttemptFailure } from './retry-policy' import type { EffectWorker, EffectWorkerTask } from './types' export interface WorkerEffectDispatchPlan { @@ -21,6 +24,7 @@ export interface WorkerEffectDispatchPlan { readonly attempt: number readonly queuedEvent?: EventEnvelope readonly task: EffectWorkerTask + readonly retry: RetryPolicy | undefined readonly scheduleToStartDeadlineAt: number | null } @@ -136,9 +140,14 @@ export function planWorkerEffects(input: { const retry = input.registry.effects.get(item.effect.type)?.retry + const scheduleToStartDeadlineAt = retry?.scheduleToStartTimeoutMs + ? input.now + retry.scheduleToStartTimeoutMs + : null + return { item, attempt, + retry, queuedEvent: existing?.status === 'queued' ? undefined @@ -153,10 +162,10 @@ export function planWorkerEffects(input: { details: existing?.heartbeatDetails, secrets: input.secrets, trace: input.state.startIdentity?.trace, + retry, + scheduleToStartAt: scheduleToStartDeadlineAt, }), - scheduleToStartDeadlineAt: retry?.scheduleToStartTimeoutMs - ? input.now + retry.scheduleToStartTimeoutMs - : null, + scheduleToStartDeadlineAt, } }) } @@ -189,14 +198,16 @@ function withinInFlightLimit( } /** - * Hand a planned attempt to the worker. A rejected dispatch may still have reached the worker, so - * with a schedule-to-start deadline it is recorded as dispatched and recovered by that timeout; - * without one it is parked as ambiguous for an operator. + * Hand a planned attempt to the worker. When an unacknowledged dispatch rejects, the task may still + * have reached the worker, so with a schedule-to-start deadline it is recorded as dispatched and + * recovered by that timeout; without one it is parked as ambiguous for an operator. An acknowledged + * rejection means nothing was sent, so the attempt fails under the retry policy. */ export function dispatchWorkerEffect( runId: string, plan: WorkerEffectDispatchPlan, worker: EffectWorker | undefined, + now: number, ): Effect.Effect { const dispatched = createEvent(runId, { type: 'runtime.effect.dispatched', @@ -233,16 +244,15 @@ export function dispatchWorkerEffect( Effect.match({ onFailure: (error) => { if (worker.dispatchSemantics === 'acknowledged') { - return createEvent(runId, { - type: 'runtime.effect.failed', - payload: { - effectId: plan.item.effectId, - error: `Worker dispatch failed: ${error.message}`, - }, - threadId: plan.item.threadId, - causationId: plan.item.causingEventId, + return planAttemptFailure({ + runId, + item: plan.item, + attempt: plan.attempt, + error: `worker dispatch failed: ${error.message}`, + now, + retry: plan.retry, origin: { type: 'system' }, - }) + }).event } return plan.scheduleToStartDeadlineAt === null @@ -253,3 +263,42 @@ export function dispatchWorkerEffect( }), ) } + +/** + * An acknowledged transport can lose a message it accepted, for example to a dead-letter queue. + * Only a schedule-to-start timeout notices, so every effect such a worker claims needs one. + */ +export function assertWorkerDeadlines( + registry: ComposedRegistry, + worker: EffectWorker | undefined, +): void { + if (worker?.dispatchSemantics !== 'acknowledged') { + return + } + + const missing = [...registry.effects.values()] + .filter( + (definition) => claimsType(worker, definition) && !definition.retry?.scheduleToStartTimeoutMs, + ) + .map((definition) => definition.type) + + if (missing.length > 0) { + throw new Error( + `An acknowledged worker claims effects without retry.scheduleToStartTimeoutMs: ${missing.join(', ')}`, + ) + } +} + +function claimsType(worker: EffectWorker, definition: EffectDefinition): boolean { + if (definition.execution) { + return definition.execution === 'worker' + } + + const { handles } = worker + + const handled = Predicate.isFunction(handles) + ? handles(definition.type) + : (handles?.includes(definition.type) ?? false) + + return handled || !definition.hasHandler +} diff --git a/packages/runtime/src/worker-protocol.test.ts b/packages/runtime/src/worker-protocol.test.ts index ffe6866..e5efefd 100644 --- a/packages/runtime/src/worker-protocol.test.ts +++ b/packages/runtime/src/worker-protocol.test.ts @@ -15,7 +15,7 @@ import { type RetryPolicy, } from '@looms/core' -import { bearerAuth, createLooms, type EffectWorkerTask } from './index' +import { bearerAuth, createLooms, createRuntime, type EffectWorkerTask } from './index' const workerDefinition = { kind: 'worker-test', @@ -65,6 +65,18 @@ function effectIdentity(events: readonly EventEnvelope[]) { return { effectId: payload.effectId, attempt: payload.attempt } } +async function waitFor(condition: () => boolean, timeoutMs = 1_000): Promise { + const deadline = Date.now() + timeoutMs + + while (!condition()) { + if (Date.now() > deadline) { + throw new Error('condition not met in time') + } + + await Bun.sleep(5) + } +} + function local(): ReadonlyArray { return [{ type: 'worker.done', payload: { local: true } }] } @@ -710,27 +722,68 @@ describe('durable external effect workers', () => { await looms.stop() }) - test('acknowledged dispatch semantics fails and retries without marking ambiguous', async () => { - let attempts = 0 + test('an acknowledged dispatch rejection retries under the policy', async () => { + const dispatched: EffectWorkerTask[] = [] const looms = createLooms({ - modules: [workerModule({ maxAttempts: 3, backoffMs: 1 })], + modules: [workerModule({ maxAttempts: 3, backoffMs: 5, scheduleToStartTimeoutMs: 60_000 })], worker: { dispatchSemantics: 'acknowledged', - dispatch: () => { - attempts += 1 - throw new Error('429 Queue Rate Exceeded') + dispatch: (task) => { + dispatched.push(task) + + if (dispatched.length === 1) { + throw new Error('429 Too Many Requests') + } }, }, }) const started = await looms.start(workerDefinition, {}) - const events = await looms.getEvents(started.runId) - expect(attempts).toBe(1) - expect(events.some((event) => event.type === 'runtime.effect.failed')).toBe(true) - expect(events.some((event) => event.type === 'runtime.effect.ambiguous')).toBe(false) + await waitFor(() => dispatched.length === 2) + + const types = (await looms.getEvents(started.runId)).map((event) => event.type) + expect(types).toContain('runtime.effect.retry.scheduled') + expect(types).not.toContain('runtime.effect.ambiguous') + expect(dispatched.map((task) => task.attempt)).toEqual([1, 2]) + await looms.stop() + }) + + test('an acknowledged worker needs a schedule-to-start timeout on every effect it claims', () => { + expect(() => + createRuntime({ + modules: [workerModule({ maxAttempts: 3 })], + worker: { dispatchSemantics: 'acknowledged', dispatch: () => undefined }, + }), + ).toThrow('worker.do') + }) + + test('a worker task carries the deadlines from the retry policy', async () => { + const dispatched: EffectWorkerTask[] = [] + + const looms = createLooms({ + modules: [ + workerModule({ + maxAttempts: 1, + scheduleToStartTimeoutMs: 1_000, + startToCloseTimeoutMs: 2_000, + heartbeatTimeoutMs: 500, + }), + ], + worker: { dispatch: (task) => dispatched.push(task) }, + }) + + const before = Date.now() + + await looms.start(workerDefinition, {}) + + const deadlines = dispatched[0]?.deadlines + + expect(deadlines?.startToCloseTimeoutMs).toBe(2_000) + expect(deadlines?.heartbeatTimeoutMs).toBe(500) + expect(deadlines?.scheduleToStartAt).toBeGreaterThanOrEqual(before + 1_000) await looms.stop() }) @@ -813,9 +866,17 @@ describe('durable external effect workers', () => { }, }) - await looms.start({ kind: 'fanout', name: 'fanout', version: 'v1' }, {}) + const started = await looms.start({ kind: 'fanout', name: 'fanout', version: 'v1' }, {}) + expect(dispatched).toHaveLength(2) + const first = { effectId: dispatched[0]?.effectId ?? '', attempt: 1 } + + await looms.workerCallback(started.runId, { ...first, kind: 'started' }) + await looms.workerCallback(started.runId, { ...first, kind: 'complete', events: [] }) + + expect(dispatched).toHaveLength(3) + await looms.stop() }) From 180681777f674d6a161ee98acffee177a49d77da Mon Sep 17 00:00:00 2001 From: Dylan Slack Date: Sat, 24 Oct 2026 06:41:50 +0200 Subject: [PATCH 23/23] feat(testing): replay recorded run logs in CI and document effect contracts Adds a production-shaped event-log corpus folded deterministically in CI, plus guidance on versioning effect types and worker handles so fleets can roll out beside each other. Co-authored-by: Cursor --- .changeset/upgrade-safety.md | 10 + docs/production.md | 23 ++ packages/testing/corpus/happy-path.json | 229 ++++++++++++++++++ .../testing/corpus/parked-for-approval.json | 90 +++++++ .../testing/corpus/rejected-approval.json | 186 ++++++++++++++ packages/testing/package.json | 5 +- packages/testing/scripts/record-corpus.ts | 81 +++++++ packages/testing/src/corpus-module.ts | 114 +++++++++ packages/testing/src/index.ts | 20 +- packages/testing/src/replay-corpus.test.ts | 51 ++++ packages/testing/src/replay-corpus.ts | 93 +++++++ 11 files changed, 892 insertions(+), 10 deletions(-) create mode 100644 .changeset/upgrade-safety.md create mode 100644 packages/testing/corpus/happy-path.json create mode 100644 packages/testing/corpus/parked-for-approval.json create mode 100644 packages/testing/corpus/rejected-approval.json create mode 100644 packages/testing/scripts/record-corpus.ts create mode 100644 packages/testing/src/corpus-module.ts create mode 100644 packages/testing/src/replay-corpus.test.ts create mode 100644 packages/testing/src/replay-corpus.ts diff --git a/.changeset/upgrade-safety.md b/.changeset/upgrade-safety.md new file mode 100644 index 0000000..b96c11d --- /dev/null +++ b/.changeset/upgrade-safety.md @@ -0,0 +1,10 @@ +--- +'@swirls/looms': minor +--- + +feat(testing): replay recorded run logs in CI and document effect contracts + +Adds a production-shaped event-log corpus under `packages/testing/corpus`, folded +deterministically in CI with `assertCorpusEntry`. Documents versioning effect +types (`payments.charge@v2`) and worker `handles` so fleets can roll out beside +each other without Temporal-style patches. diff --git a/docs/production.md b/docs/production.md index 5ebbeb4..8cdffdd 100644 --- a/docs/production.md +++ b/docs/production.md @@ -275,6 +275,29 @@ and schemas needed by active old-version runs. If a new projector is bad, stop delivery, deploy a corrected new version, and rebuild from archive; do not move a checkpoint backward after source events were trimmed. +## Effect contracts and worker versioning + +An effect `type` is a durable contract between the run that invokes it and the +worker that claims it. Changing what an input or outcome means under the same +type breaks in-flight attempts and recorded logs. Version the type instead: + +- Prefer an explicit suffix such as `payments.charge@v2` (or a new leaf name) + when the input schema, side effects, or completion events change. +- Keep the old type and its handler until every outstanding attempt of that type + has settled. New runs can route to the new type immediately. +- A worker's `handles` list (or predicate) is how fleets version: run a v1 fleet + that claims only `payments.charge` beside a v2 fleet that claims + `payments.charge@v2`. Do not have two fleets claim the same type with + incompatible behavior. +- Recorded run logs stay valid across binary upgrades because fold is a pure + function of the log and the registered modules. The replay corpus under + `packages/testing/corpus` is folded in CI with `assertCorpusEntry`; re-record + a fixture only after confirming a fold or reducer change is intentional + (`bun packages/testing/scripts/record-corpus.ts`). + +This replaces Temporal `patched()` and worker build-id versioning for Looms +effects: the type string is the contract, and the log is the source of truth. + ## Resolving definitions Pass `definitionResolver` to `createLooms`, `createRuntime`, an actor host, or diff --git a/packages/testing/corpus/happy-path.json b/packages/testing/corpus/happy-path.json new file mode 100644 index 0000000..e8e4f3d --- /dev/null +++ b/packages/testing/corpus/happy-path.json @@ -0,0 +1,229 @@ +{ + "name": "happy-path", + "runId": "corpus_happy", + "stateHash": "54c93dd3bef5a0d8", + "events": [ + { + "id": "evt_mvlc1fqp_2_26de35616ef343748332aa7cc50de476", + "runId": "corpus_happy", + "ts": 1792781874673, + "type": "runtime.run.started", + "payload": { + "rootThreadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7", + "kind": "corpus.checkout", + "definitionName": "checkout", + "definitionVersion": "v1", + "input": 42, + "requestedThreadId": null + }, + "threadId": null, + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 1 + }, + { + "id": "evt_mvlc1fqp_3_6d411648eed140fb948e9a2d501a70b7", + "runId": "corpus_happy", + "ts": 1792781874673, + "type": "runtime.thread.started", + "payload": { + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7", + "kind": "corpus.checkout", + "definitionName": "checkout", + "definitionVersion": "v1", + "input": 42, + "parentThreadId": null + }, + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7", + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 2 + }, + { + "id": "evt_mvlc1fqs_4_5f2054e093a9444aae7453b3c80ccbe4", + "runId": "corpus_happy", + "ts": 1792781874676, + "type": "runtime.effect.attempt.started", + "payload": { + "effectId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7:approval", + "attempt": 1 + }, + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7", + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 3 + }, + { + "id": "evt_mvlc1fqt_5_97929aa090b0463797f8b717e918f3c4", + "runId": "corpus_happy", + "ts": 1792781874677, + "type": "runtime.wait.registered", + "payload": { + "waitId": "approval", + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7", + "on": { + "type": "corpus.approved" + }, + "tag": null + }, + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7", + "parentThreadId": null, + "causationId": "evt_mvlc1fqp_3_6d411648eed140fb948e9a2d501a70b7", + "effectId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7:approval", + "origin": { + "type": "thread", + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7" + }, + "idempotencyKey": null, + "seq": 4 + }, + { + "id": "evt_mvlc1fqv_6_d20e2e807dcd493d8c8354f9beb9f0ac", + "runId": "corpus_happy", + "ts": 1792781874679, + "type": "corpus.approved", + "payload": { + "decision": "approve" + }, + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7", + "parentThreadId": null, + "effectId": null, + "origin": { + "type": "external" + }, + "idempotencyKey": null, + "seq": 5 + }, + { + "id": "evt_mvlc1fqv_7_e3ce4ddf320c474294b816cc44402aff", + "runId": "corpus_happy", + "ts": 1792781874679, + "type": "runtime.wait.satisfied", + "payload": { + "waitId": "approval", + "tag": null, + "event": { + "id": "evt_mvlc1fqv_6_d20e2e807dcd493d8c8354f9beb9f0ac", + "type": "corpus.approved", + "payload": { + "decision": "approve" + } + } + }, + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7", + "parentThreadId": null, + "causationId": "evt_mvlc1fqv_6_d20e2e807dcd493d8c8354f9beb9f0ac", + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 6 + }, + { + "id": "evt_mvlc1fqv_8_4f337cf8ea974e3ab577d93d601120f6", + "runId": "corpus_happy", + "ts": 1792781874679, + "type": "runtime.effect.attempt.started", + "payload": { + "effectId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7:charge", + "attempt": 1 + }, + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7", + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 7 + }, + { + "id": "evt_mvlc1fqv_9_d153a6c35c8845e380ca7fac51282fe0", + "runId": "corpus_happy", + "ts": 1792781874679, + "type": "corpus.charged", + "payload": { + "amount": 42, + "version": "v1" + }, + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7", + "parentThreadId": null, + "causationId": "evt_mvlc1fqv_6_d20e2e807dcd493d8c8354f9beb9f0ac", + "effectId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7:charge", + "origin": { + "type": "thread", + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7" + }, + "idempotencyKey": null, + "seq": 8 + }, + { + "id": "evt_mvlc1fqv_a_292fd8b584394c16b528ed5e59c3fd7e", + "runId": "corpus_happy", + "ts": 1792781874679, + "type": "runtime.effect.attempt.started", + "payload": { + "effectId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7:8:0", + "attempt": 1 + }, + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7", + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 9 + }, + { + "id": "evt_mvlc1fqv_b_e41da8691231444fadf9b320a5cbc3b7", + "runId": "corpus_happy", + "ts": 1792781874679, + "type": "runtime.thread.completed", + "payload": { + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7", + "output": { + "amount": 42, + "ok": true + } + }, + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7", + "parentThreadId": null, + "causationId": "evt_mvlc1fqv_9_d153a6c35c8845e380ca7fac51282fe0", + "effectId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7:8:0", + "origin": { + "type": "thread", + "threadId": "thr_mvlc1fqm_1_5860e37710f14fcc977e4df1bd3af5f7" + }, + "idempotencyKey": null, + "seq": 10 + }, + { + "id": "evt_mvlc1fqv_c_a473a59de74744f4b48a835fe682d5f4", + "runId": "corpus_happy", + "ts": 1792781874679, + "type": "runtime.run.completed", + "payload": { + "output": { + "amount": 42, + "ok": true + }, + "error": null + }, + "threadId": null, + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 11 + } + ] +} diff --git a/packages/testing/corpus/parked-for-approval.json b/packages/testing/corpus/parked-for-approval.json new file mode 100644 index 0000000..9c334c7 --- /dev/null +++ b/packages/testing/corpus/parked-for-approval.json @@ -0,0 +1,90 @@ +{ + "name": "parked-for-approval", + "runId": "corpus_parked", + "stateHash": "f27e896ad3aa1a4c", + "events": [ + { + "id": "evt_mvlc1fqz_o_a67d820299d8441b9178b509cc3a1a88", + "runId": "corpus_parked", + "ts": 1792781874683, + "type": "runtime.run.started", + "payload": { + "rootThreadId": "thr_mvlc1fqz_n_0fe90ec68d824cc7beb0817d5367a6cc", + "kind": "corpus.checkout", + "definitionName": "checkout", + "definitionVersion": "v1", + "input": 99, + "requestedThreadId": null + }, + "threadId": null, + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 1 + }, + { + "id": "evt_mvlc1fqz_p_a91976be42584a0da016fc199ce3637f", + "runId": "corpus_parked", + "ts": 1792781874683, + "type": "runtime.thread.started", + "payload": { + "threadId": "thr_mvlc1fqz_n_0fe90ec68d824cc7beb0817d5367a6cc", + "kind": "corpus.checkout", + "definitionName": "checkout", + "definitionVersion": "v1", + "input": 99, + "parentThreadId": null + }, + "threadId": "thr_mvlc1fqz_n_0fe90ec68d824cc7beb0817d5367a6cc", + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 2 + }, + { + "id": "evt_mvlc1fqz_q_230db458cb574fbe9b2021a015dc3fe3", + "runId": "corpus_parked", + "ts": 1792781874683, + "type": "runtime.effect.attempt.started", + "payload": { + "effectId": "thr_mvlc1fqz_n_0fe90ec68d824cc7beb0817d5367a6cc:approval", + "attempt": 1 + }, + "threadId": "thr_mvlc1fqz_n_0fe90ec68d824cc7beb0817d5367a6cc", + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 3 + }, + { + "id": "evt_mvlc1fqz_r_3185fc3e8d0a48a0b034576af9d69d24", + "runId": "corpus_parked", + "ts": 1792781874683, + "type": "runtime.wait.registered", + "payload": { + "waitId": "approval", + "threadId": "thr_mvlc1fqz_n_0fe90ec68d824cc7beb0817d5367a6cc", + "on": { + "type": "corpus.approved" + }, + "tag": null + }, + "threadId": "thr_mvlc1fqz_n_0fe90ec68d824cc7beb0817d5367a6cc", + "parentThreadId": null, + "causationId": "evt_mvlc1fqz_p_a91976be42584a0da016fc199ce3637f", + "effectId": "thr_mvlc1fqz_n_0fe90ec68d824cc7beb0817d5367a6cc:approval", + "origin": { + "type": "thread", + "threadId": "thr_mvlc1fqz_n_0fe90ec68d824cc7beb0817d5367a6cc" + }, + "idempotencyKey": null, + "seq": 4 + } + ] +} diff --git a/packages/testing/corpus/rejected-approval.json b/packages/testing/corpus/rejected-approval.json new file mode 100644 index 0000000..3e84e2f --- /dev/null +++ b/packages/testing/corpus/rejected-approval.json @@ -0,0 +1,186 @@ +{ + "name": "rejected-approval", + "runId": "corpus_reject", + "stateHash": "c9a3b9d7c73c15f8", + "events": [ + { + "id": "evt_mvlc1fqx_e_91f5282a15a342258cc8fb6f6bd17546", + "runId": "corpus_reject", + "ts": 1792781874681, + "type": "runtime.run.started", + "payload": { + "rootThreadId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba", + "kind": "corpus.checkout", + "definitionName": "checkout", + "definitionVersion": "v1", + "input": 7, + "requestedThreadId": null + }, + "threadId": null, + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 1 + }, + { + "id": "evt_mvlc1fqx_f_f78411c3dc1d4257941ddbd465c64b53", + "runId": "corpus_reject", + "ts": 1792781874681, + "type": "runtime.thread.started", + "payload": { + "threadId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba", + "kind": "corpus.checkout", + "definitionName": "checkout", + "definitionVersion": "v1", + "input": 7, + "parentThreadId": null + }, + "threadId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba", + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 2 + }, + { + "id": "evt_mvlc1fqx_g_fb7b092e61ca42f1b7c3bbf74f759d33", + "runId": "corpus_reject", + "ts": 1792781874681, + "type": "runtime.effect.attempt.started", + "payload": { + "effectId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba:approval", + "attempt": 1 + }, + "threadId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba", + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 3 + }, + { + "id": "evt_mvlc1fqx_h_1799a9de23cc41fda112ce3a742f91a3", + "runId": "corpus_reject", + "ts": 1792781874681, + "type": "runtime.wait.registered", + "payload": { + "waitId": "approval", + "threadId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba", + "on": { + "type": "corpus.approved" + }, + "tag": null + }, + "threadId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba", + "parentThreadId": null, + "causationId": "evt_mvlc1fqx_f_f78411c3dc1d4257941ddbd465c64b53", + "effectId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba:approval", + "origin": { + "type": "thread", + "threadId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba" + }, + "idempotencyKey": null, + "seq": 4 + }, + { + "id": "evt_mvlc1fqy_i_373201b7aabe4d5bbc13a51f467de48d", + "runId": "corpus_reject", + "ts": 1792781874682, + "type": "corpus.approved", + "payload": { + "decision": "reject" + }, + "threadId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba", + "parentThreadId": null, + "effectId": null, + "origin": { + "type": "external" + }, + "idempotencyKey": null, + "seq": 5 + }, + { + "id": "evt_mvlc1fqy_j_72d30ec3a86946238ba94ca6e5e95b32", + "runId": "corpus_reject", + "ts": 1792781874682, + "type": "runtime.wait.satisfied", + "payload": { + "waitId": "approval", + "tag": null, + "event": { + "id": "evt_mvlc1fqy_i_373201b7aabe4d5bbc13a51f467de48d", + "type": "corpus.approved", + "payload": { + "decision": "reject" + } + } + }, + "threadId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba", + "parentThreadId": null, + "causationId": "evt_mvlc1fqy_i_373201b7aabe4d5bbc13a51f467de48d", + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 6 + }, + { + "id": "evt_mvlc1fqy_k_2c40eb4b74c54305b825033b74689681", + "runId": "corpus_reject", + "ts": 1792781874682, + "type": "runtime.effect.attempt.started", + "payload": { + "effectId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba:5:0", + "attempt": 1 + }, + "threadId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba", + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 7 + }, + { + "id": "evt_mvlc1fqy_l_08d314d59d174fc0a6fdfae7ef3332d3", + "runId": "corpus_reject", + "ts": 1792781874682, + "type": "runtime.thread.failed", + "payload": { + "threadId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba", + "error": "rejected" + }, + "threadId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba", + "parentThreadId": null, + "causationId": "evt_mvlc1fqy_i_373201b7aabe4d5bbc13a51f467de48d", + "effectId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba:5:0", + "origin": { + "type": "thread", + "threadId": "thr_mvlc1fqx_d_8fefc05a43bb4db98307621ac0e2c5ba" + }, + "idempotencyKey": null, + "seq": 8 + }, + { + "id": "evt_mvlc1fqy_m_635090a103124f279e34980782a759a7", + "runId": "corpus_reject", + "ts": 1792781874682, + "type": "runtime.run.completed", + "payload": { + "output": null, + "error": "rejected" + }, + "threadId": null, + "parentThreadId": null, + "origin": { + "type": "system" + }, + "idempotencyKey": null, + "seq": 9 + } + ] +} diff --git a/packages/testing/package.json b/packages/testing/package.json index 869a1bb..a47e17a 100644 --- a/packages/testing/package.json +++ b/packages/testing/package.json @@ -2,7 +2,7 @@ "name": "@looms/testing", "version": "0.0.1", "private": true, - "description": "Test helpers and module conformance for Looms", + "description": "Test helpers, module conformance, and upgrade-safety replay corpus for Looms", "license": "Apache-2.0", "files": [ "dist", @@ -21,7 +21,8 @@ "scripts": { "build": "bun x tsc -p tsconfig.build.json", "check-types": "bun x tsc -p tsconfig.build.json --noEmit", - "test": "bun test src" + "test": "bun test src", + "record-corpus": "bun scripts/record-corpus.ts" }, "dependencies": { "@looms/core": "workspace:*", diff --git a/packages/testing/scripts/record-corpus.ts b/packages/testing/scripts/record-corpus.ts new file mode 100644 index 0000000..d1ea08f --- /dev/null +++ b/packages/testing/scripts/record-corpus.ts @@ -0,0 +1,81 @@ +/** + * Records production-shaped run logs into packages/testing/corpus/. + * Run from the repo root: `bun packages/testing/scripts/record-corpus.ts` + */ +import { mkdirSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { Option, Schema } from 'effect' + +import { + encodeLoomsEvent, + hashRunState, + stringifyJsonPretty, + type EventEnvelope, + type EventInput, + type JsonValue, +} from '@looms/core' +import { createLooms } from '@looms/runtime' + +import { corpusDefinition, corpusModule } from '../src/corpus-module' +import { assertReplayDeterministicEvents, corpusRegistry } from '../src/replay-corpus' + +const corpusDir = join(dirname(fileURLToPath(import.meta.url)), '..', 'corpus') + +const ChargeInputSchema = Schema.Struct({ amount: Schema.Finite }) +const decodeChargeInput = Schema.decodeUnknownOption(ChargeInputSchema) + +function chargeHandler(input: JsonValue): ReadonlyArray { + const amount = Option.getOrElse(decodeChargeInput(input), () => ({ amount: 0 })).amount + + return [{ type: 'corpus.charged', payload: { amount, version: 'v1' } }] +} + +async function record( + name: string, + run: (looms: ReturnType) => Promise<{ runId: string }>, +) { + const looms = createLooms({ modules: [corpusModule({ charge: chargeHandler })] }) + const { runId } = await run(looms) + const events = await looms.getEvents(runId) + const state = assertReplayDeterministicEvents(events, corpusRegistry(), runId) + const stateHash = hashRunState(state) + + const entry = { + name, + runId, + stateHash, + events: events.map((event: EventEnvelope) => encodeLoomsEvent(event)), + } + + writeFileSync(join(corpusDir, `${name}.json`), `${stringifyJsonPretty(entry)}\n`) + await looms.stop() + console.log(`recorded ${name} (${events.length} events, hash ${stateHash})`) +} + +mkdirSync(corpusDir, { recursive: true }) + +await record('happy-path', async (looms) => { + const started = await looms.start(corpusDefinition, 42, { runId: 'corpus_happy' }) + + await looms.signal(started.runId, [ + { type: 'corpus.approved', payload: { decision: 'approve' }, threadId: started.threadId }, + ]) + + return started +}) + +await record('rejected-approval', async (looms) => { + const started = await looms.start(corpusDefinition, 7, { runId: 'corpus_reject' }) + + await looms.signal(started.runId, [ + { type: 'corpus.approved', payload: { decision: 'reject' }, threadId: started.threadId }, + ]) + + return started +}) + +await record('parked-for-approval', async (looms) => { + return looms.start(corpusDefinition, 99, { runId: 'corpus_parked' }) +}) diff --git a/packages/testing/src/corpus-module.ts b/packages/testing/src/corpus-module.ts new file mode 100644 index 0000000..80a1a59 --- /dev/null +++ b/packages/testing/src/corpus-module.ts @@ -0,0 +1,114 @@ +import { Option, Schema } from 'effect' + +import { + complete, + defineEffect, + defineEventCatalog, + defineRuntimeModule, + defineThread, + fail, + invoke, + payload, + wait, + type EventInput, + type JsonValue, +} from '@looms/core' + +/** + * A small production-shaped module used to record and replay the upgrade-safety corpus. + * Keep its event and effect contracts stable; add a new type when meaning changes. + */ +export const corpusDefinition = { + kind: 'corpus.checkout', + name: 'checkout', + version: 'v1', +} + +export const corpusChargeV1 = 'payments.charge' +export const corpusChargeV2 = 'payments.charge@v2' + +const DecisionSchema = Schema.Union([Schema.Literal('approve'), Schema.Literal('reject')]) +const ApprovedPayloadSchema = Schema.Struct({ decision: DecisionSchema }) +const CheckoutInputSchema = Schema.Finite +const decodeApproved = Schema.decodeUnknownOption(ApprovedPayloadSchema) +const decodeCheckoutInput = Schema.decodeUnknownOption(CheckoutInputSchema) + +export function corpusModule(options?: { + readonly charge?: (input: JsonValue) => ReadonlyArray +}) { + const charge = defineEffect({ + type: corpusChargeV1, + retry: { maxAttempts: 3, backoffMs: 1, scheduleToStartTimeoutMs: 60_000 }, + execute: options?.charge, + }) + + const chargeV2 = defineEffect({ + type: corpusChargeV2, + retry: { maxAttempts: 1, scheduleToStartTimeoutMs: 60_000 }, + execute: options?.charge, + }) + + return defineRuntimeModule({ + namespace: 'corpus', + protocolVersion: '1.0.0', + events: defineEventCatalog('corpus', { + charged: payload<{ amount: number; version: string }>(), + approved: payload<{ decision: 'approve' | 'reject' }>(), + }), + effects: { charge, chargeV2 }, + threads: { + 'corpus.checkout': defineThread({ + kind: 'corpus.checkout', + shape: Schema.Struct({ + amount: Schema.Finite, + charged: Schema.Boolean, + decision: Schema.NullOr(DecisionSchema), + }), + initialState: (ctx) => { + const decision: 'approve' | 'reject' | null = null + + return { + amount: Option.getOrElse(decodeCheckoutInput(ctx.input), () => 0), + charged: false, + decision, + } + }, + step: (state, event) => { + if (event.type === 'corpus.approved') { + return Option.match(decodeApproved(event.payload), { + onNone: () => state, + onSome: ({ decision }) => ({ ...state, decision }), + }) + } + + if (event.type === 'corpus.charged') { + return { ...state, charged: true } + } + + return state + }, + effects: (state) => { + if (state.decision === 'reject') { + return [fail('rejected')] + } + + if (state.decision === null) { + return [ + wait({ + waitId: 'approval', + on: { type: 'corpus.approved' }, + }), + ] + } + + if (!state.charged) { + return [invoke(charge, { amount: state.amount }, 'charge')] + } + + return [complete({ amount: state.amount, ok: true })] + }, + }), + }, + definitions: [corpusDefinition], + }) +} diff --git a/packages/testing/src/index.ts b/packages/testing/src/index.ts index 27aae46..17246e6 100644 --- a/packages/testing/src/index.ts +++ b/packages/testing/src/index.ts @@ -3,12 +3,10 @@ import { Effect, Predicate } from 'effect' import { composeModules, EventStoreTag, - foldRun, makeMemoryEventStore, ModuleCompositionError, EventStoreError, snapshotStoreOf, - stringifyJson, withSnapshotStore, type AnyRuntimeModule, type AppendOptions, @@ -17,6 +15,8 @@ import { } from '@looms/core' import { createRuntime, type LoomsRuntime } from '@looms/runtime' +import { assertReplayDeterministicEvents } from './replay-corpus' + type NamedEffect = { readonly type: string readonly hasExecute: boolean @@ -71,15 +71,19 @@ export function createTestRuntime(modules: readonly AnyRuntimeModule[]): Promise export function assertReplayDeterministic(test: TestRuntime, runId: string): Promise { return test.run(test.runtime.getEvents(runId)).then((events) => { - const first = foldRun(events, test.runtime.registry, { runId }) - const second = foldRun(events, test.runtime.registry, { runId }) - - if (stringifyJson(first) !== stringifyJson(second)) { - throw new Error(`Replay is not deterministic for run ${runId}`) - } + assertReplayDeterministicEvents(events, test.runtime.registry, runId) }) } +export { + assertCorpusEntry, + assertReplayDeterministicEvents, + corpusRegistry, + readCorpusEntry, + type CorpusEntry, +} from './replay-corpus' +export { corpusChargeV1, corpusChargeV2, corpusDefinition, corpusModule } from './corpus-module' + export function moduleConformance(module: AnyRuntimeModule): string[] { const errors: string[] = [] diff --git a/packages/testing/src/replay-corpus.test.ts b/packages/testing/src/replay-corpus.test.ts new file mode 100644 index 0000000..0a888bd --- /dev/null +++ b/packages/testing/src/replay-corpus.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, test } from 'bun:test' +import { join } from 'node:path' + +import { Schema } from 'effect' + +import { + assertCorpusEntry, + corpusRegistry, + readCorpusEntry, + type CorpusEntry, +} from './replay-corpus' + +const decodeJson = Schema.decodeUnknownOption(Schema.Json) + +async function loadCorpus(): Promise> { + const directory = join(import.meta.dir, '..', 'corpus') + const listing = await Array.fromAsync(new Bun.Glob('*.json').scan({ cwd: directory })) + + return Promise.all( + listing.toSorted().map(async (name) => { + const file = Bun.file(join(directory, name)) + const decoded = decodeJson(await file.json()) + + if (decoded._tag === 'None') { + throw new Error(`corpus fixture ${name} is not JSON`) + } + + return readCorpusEntry(decoded.value) + }), + ) +} + +describe('upgrade-safety replay corpus', () => { + test('every recorded log folds deterministically to its stored state hash', async () => { + const corpus = await loadCorpus() + const registry = corpusRegistry() + + expect(corpus.map((entry) => entry.name)).toEqual([ + 'happy-path', + 'parked-for-approval', + 'rejected-approval', + ]) + + for (const entry of corpus) { + const state = assertCorpusEntry(entry, registry) + + expect(state.runId).toBe(entry.runId) + expect(entry.events.length).toBeGreaterThan(0) + } + }) +}) diff --git a/packages/testing/src/replay-corpus.ts b/packages/testing/src/replay-corpus.ts new file mode 100644 index 0000000..9fac771 --- /dev/null +++ b/packages/testing/src/replay-corpus.ts @@ -0,0 +1,93 @@ +import { Schema } from 'effect' + +import { + composeModules, + EventEnvelopeSchema, + foldRun, + fromWireEvent, + hashRunState, + stringifyJson, + type AnyRuntimeModule, + type EventEnvelope, + type FoldRegistry, + type JsonValue, + type RunState, +} from '@looms/core' + +import { corpusModule } from './corpus-module' + +const CorpusEntrySchema = Schema.Struct({ + name: Schema.String, + runId: Schema.String, + events: Schema.Array(EventEnvelopeSchema), + /** Hash of the folded state when the log was recorded. */ + stateHash: Schema.String, +}) + +export type CorpusEntry = { + readonly name: string + readonly runId: string + readonly events: ReadonlyArray + readonly stateHash: string +} + +const decodeEntry = Schema.decodeUnknownSync(CorpusEntrySchema) + +/** Modules every corpus fixture was recorded against. Keep them stable across releases. */ +export function corpusRegistry( + modules: ReadonlyArray = [corpusModule()], +): FoldRegistry { + return composeModules(modules) +} + +/** + * Fold the same log twice and require a byte-identical {@link RunState}. Use this in CI against a + * recorded corpus so a fold or protocol change that breaks replay fails before release. + */ +export function assertReplayDeterministicEvents( + events: ReadonlyArray, + registry: FoldRegistry, + runId: string, +): RunState { + const first = foldRun(events, registry, { runId }) + const second = foldRun(events, registry, { runId }) + + if (stringifyJson(first) !== stringifyJson(second)) { + throw new Error(`Replay is not deterministic for run ${runId}`) + } + + return first +} + +/** Parse one recorded corpus JSON document. */ +export function readCorpusEntry(raw: JsonValue): CorpusEntry { + const entry = decodeEntry(raw) + + return { + name: entry.name, + runId: entry.runId, + stateHash: entry.stateHash, + events: entry.events.map(fromWireEvent), + } +} + +/** + * Replay one recorded log: fold must be deterministic and must match the hash taken when the + * fixture was written. A changed hash means the fold or a domain reducer moved under a recorded + * production-shaped log. + */ +export function assertCorpusEntry( + entry: CorpusEntry, + registry: FoldRegistry = corpusRegistry(), +): RunState { + const state = assertReplayDeterministicEvents(entry.events, registry, entry.runId) + const actual = hashRunState(state) + + if (actual !== entry.stateHash) { + throw new Error( + `Corpus "${entry.name}" folded to ${actual}, expected ${entry.stateHash}. Re-record only after confirming the change is intentional.`, + ) + } + + return state +}