diff --git a/.changeset/blob-offload.md b/.changeset/blob-offload.md new file mode 100644 index 0000000..8a46da6 --- /dev/null +++ b/.changeset/blob-offload.md @@ -0,0 +1,5 @@ +--- +'@swirls/looms': minor +--- + +`createLooms({ blobStore })` and `LoomsDurableObject` `blobs` store strings over 64 KiB by SHA-256 and keep a reference in the event or snapshot. A payload that is still over the 1 MiB event limit is stored as one blob. Reads, subscribers, folds, projections, and snapshot loads resolve the reference, so a run can complete with a payload larger than that limit. Trim removes a blob body once nothing retained still points at it. `memoryBlobStore`, `filesystemBlobStore`, and `r2BlobStore` are the included stores. Without a blob store, events over 1 MiB and snapshots over 8 MiB are still rejected. diff --git a/.changeset/child-runs.md b/.changeset/child-runs.md new file mode 100644 index 0000000..3be0a62 --- /dev/null +++ b/.changeset/child-runs.md @@ -0,0 +1,9 @@ +--- +'@swirls/looms': minor +--- + +feat(runtime)!: launch workflow children as their own runs + +A workflow can set `childExecution: 'run'` so spawn, map, fanout, and loop start a child run instead of a thread. The child run id is stable across retries. When the child finishes it signals `runtime.child.settled`, which completes the parent's wait. `childClose` is `terminate` or `wait`. The default remains in-process threads. + +BREAKING CHANGE: `RunLauncher` implementations must provide `signal` and `cancel`. diff --git a/.changeset/definition-resolver.md b/.changeset/definition-resolver.md new file mode 100644 index 0000000..9314e40 --- /dev/null +++ b/.changeset/definition-resolver.md @@ -0,0 +1,7 @@ +--- +'@swirls/looms': minor +--- + +Definitions can be loaded at start, spawn, and recovery through `definitionResolver`, with registered definitions checked first and an LRU cache in front. A missing definition fails the start before any event is written, and the accepted version stays pinned on the run. Evaluator definitions now include `version`, defaulting to `v1`, and lookup takes the name and version. + +BREAKING CHANGE: `EvaluatorDefinition` requires `version`, and evaluator lookup is `(name, version)` instead of `(name)`. `defineEvaluator` still defaults the version to `v1`. diff --git a/.changeset/effect-reliability.md b/.changeset/effect-reliability.md new file mode 100644 index 0000000..5e278e9 --- /dev/null +++ b/.changeset/effect-reliability.md @@ -0,0 +1,9 @@ +--- +'@swirls/looms': minor +--- + +Worker callbacks now answer `{ status: 'accepted' | 'stale' | 'cancelled', cancelRequested }` instead of the run state. `createLooms().workerCallback` and `createLoomsClient().workerCallback` return that result. `cancelRequested` tells an attempt that is already running to stop, because a dispatch that has left the host cannot be recalled. A stale report records nothing. + +`failEffect({ error, nonRetryable, retryAfterMs })` fails the current attempt. `nonRetryable` ends the effect even when the retry policy has attempts left. `retryAfterMs` waits at least that long before the next attempt and never shortens the policy backoff. Worker `fail` callbacks take the same two fields. + +A heartbeat may include `details`. The latest details are kept across retries and included on the next `EffectWorkerTask`, so a resumed attempt can continue from the recorded progress. diff --git a/.changeset/hibernating-websockets.md b/.changeset/hibernating-websockets.md new file mode 100644 index 0000000..152ff72 --- /dev/null +++ b/.changeset/hibernating-websockets.md @@ -0,0 +1,5 @@ +--- +'@swirls/looms': minor +--- + +Durable Objects accept a hibernating WebSocket on `GET /runs/:id/events`. The socket replays the retained log, then receives later events without a subscription held in memory. SSE sends the same `hello` and `resync` frames. A cursor below the retained head is told to resume from `head` instead of silently skipping trimmed events. `subscribeEvents` and the React store take `transport: "websocket" | "sse" | "auto"`; the default remains SSE, and a numeric reconnect delay still works. diff --git a/.changeset/otel-trace.md b/.changeset/otel-trace.md new file mode 100644 index 0000000..3580c53 --- /dev/null +++ b/.changeset/otel-trace.md @@ -0,0 +1,5 @@ +--- +'@swirls/looms': minor +--- + +Runs carry the caller's W3C trace context. `POST /runs` reads the `traceparent` and `tracestate` headers, and `start`, `startRun`, and `client.startRun` accept `trace`. A valid context is recorded on `runtime.run.started` and `startIdentity.trace`, then passed unchanged to effect handlers (`ctx.trace`), worker tasks (`task.trace`), and every launched or child run unless the launch sets its own. diff --git a/.changeset/queue-worker.md b/.changeset/queue-worker.md new file mode 100644 index 0000000..d3b2f78 --- /dev/null +++ b/.changeset/queue-worker.md @@ -0,0 +1,9 @@ +--- +'@swirls/looms': minor +--- + +feat(cloudflare): run worker effects over Cloudflare Queues + +`cloudflareQueueWorker` sends each attempt to a queue as JSON, with the run's trace context and the attempt's deadlines; inputs over 100 KB travel as a blob reference and secrets stay off the queue. `queueConsumer` (push) and `createQueuePullConsumer` with `cloudflareQueuePullSource` (HTTP pull) run handlers that heartbeat, read secrets through the worker callback API, and stop when the run is cancelled. The consumer acks after `started`, so Looms owns retries; duplicate and stale deliveries are dropped, and dead-lettered messages are recovered by the schedule-to-start timeout. The pull consumer batches acks, backs off when idle, reports the backlog, and drains on `stop()`. `createMemoryQueue` gives tests the same lease and dead-letter behavior. + +In the runtime, `EffectWorker.dispatchSemantics: 'acknowledged'` fails a rejected dispatch under the retry policy instead of parking it as ambiguous, and creating a runtime fails if such a worker claims an effect without `retry.scheduleToStartTimeoutMs`. Worker tasks carry `deadlines`, and `maxInFlightWorkerEffects` caps the worker attempts one run has in flight. diff --git a/.changeset/run-context.md b/.changeset/run-context.md new file mode 100644 index 0000000..d38fe7a --- /dev/null +++ b/.changeset/run-context.md @@ -0,0 +1,5 @@ +--- +'@swirls/looms': minor +--- + +Runs accept immutable `attributes` and a `secrets` context. Attributes are recorded on `runtime.run.started`, the run state, and the run summary, and are passed to `authorize` when present. Secrets stay in a side store: they are available to effect handlers and in-process worker dispatch, and they are not written to events, snapshots, or run responses. A worker whose transport persists tasks reads them with `POST /runs/:runId/effects/:effectId/:attempt/secrets` (`client.workerSecrets`), which only answers the attempt that is currently running. diff --git a/.changeset/run-grants.md b/.changeset/run-grants.md new file mode 100644 index 0000000..4e1f007 --- /dev/null +++ b/.changeset/run-grants.md @@ -0,0 +1,7 @@ +--- +'@swirls/looms': minor +--- + +feat(runtime): authorize browser event streams with a signed run grant + +`grantAuth` checks a short-lived HMAC grant from the `grant` query parameter or a `v1.` WebSocket subprotocol. The grant allows one principal to read one run, which is the credential a browser can send on EventSource or WebSocket. The Durable Object socket echoes a subprotocol grant on the upgrade, and `grantSubprotocol` gives other hosts the value to echo. diff --git a/.oxlintrc.json b/.oxlintrc.json index e56ea0a..4c413cb 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -85,7 +85,8 @@ "files": [ "apps/demo/src/server.ts", "packages/debugger/src/server/**/*.ts", - "packages/s2/src/lite.ts" + "packages/s2/src/lite.ts", + "packages/runtime/src/filesystem-blob-store.ts" ], "rules": { "effecttsgo/global-fetch": "off", diff --git a/apps/docs/src/docs-manifest.json b/apps/docs/src/docs-manifest.json index 4b05ea8..1fdc569 100644 --- a/apps/docs/src/docs-manifest.json +++ b/apps/docs/src/docs-manifest.json @@ -830,7 +830,7 @@ "level": 2 } ], - "text": "Reliability and recovery Durability depends on the host and storage you choose. In-memory examples lose history on exit. SQLite persists locally; production actor hosts must preserve one writer per run and provide wake scheduling. Replay and recovery are different operations Historical replay folds recorded events to reconstruct state and does not dispatch effects. Recovery loads a snapshot and later events, then resumes unfinished work. An effect that lacks a recorded outcome may execute again. Looms does not promise exactly-once actions across arbitrary external services. A crash after an external action Looms requests an external action. The service performs it. The host stops before recording success. Recovery cannot infer success from the local log alone. Pass ctx.effectId as an idempotency key where the external API supports it. Check that the service retains the key for as long as you may need to retry the action. For unsupported APIs, store a durable operation ID, query the provider for the outcome, or reconcile before retrying. Never assume adding an arbitrary header makes an API idempotent. Retries and errors Effect retry policies count total attempts: maxAttempts: 3 means one initial attempt and up to two retries. Attempt starts and retry deadlines are recorded before the runtime waits, so a restart preserves the retry budget and backoff. Configure start, heartbeat, and execution timeouts for work delivered to external workers. Worker callbacks include the effect ID and attempt number; stale callbacks cannot complete a newer attempt. Exhaustion records runtime.effect.failed . Signals and webhook delivery Use the embed start/signal idempotencyKey option with a stable runId for retries of the same logical request. Verify webhook signatures and transform allowed webhook payloads into typed events on the server. Deduplicate by the provider delivery ID and correlate by run, thread, and domain identifiers. Retrying with a fresh key defeats deduplication. Interrupted model streams A model call can produce partial deltas before failing. Those deltas may already be recorded; resuming work may incur another model call and different output. Render partial content as provisional, inspect completion events, and decide how your application handles abandoned partial turns. The adapter does not guarantee exactly-once billing or identical regenerated text. Snapshots, retention, and projections A compatible snapshot bounds replay work; it is not a backup or a schema migration. Trimming events removes older audit and time-travel history and can prevent rebuilding new projections from the beginning. Define a retention and archive policy before enabling trimAfterSnapshot. Projectors run after local commit. Durable delivery stores a cursor per projector name and version, retries without advancing that cursor, and dead-letters exhausted batches. A run can finish before an external index or webhook catches up, so monitor lag and keep projector writes idempotent. Test recovery Test a restart while parked, an interruption during an external action, duplicate webhook delivery, and a deployment with existing runs. The testing guide provides the local restart test. Review versioning before changing definitions. ) }" + "text": "Reliability and recovery Durability depends on the host and storage you choose. In-memory examples lose history on exit. SQLite persists locally; production actor hosts must preserve one writer per run and provide wake scheduling. Replay and recovery are different operations Historical replay folds recorded events to reconstruct state and does not dispatch effects. Recovery loads a snapshot and later events, then resumes unfinished work. An effect that lacks a recorded outcome may execute again. Looms does not promise exactly-once actions across arbitrary external services. A crash after an external action Looms requests an external action. The service performs it. The host stops before recording success. Recovery cannot infer success from the local log alone. Pass ctx.effectId as an idempotency key where the external API supports it. Check that the service retains the key for as long as you may need to retry the action. For unsupported APIs, store a durable operation ID, query the provider for the outcome, or reconcile before retrying. Never assume adding an arbitrary header makes an API idempotent. Retries and errors Effect retry policies count total attempts: maxAttempts: 3 means one initial attempt and up to two retries. Attempt starts and retry deadlines are recorded before the runtime waits, so a restart preserves the retry budget and backoff. Configure start, heartbeat, and execution timeouts for work delivered to external workers. Worker callbacks include the effect ID and attempt number; stale callbacks cannot complete a newer attempt. Exhaustion records runtime.effect.failed . A handler can stop that budget early with failEffect and nonRetryable: true , or ask the next attempt to wait at least retryAfterMs without shortening the policy backoff. A worker heartbeat can carry details ; the next dispatch of that effect includes the latest details so the attempt can resume. The callback response is {'{ status, cancelRequested }'} , not the run state. cancelRequested tells an attempt that is already running to stop. Signals and webhook delivery Use the embed start/signal idempotencyKey option with a stable runId for retries of the same logical request. Verify webhook signatures and transform allowed webhook payloads into typed events on the server. Deduplicate by the provider delivery ID and correlate by run, thread, and domain identifiers. Retrying with a fresh key defeats deduplication. Interrupted model streams A model call can produce partial deltas before failing. Those deltas may already be recorded; resuming work may incur another model call and different output. Render partial content as provisional, inspect completion events, and decide how your application handles abandoned partial turns. The adapter does not guarantee exactly-once billing or identical regenerated text. Snapshots, retention, and projections A compatible snapshot bounds replay work; it is not a backup or a schema migration. Trimming events removes older audit and time-travel history and can prevent rebuilding new projections from the beginning. Define a retention and archive policy before enabling trimAfterSnapshot. Projectors run after local commit. Durable delivery stores a cursor per projector name and version, retries without advancing that cursor, and dead-letters exhausted batches. A run can finish before an external index or webhook catches up, so monitor lag and keep projector writes idempotent. Test recovery Test a restart while parked, an interruption during an external action, duplicate webhook delivery, and a deployment with existing runs. The testing guide provides the local restart test. Review versioning before changing definitions. ) }" }, { "path": "/docs/schedules", diff --git a/apps/docs/src/routes/docs.reliability.tsx b/apps/docs/src/routes/docs.reliability.tsx index 243e287..4ccc211 100644 --- a/apps/docs/src/routes/docs.reliability.tsx +++ b/apps/docs/src/routes/docs.reliability.tsx @@ -67,7 +67,13 @@ function Page() { a restart preserves the retry budget and backoff. Configure start, heartbeat, and execution timeouts for work delivered to external workers. Worker callbacks include the effect ID and attempt number; stale callbacks cannot complete a newer attempt. Exhaustion records{' '} - runtime.effect.failed. + runtime.effect.failed. A handler can stop that budget early with{' '} + failEffect and nonRetryable: true, or ask the next attempt to wait + at least retryAfterMs without shortening the policy backoff. A worker heartbeat + can carry details; the next dispatch of that effect includes the latest details + so the attempt can resume. The callback response is{' '} + {'{ status, cancelRequested }'}, not the run state.{' '} + cancelRequested tells an attempt that is already running to stop.

Signals and webhook delivery diff --git a/bun.lock b/bun.lock index d5649cb..3601304 100644 --- a/bun.lock +++ b/bun.lock @@ -250,6 +250,7 @@ }, "devDependencies": { "@cloudflare/workers-types": "^5.20260914.1", + "@looms/client": "workspace:*", "@looms/schedule": "workspace:*", "@looms/workflow": "workspace:*", "@types/bun": "^1.3.9", diff --git a/docs/modules.md b/docs/modules.md index d8600a3..d8e929a 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -44,6 +44,8 @@ Each module owns its definitions. `workflow({ definitions: [checkout] })` instal At initialization, Looms rejects duplicate `(kind, name)` registrations and definitions whose kind is not implemented by their owning module. Starting or spawning an unregistered definition fails before any run events are written. An unsupported thread kind is rejected the same way; if such an event somehow appears in a log, fold marks the thread failed instead of inventing empty state. Pass all child definitions to their owning modules too. +`defineWorkflow({ childExecution: 'run' })` launches each spawned, mapped, or looped child as its own run instead of a thread in the parent. The child run id is derived from the parent run and the child thread id, so a retry finds the same run. The child appends `runtime.child.settled` to the parent when it finishes, and that event completes the parent's wait. `childClose: 'terminate'` (the default) cancels those runs when the parent is cancelled. `childClose: 'wait'` cancels the children first and cancels the parent only after they have finished. A missing `RunLauncher` fails the launch. The default `childExecution` is `'thread'`, which keeps the child inside the parent run. + `defineModule(options, setup)` returns a complete runtime module. The callback receives typed `effect`, `projection`, `thread`, `input`, and `emit` builders. Only returned members are installed; access them through `payments.effects.charge` and `payments.projections.ledger`. Event schemas can be inline or supplied as a reusable `defineEventCatalog`. Modules that implement a custom thread kind can return `definitions` alongside `threads`. The runtime gathers these for `start`, HTTP starts, and cross-module child spawning. diff --git a/docs/production.md b/docs/production.md index 866dee0..5ebbeb4 100644 --- a/docs/production.md +++ b/docs/production.md @@ -13,25 +13,171 @@ coalesced. A deployment, eviction, or alarm retry may repeat a wake; event preconditions make that safe. Cross-object queries and transactions are not provided. Build global run lists with a projector. +Live subscribers on a Durable Object should open a WebSocket (`Upgrade: websocket` on `GET /runs/:id/events`). The object hibernates the socket, answers `ping` with `pong` without waking, and fans out events when the log is appended. SSE stays available, including on `looms.serve`, but its keepalive keeps a Durable Object awake. Writes stay on HTTP. A browser cannot send `Authorization` on the upgrade, so pass a read grant (see `grantAuth` below); `@looms/client` puts `grant` on the socket URL. An idle-socket billed-duration check on a real Cloudflare account has not been run. + +A workflow can launch children with `childExecution: 'run'`. Each child is its own run, addressed by a deterministic id, and reports `runtime.child.settled` back to the parent. Cancelling the parent cancels those children. `childClose: 'wait'` records `runtime.child.close.requested` and cancels the parent only after the children reach a terminal status. `childClose: 'terminate'` cancels them as the parent closes. Children stay threads in the parent run unless `childExecution` is set. + All built-in stores reject an event over 1 MiB and a snapshot over 8 MiB, measured as serialized UTF-8, before storage is mutated. The stable error names are `EventPayloadTooLargeError` and `SnapshotPayloadTooLargeError`. +Pass `blobStore` to `createLooms`, or `blobs` to `LoomsDurableObject` / +`durableObjectEventStore`, to keep larger JSON. Strings over 64 KiB are replaced +with `{ "$looms": "blob", "sha256", "bytes", "mediaType": "application/json" }` +before those limits run. That object is reserved in stored JSON. If an event is +still over 1 MiB, its payload is stored as one blob. The body is SHA-256 +content, so identical JSON is stored once. `read`, live subscribers, folds, +projections, and snapshot loads resolve references, including references nested +inside another blob. Objects and arrays stay inline so snapshot state still +decodes; a snapshot that is over 8 MiB after large strings are removed is still +rejected. Trim deletes a body when no retained event and no latest snapshot in +any run still needs it. Without a blob store, oversized events and snapshots +are still rejected. + +`memoryBlobStore` is the in-memory implementation. `filesystemBlobStore(directory)` +is the Bun and Node implementation. `r2BlobStore(bucket)` writes an R2 bucket +under `looms/blobs/`. Every run's Durable Object can share one bucket. Each run +writes a marker per body it references, so trimming one run checks only that +body's markers, never other runs' logs. If a body is deleted while another run +is referencing it, that run's append fails and its retry uploads the body again. + +## Run attributes and secret context + +`attributes` on `start` are immutable labels (tenant, project, org). They are +written on `runtime.run.started`, folded into run state, and copied onto the +run summary. Authorization receives them on `route.attributes` when a run has +any. They must be safe for every reader of the run. + +`secrets` are bearer material for the run. The memory and SQLite stores keep +them in a side table, `looms_run_secrets`, which is not part of the event log +or a snapshot. Effect handlers read them on `ctx.secrets`. An in-process worker +receives them on the dispatch task. A transport that persists that task must +omit `secrets`. Its worker reads them with +`POST /runs/:runId/effects/:effectId/:attempt/secrets` (`client.workerSecrets`) +after its `started` callback is accepted. The route uses `worker` access, so it +returns 503 until the host configures worker auth. It answers +`{ status: 'accepted', secrets }` only while that attempt is started and +unsettled, and `{ status: 'stale' }` or `{ status: 'cancelled' }` otherwise. +Responses are sent with `cache-control: no-store`. A handler must not copy +secrets into an emitted event. + +A retry of the same start may omit `secrets` and keeps the stored context. Sending +a different context conflicts, and the error does not include the secret values. +Forks and effect-launched runs inherit the parent's attributes and secrets. +A store without `runSecrets` rejects a start that supplies secrets. + +## Trace context + +`POST /runs` reads the W3C `traceparent` and `tracestate` headers. In process, +pass `trace: { traceparent, tracestate }` to `start` or `startRun`, and +`client.startRun({ trace })` sends it as those headers. A valid version-00 +`traceparent` is recorded on `runtime.run.started` and folded into +`startIdentity.trace`. Anything else is dropped, and so is a `tracestate` over +512 characters. It is not part of the start identity, so a retry from another +trace returns the same run. + +The run's trace context is passed unchanged to every effect handler as +`ctx.trace`, to every worker dispatch as `task.trace`, and to every run an +effect launches, including workflow child runs. A launch can set its own +`trace`, for example to parent the child under a span the handler started. +Forks keep the source's context. Looms creates no spans itself. The object's +keys are the W3C header names, so with OpenTelemetry, +`propagation.extract(context.active(), ctx.trace ?? {})` gives the parent +context for a handler's span. + ## Worker effect protocol -1. The runtime dispatches `{ runId, effectId, attempt, type, input }`. +1. The runtime dispatches `{ runId, effectId, attempt, type, input, details?, trace?, deadlines? }`. + `details` is the progress carried by the previous attempt's last heartbeat. + `deadlines` holds the schedule-to-start time and the start-to-close and + heartbeat timeouts from the effect's retry policy. 2. The worker calls `started` before the schedule-to-start deadline. -3. Long work calls `heartbeat`; each accepted heartbeat extends the heartbeat - deadline without exceeding start-to-close. -4. The worker calls exactly one of `complete` or `fail`. +3. Long work calls `heartbeat`. Each accepted heartbeat extends the heartbeat + deadline without exceeding start-to-close. A heartbeat may include `details`; + the latest details are kept across retries and sent on the next dispatch. +4. The worker calls exactly one of `complete` or `fail`. `fail` may set + `nonRetryable` to stop the retry policy, or `retryAfterMs` to wait at least + that long before the next attempt (the policy backoff is never shortened). + A local handler does the same by failing with `failEffect`. 5. On cancellation, stop side effects and call `cancelled`. +Each callback response is `{ status: 'accepted' | 'stale' | 'cancelled', cancelRequested }`. +The host no longer returns the run state. `cancelRequested` is how a worker learns +that an attempt already in progress should stop: a queued dispatch cannot be recalled. +`stale` means the report was for another attempt, or the attempt had already settled, +and the host recorded nothing. + Callbacks include the same `effectId` and `attempt`. Stale callbacks are ignored. -A dispatch transport failure is recorded as `ambiguous`: an operator must first +A worker with `dispatchSemantics: 'acknowledged'` promises that a rejected +`dispatch` sent nothing, so the attempt fails under the retry policy. Because its +transport can still lose a message it accepted, creating a runtime fails unless +every effect that worker claims sets `retry.scheduleToStartTimeoutMs`. For other +workers, a dispatch transport failure is recorded as `ambiguous`: an operator must first verify the destination's idempotency record, then use `retryEffect` or `POST /runs/:runId/effects/:effectId/retry`. Never blindly retry a non-idempotent effect. +### Cloudflare Queues transport + +`cloudflareQueueWorker` sends each attempt to a Cloudflare Queue as JSON. Every +routed effect needs `retry.scheduleToStartTimeoutMs`. An input that would push the +message past 100 KB is stored in `blobs` and sent as `inputRef`; without a blob +store that dispatch fails. Secrets never go on the queue. + +```ts +import { cloudflareQueueWorker } from '@swirls/looms/cloudflare' + +const worker = cloudflareQueueWorker({ + routes: [{ handles: ['sandbox.exec'], queue: env.SANDBOX_QUEUE }], + blobs: r2BlobStore(env.BLOBS), +}) + +createLooms({ modules, worker, maxInFlightWorkerEffects: 16 }) +``` + +A consumer runs handlers of the form `(task, ctx) => events`. `ctx.heartbeat(details)` +reports progress, `ctx.secrets()` reads the run's secret context through the +authenticated callback, and `ctx.signal` aborts when the run is cancelled. Throw a +`QueueTaskError` with `nonRetryable` or `retryAfterMs` to control the retry. + +```ts +import { createLoomsClient } from '@swirls/looms/client' +import { + cloudflareQueuePullSource, + createQueuePullConsumer, + queueConsumer, +} from '@swirls/looms/cloudflare' + +const callbacks = createLoomsClient({ baseUrl, workerCallbackToken }) +const handlers = { 'sandbox.exec': runSandbox } + +// In a Worker: export default { queue: queueConsumer({ callbacks, handlers, blobs }) } + +const consumer = createQueuePullConsumer({ + source: cloudflareQueuePullSource({ accountId, queueId, token }), + callbacks, + handlers, + blobs, + concurrency: 8, +}) + +process.once('SIGTERM', () => consumer.stop()) +``` + +- The consumer acks a message once Looms accepts `started`. After that Looms owns + retries through heartbeat and start-to-close timeouts, so a long effect does not + depend on the queue's lease. A duplicate or stale delivery is acked without running. +- A message that cannot start yet is retried by the queue with backoff. One that + never starts lands in the dead-letter queue, and the schedule-to-start timeout + schedules a new attempt. +- A queue send that fails, such as a 429, fails the attempt under the retry policy. +- The pull consumer batches acks, backs off while the queue is empty or the API + errors, and reports `backlog()` from Cloudflare's `message_backlog_count`. + `stop()` stops pulling, waits for running attempts, and sends the last acks. +- `maxInFlightWorkerEffects` caps how many worker attempts one run has in flight, + so a wide fan-out cannot fill a shared queue. +- `createMemoryQueue()` has Cloudflare's lease, retry, and dead-letter behavior for tests. + ## Inspection and repair - `inspectRun(runId)` and `GET /runs/:runId/status` expose stream bounds, state, @@ -60,6 +206,38 @@ Each configured level requires `Authorization: Bearer `; levels without a token keep the default behavior. The client's `workerCallbackToken` option sends the worker token on `workerCallback` requests. +Browsers cannot set headers on `EventSource` or `WebSocket`. `grantAuth` verifies +a short-lived HMAC grant from the `grant` query parameter, or from a `v1.` value +in `Sec-WebSocket-Protocol` when the query parameter is absent. The grant names +one run, one principal, and `read` access. It does not authorize writes, worker +callbacks, operations, the debugger, or a different run. `exp` is an exclusive +Unix millisecond timestamp. A host that accepts the grant as a WebSocket +subprotocol must echo that protocol on the upgrade response (`grantSubprotocol` +returns it); the Durable Object socket already does. The query parameter needs no +echo. + +```ts +import { Effect } from 'effect' +import { grantAuth, issueRunGrant } from '@swirls/looms/runtime' + +const secret = env.GRANT_SECRET +const grant = await Effect.runPromise( + issueRunGrant(secret, { + runId, + principal: userId, + access: 'read', + exp: Date.now() + 5 * 60_000, + }), +) + +createLooms({ + modules, + authorize: grantAuth({ secret }), +}) + +const events = new EventSource(`/runs/${runId}/events?live=true&grant=${grant}`) +``` + ## Retention and compaction Snapshots accelerate runtime recovery but do not rebuild arbitrary projections. @@ -97,6 +275,21 @@ and schemas needed by active old-version runs. If a new projector is bad, stop delivery, deploy a corrected new version, and rebuild from archive; do not move a checkpoint backward after source events were trimmed. +## Resolving definitions + +Pass `definitionResolver` to `createLooms`, `createRuntime`, an actor host, or +`LoomsDurableObjectConfig` when a definition is not in the module's static list. +`resolve(kind, name, version)` returns the full definition object, or `undefined` +when that version does not exist. Registered module definitions are checked +first. Results, including misses, stay in an LRU cache. + +Start, spawn, and launch resolve before they write `runtime.thread.started`. +A miss fails the start with `UnknownDefinitionError` and writes no events. The +accepted version is pinned on `runtime.run.started` and `runtime.thread.started`. +Before a wake replays a log, those pinned identities are resolved again so fold +can read them synchronously. An unknown workflow version fails that thread. +Evaluator definitions include `version`, which defaults to `v1`. + ## Production configuration Configure durable storage, actor routing, worker and operations tokens, bounded diff --git a/docs/protocol.md b/docs/protocol.md index d33fd6e..582915e 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -26,19 +26,21 @@ Matching is by event `type` plus an optional payload subset (`match: { approvalI ## HTTP -| Method | Path | Purpose | -| ------ | ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `GET` | `/health` | Liveness | -| `POST` | `/runs` | Start `{ kind, definitionName, input }` (`Accept: text/event-stream` or `?stream=true` streams the log live) | -| `GET` | `/runs` | List run ids (`createLooms` / shared store). Actor hosts (`createLocalActorHost`, Durable Objects, celld) return `501` — use a projector lake or client-side history instead | -| `GET` | `/runs/:id` | Current run state | -| `GET` | `/runs/:id/events` | Event log (`?fromSeq=&limit=`), or SSE with `live=true` / `Accept: text/event-stream`; resumes after `Last-Event-ID` | -| `POST` | `/runs/:id/events` | Signal the run | -| `GET` | `/runs/:id/threads` | Child threads | -| `POST` | `/runs/:id/wake` | Resume processing | -| `GET` | `/runs/:id/replay?seq=` | State before / after an event | -| `GET` | `/runs/:id/projections/:name` | Named read model | +| Method | Path | Purpose | +| ------ | ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `GET` | `/health` | Liveness | +| `POST` | `/runs` | Start `{ kind, definitionName, input }` (`Accept: text/event-stream` or `?stream=true` streams the log live) | +| `GET` | `/runs` | List run ids (`createLooms` / shared store). Actor hosts (`createLocalActorHost`, Durable Objects, celld) return `501` — use a projector lake or client-side history instead | +| `GET` | `/runs/:id` | Current run state | +| `GET` | `/runs/:id/events` | Event log (`?fromSeq=&limit=`). SSE with `live=true` / `Accept: text/event-stream`, or a WebSocket with `Upgrade: websocket` on a Durable Object. Resumes after `Last-Event-ID` or `fromSeq`. `threadId` and `includeEphemeral=false` filter the stream. | +| `POST` | `/runs/:id/events` | Signal the run | +| `GET` | `/runs/:id/threads` | Child threads | +| `POST` | `/runs/:id/wake` | Resume processing | +| `GET` | `/runs/:id/replay?seq=` | State before / after an event | +| `GET` | `/runs/:id/projections/:name` | Named read model | `looms.fetch(request)` handles these paths. It returns `null` for everything else so you can mount Looms next to your own UI. +A live subscription speaks three frames. `{"type":"hello","head":1,"tail":4}` is sent first and reports the retained bounds. Event frames stay `{"batch":[...encoded events]}` and, on SSE, still use `id: `. `{"type":"resync","head":5,"reason":"trimmed"|"backpressure"}` means the cursor is below the retained log, or the socket fell behind; the server then closes the WebSocket with code `4000`. Resume from `head`. Clients that only read `batch` ignore the other two frames. + Use `@swirls/looms/client` (`createLoomsClient`) instead of hand-rolling fetch when you can. diff --git a/packages/actor/src/cell.ts b/packages/actor/src/cell.ts index b5540b3..6b586ea 100644 --- a/packages/actor/src/cell.ts +++ b/packages/actor/src/cell.ts @@ -3,6 +3,7 @@ import { Effect } from 'effect' import { EventStoreTag, type AnyRuntimeModule, + type DefinitionResolver, type EventInput, type EventStore, type EventStoreTrimCoverage, @@ -32,6 +33,8 @@ export interface ActorCellOptions< readonly store: EventStore readonly scheduler?: WakeScheduler readonly modules: TModules + /** Loads definitions that were not registered on a module. See `CreateRuntimeOptions`. */ + readonly definitionResolver?: DefinitionResolver readonly snapshotEvery?: number readonly maxWakeIterations?: number readonly maxPendingRunOperations?: number @@ -62,8 +65,14 @@ export interface ActorCell< startRun(args: StartRunArgs): Promise signal( events: ReadonlyArray, - options?: { idempotencyKey?: string }, + options?: { idempotencyKey?: string; coalesce?: boolean }, ): Promise + /** Append runtime-originated events. Not the client signal path. */ + accept( + events: ReadonlyArray, + options?: { idempotencyKey?: string; coalesce?: boolean }, + ): Promise + cancel(): Promise getRun(): Promise } @@ -78,6 +87,7 @@ export function createActorCell< const runtime = createRuntime({ modules: options.modules, + definitionResolver: options.definitionResolver, store, scheduler, launcher: options.launcher, @@ -138,6 +148,10 @@ export function createActorCell< signal: (events, signalOptions) => provide(runtime.signal(runId, events, signalOptions)), + accept: (events, signalOptions) => provide(runtime.accept(runId, events, signalOptions)), + + cancel: () => provide(runtime.cancel(runId).pipe(Effect.asVoid)), + getRun: () => provide(runtime.getRun(runId)), } } diff --git a/packages/actor/src/host.ts b/packages/actor/src/host.ts index 7976f53..99c0b15 100644 --- a/packages/actor/src/host.ts +++ b/packages/actor/src/host.ts @@ -2,6 +2,7 @@ import { Data, Effect } from 'effect' import type { AnyRuntimeModule, + DefinitionResolver, EventInput, EventStore, EventStoreTrimCoverage, @@ -54,6 +55,8 @@ export interface LocalActorHostOptions< > { readonly createStore: (runId: string) => EventStore | Promise readonly modules: TModules + /** Loads definitions that were not registered on a module. See `CreateRuntimeOptions`. */ + readonly definitionResolver?: DefinitionResolver readonly snapshotStore?: SnapshotStore readonly snapshotEvery?: number readonly maxWakeIterations?: number @@ -101,6 +104,9 @@ export function createLocalActorHost< const launcher = createRemoteRunLauncher({ launch: (runId, args) => getCell(runId).then((cell) => cell.launch(args)), status: (runId) => getCell(runId).then((cell) => cell.status()), + accept: (runId, events, signalOptions) => + getCell(runId).then((cell) => cell.accept(events, signalOptions).then(() => undefined)), + cancel: (runId) => getCell(runId).then((cell) => cell.cancel()), }) getCell = (runId: string): Promise> => { @@ -134,6 +140,7 @@ export function createLocalActorHost< store, scheduler, modules: options.modules, + definitionResolver: options.definitionResolver, snapshotStore: options.snapshotStore, snapshotEvery: options.snapshotEvery, maxWakeIterations: options.maxWakeIterations, diff --git a/packages/agent/src/definitions-store.ts b/packages/agent/src/definitions-store.ts index dd1c8f7..aeaca61 100644 --- a/packages/agent/src/definitions-store.ts +++ b/packages/agent/src/definitions-store.ts @@ -17,5 +17,5 @@ export function makeAgentDefinitionStore( return makeDefinitionStore('agent', definitions) } -export const AgentDefinitionsLive = (definitions: ReadonlyArray) => - Layer.succeed(AgentDefinitionsTag, makeAgentDefinitionStore(definitions)) +export const AgentDefinitionsLive = (definitions: AgentDefinitionStore) => + Layer.succeed(AgentDefinitionsTag, definitions) diff --git a/packages/agent/src/effects.ts b/packages/agent/src/effects.ts index 10f7d88..ebd9cf7 100644 --- a/packages/agent/src/effects.ts +++ b/packages/agent/src/effects.ts @@ -4,6 +4,7 @@ import { createThreadId, DEFAULT_DEFINITION_VERSION, validateInputEffect, + type DefinitionResolveError, type EventInputOf, type JsonValue, } from '@looms/core' @@ -86,7 +87,7 @@ function runCallLlm(args: { emit: (event: EventInputOf) => Promise }): Effect.Effect< ReadonlyArray>, - AgentExecutionError | LlmError, + AgentExecutionError | LlmError | DefinitionResolveError, LlmTag | AgentDefinitionsTag > { return Effect.gen(function* () { @@ -104,7 +105,10 @@ function runCallLlm(args: { const agents = yield* AgentDefinitionsTag const llm = yield* LlmTag - const definition = definitionName ? agents.get(definitionName, definitionVersion) : undefined + + const definition = definitionName + ? yield* agents.resolve(definitionName, definitionVersion) + : undefined if (!definition) { return [ @@ -240,7 +244,7 @@ export const executeToolEffect = agentModule.effect({ const agents = yield* AgentDefinitionsTag const definition = input.definitionName - ? agents.get(input.definitionName, input.definitionVersion) + ? yield* agents.resolve(input.definitionName, input.definitionVersion) : undefined if (!definition) { diff --git a/packages/agent/src/module.ts b/packages/agent/src/module.ts index fe7d98d..95854d1 100644 --- a/packages/agent/src/module.ts +++ b/packages/agent/src/module.ts @@ -21,8 +21,9 @@ export interface AgentModuleOptions { export function agent(options: AgentModuleOptions = {}) { const definitions = options.definitions ?? [] - const agentDefinitions = definitions.filter( - (definition): definition is AgentDefinition => definition.kind === 'agent', + const agentDefinitions = makeDefinitionStore( + 'agent', + definitions.filter((definition): definition is AgentDefinition => definition.kind === 'agent'), ) const sessionDefinitions = makeDefinitionStore( @@ -53,6 +54,10 @@ export function agent(options: AgentModuleOptions = {}) { }, services: () => Layer.merge(llmLayer, AgentDefinitionsLive(agentDefinitions)), interruptsOnSignal, + bindHost(host) { + agentDefinitions.useResolver(host.definitionResolver) + sessionDefinitions.useResolver(host.definitionResolver) + }, })) } diff --git a/packages/client/src/client.test.ts b/packages/client/src/client.test.ts index f454244..a107224 100644 --- a/packages/client/src/client.test.ts +++ b/packages/client/src/client.test.ts @@ -66,6 +66,60 @@ describe('createLoomsClient', () => { ]) }) + test('startRun sends the trace context as W3C headers, not in the body', async () => { + const calls: { traceparent: string | null; tracestate: string | null; body: unknown }[] = [] + + const client = createLoomsClient({ + fetch: async (_input, init) => { + const headers = new Headers(init?.headers) + + calls.push({ + traceparent: headers.get('traceparent'), + tracestate: headers.get('tracestate'), + body: Predicate.isString(init?.body) ? JSON.parse(init.body) : null, + }) + + return Response.json({ runId: 'run_1', threadId: 'thr_1', state: emptyRunState('run_1') }) + }, + }) + + const traceparent = '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01' + + await client.startRun({ + kind: 'agent', + definitionName: 'echo', + trace: { traceparent, tracestate: 'k=v' }, + }) + + expect(calls).toEqual([ + { traceparent, tracestate: 'k=v', body: { kind: 'agent', definitionName: 'echo' } }, + ]) + }) + + test('workerSecrets posts to the attempt with the worker token', async () => { + const calls: { href: string; authorization?: string | null }[] = [] + + const client = createLoomsClient({ + workerCallbackToken: 'worker-token', + fetch: async (input, init) => { + calls.push({ + href: hrefOf(input), + authorization: new Headers(init?.headers).get('authorization'), + }) + + return Response.json({ status: 'accepted', secrets: { token: 'abc' } }) + }, + }) + + const result = await client.workerSecrets('run 1', { effectId: 'eff/1', attempt: 2 }) + + expect(result).toEqual({ status: 'accepted', secrets: { token: 'abc' } }) + + expect(calls).toEqual([ + { href: '/runs/run%201/effects/eff%2F1/2/secrets', authorization: 'Bearer worker-token' }, + ]) + }) + test('listDefinitions reads the definitions route', async () => { const calls: string[] = [] @@ -166,6 +220,175 @@ describe('createLoomsClient', () => { stop() }) + test('subscribeEvents puts the read grant on the event stream', async () => { + const calls: string[] = [] + + const client = createLoomsClient({ + grant: 'v1.abc.def', + fetch: async (input) => { + calls.push(hrefOf(input)) + + return new Response(null, { status: 401 }) + }, + }) + + const stop = client.subscribeEvents('run_1', () => undefined) + const start = Date.now() + + while (calls.length === 0 && Date.now() - start < 2_000) { + await Bun.sleep(10) + } + + stop() + expect(calls[0]).toContain('/runs/run_1/events') + expect(calls[0]).toContain('grant=v1.abc.def') + }) + + test('subscribeEvents resumes from a resync head', async () => { + const calls: string[] = [] + let attempt = 0 + + const started = { + id: 'evt_4', + runId: 'run_1', + seq: 4, + ts: 1, + type: 'runtime.run.started', + payload: {}, + threadId: 'thr_1', + parentThreadId: null, + causationId: null, + correlationId: null, + effectId: null, + ephemeral: false, + origin: { type: 'system' }, + } + + const client = createLoomsClient({ + fetch: async (input) => { + calls.push(hrefOf(input)) + attempt += 1 + const encoder = new TextEncoder() + + const frame = + attempt === 1 + ? `data: ${JSON.stringify({ type: 'resync', head: 4, reason: 'trimmed' })}\n\n` + : `id: 4\ndata: ${JSON.stringify({ batch: [started] })}\n\n` + + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(frame)) + controller.close() + }, + }), + { headers: { 'content-type': 'text/event-stream' } }, + ) + }, + }) + + const received: number[] = [] + + const stop = client.subscribeEvents( + 'run_1', + (event) => { + received.push(event.seq) + }, + { reconnectDelayMs: 20 }, + ) + + const start = Date.now() + + while (received.length === 0 && Date.now() - start < 2_000) { + await Bun.sleep(10) + } + + expect(received).toEqual([4]) + expect(calls[0]).toContain('fromSeq=1') + expect(calls.some((href) => href.includes('fromSeq=4'))).toBe(true) + stop() + }) + + test('subscribeEvents can follow a websocket with the read grant', async () => { + const received: number[] = [] + + class FakeSocket { + static latest: FakeSocket | undefined + readonly url: string + private readonly listeners = new Map< + string, + Array<(event: { readonly data?: unknown }) => void> + >() + + constructor(url: string) { + this.url = url + FakeSocket.latest = this + } + + addEventListener(type: string, listener: (event: { readonly data?: unknown }) => void): void { + const current = this.listeners.get(type) ?? [] + current.push(listener) + this.listeners.set(type, current) + } + + emit(type: string, data?: string): void { + for (const listener of this.listeners.get(type) ?? []) { + listener({ data }) + } + } + + close(): void { + this.emit('close') + } + } + + const client = createLoomsClient({ baseUrl: 'http://looms.test', grant: 'v1.abc.def' }) + + const stop = client.subscribeEvents( + 'run_1', + (event) => { + received.push(event.seq) + }, + { transport: 'websocket', webSocket: FakeSocket, reconnectDelayMs: 60_000 }, + ) + + const socket = FakeSocket.latest + expect(socket?.url).toBe('ws://looms.test/runs/run_1/events?fromSeq=1&grant=v1.abc.def') + socket?.emit('message', JSON.stringify({ type: 'hello', head: 1, tail: 1 })) + + socket?.emit( + 'message', + JSON.stringify({ + batch: [ + { + id: 'evt_1', + runId: 'run_1', + seq: 1, + ts: 1, + type: 'runtime.run.started', + payload: {}, + threadId: 'thr_1', + parentThreadId: null, + causationId: null, + correlationId: null, + effectId: null, + ephemeral: false, + origin: { type: 'system' }, + }, + ], + }), + ) + + const start = Date.now() + + while (received.length === 0 && Date.now() - start < 2_000) { + await Bun.sleep(10) + } + + expect(received).toEqual([1]) + stop() + }) + test('stream yields live deltas then the completed run events', async () => { const client = createLoomsClient({ fetch: async (input, init) => { diff --git a/packages/client/src/client.ts b/packages/client/src/client.ts index 413b98e..47ea393 100644 --- a/packages/client/src/client.ts +++ b/packages/client/src/client.ts @@ -1,20 +1,30 @@ -import { Data, Effect, Option, Schema, Stream } from 'effect' +import { Data, Effect, Option, Predicate, Schema, Stream } from 'effect' import { createRunId, DefinitionCatalogSchema, EventEnvelopeSchema, + readSubscriptionData, ReplayStepSchema, RunStateSchema, RunSummariesSchema, + SecretContextSchema, stringifyJson, + traceHeaders, type DefinitionInput, type DefinitionRef, type EventEnvelope, type EventInput, type JsonValue, + type RunAttributes, type RunState, + type SecretContext, + type SubscriptionTransport, + type TraceContext, + type WorkerCallback, type WorkerCallbackInput, + type WorkerCallbackResult, + type WorkerSecretsResult, } from '@looms/core' import { consumeSseStreamEffect, eventsFromSseData, sseFrames } from './sse' @@ -41,6 +51,62 @@ const StartResultSchema = Schema.Struct({ const RunResultSchema = Schema.Struct({ runId: Schema.String, state: RunStateSchema }) +interface WorkerFailRequestBody { + error: string + nonRetryable?: boolean + retryAfterMs?: number +} + +function attemptPath(runId: string, attempt: WorkerCallback): string { + return `/runs/${encodeURIComponent(runId)}/effects/${encodeURIComponent(attempt.effectId)}/${attempt.attempt}` +} + +function workerCallbackBody(input: WorkerCallbackInput) { + switch (input.kind) { + case 'complete': + return { events: input.events } + + case 'fail': { + const body: WorkerFailRequestBody = { error: input.error } + + if (input.nonRetryable !== undefined) { + body.nonRetryable = input.nonRetryable + } + + if (input.retryAfterMs !== undefined) { + body.retryAfterMs = input.retryAfterMs + } + + return body + } + + case 'heartbeat': + return input.details === undefined ? {} : { details: input.details } + case 'started': + case 'cancelled': + return {} + + default: { + const exhaustive: never = input + return exhaustive + } + } +} + +const WorkerCallbackResultSchema = Schema.Struct({ + status: Schema.Union([ + Schema.Literal('accepted'), + Schema.Literal('stale'), + Schema.Literal('cancelled'), + ]), + cancelRequested: Schema.Boolean, +}) + +const WorkerSecretsResultSchema = Schema.Union([ + Schema.Struct({ status: Schema.Literal('accepted'), secrets: SecretContextSchema }), + Schema.Struct({ status: Schema.Union([Schema.Literal('stale'), Schema.Literal('cancelled')]) }), +]) + /** `appended` counts the events the run gained while the wake ran. */ const WakeResultSchema = Schema.Struct({ runId: Schema.String, @@ -68,6 +134,11 @@ export interface LoomsClientOptions { baseUrl?: string fetch?: typeof fetch workerCallbackToken?: string + /** + * Appended as `grant` to the event-stream URL, for both SSE and WebSocket subscriptions. A read + * grant cannot authorize other requests, so it is not sent on them. + */ + grant?: string } export interface StartResult { @@ -118,14 +189,29 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { const subscribeEvents = ( runId: string, onEvent: (event: EventEnvelope) => void, - reconnectDelayMs = 1_000, + third: number | SubscribeEventsOptions = 1_000, ) => { + const settings = subscriptionSettings(third, options.grant) let fromSeq = 1 const controller = new AbortController() const run = Effect.gen(function* () { while (!controller.signal.aborted) { yield* Effect.gen(function* () { + if (useWebSocket(settings)) { + return yield* consumeWebSocket({ + baseUrl, + runId, + settings, + signal: controller.signal, + currentSeq: () => fromSeq, + setSeq: (next) => { + fromSeq = next + }, + onEvent, + }) + } + const cursor = Math.max(0, fromSeq - 1) const headers = new Headers({ accept: 'text/event-stream' }) @@ -135,7 +221,7 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { const res = yield* Effect.tryPromise({ try: (signal) => - fetchImpl(`${baseUrl}/runs/${encodeURIComponent(runId)}/events?fromSeq=${fromSeq}`, { + fetchImpl(`${baseUrl}${eventsQuery(runId, fromSeq, settings)}`, { headers, signal, }), @@ -146,10 +232,22 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { if (res.ok && body) { return yield* consumeSseStreamEffect(body, (frame) => { - for (const event of eventsFromSseData(frame.data, runId)) { - onEvent(event) - fromSeq = Math.max(fromSeq, event.seq + 1) - } + let resync = false + + deliverSubscriptionData({ + data: frame.data, + runId, + currentSeq: () => fromSeq, + setSeq: (next) => { + fromSeq = next + }, + onEvent, + onResync: () => { + resync = true + }, + }) + + return resync ? false : undefined }) } @@ -162,7 +260,7 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { break } - yield* Effect.sleep(reconnectDelayMs) + yield* Effect.sleep(settings.reconnectDelayMs) } }) @@ -179,37 +277,55 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { definitionVersion?: string input?: JsonValue runId?: string - }) => request('/runs', StartResultSchema, { method: 'POST', body: stringifyJson(args) }) - - const workerCallback = (runId: string, input: WorkerCallbackInput) => { - const body = - input.kind === 'complete' - ? { events: input.events } - : input.kind === 'fail' - ? { error: input.error } - : {} - - return request( - `/runs/${encodeURIComponent(runId)}/effects/${encodeURIComponent(input.effectId)}/${input.attempt}/${input.kind}`, - RunResultSchema, - { - method: 'POST', - headers: options.workerCallbackToken - ? { authorization: `Bearer ${options.workerCallbackToken}` } - : undefined, - body: stringifyJson(body), - }, - ) + attributes?: RunAttributes + secrets?: SecretContext + /** Sent as the W3C `traceparent` and `tracestate` headers. */ + trace?: TraceContext + }) => { + const { trace, ...body } = args + + return request('/runs', StartResultSchema, { + method: 'POST', + headers: traceHeaders(trace), + body: stringifyJson(body), + }) } + const workerHeaders = options.workerCallbackToken + ? { authorization: `Bearer ${options.workerCallbackToken}` } + : undefined + + const workerCallback = ( + runId: string, + input: WorkerCallbackInput, + ): Promise => { + const body = workerCallbackBody(input) + + return request(`${attemptPath(runId, input)}/${input.kind}`, WorkerCallbackResultSchema, { + method: 'POST', + headers: workerHeaders, + body: stringifyJson(body), + }) + } + + /** Read the run's secret context. Only the attempt that is currently running gets it. */ + const workerSecrets = (runId: string, attempt: WorkerCallback): Promise => + request(`${attemptPath(runId, attempt)}/secrets`, WorkerSecretsResultSchema, { + method: 'POST', + headers: workerHeaders, + body: '{}', + }) + const streamRun = (args: { kind: string definitionName: string definitionVersion?: string input?: JsonValue runId?: string + trace?: TraceContext }): StreamHandle => { - const runId = args.runId ?? createRunId() + const { trace, ...body } = args + const runId = body.runId ?? createRunId() const stream = Stream.unwrap( Effect.tryPromise({ @@ -217,10 +333,11 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { fetchImpl(`${baseUrl}/runs`, { method: 'POST', headers: { + ...traceHeaders(trace), 'content-type': 'application/json', accept: 'text/event-stream', }, - body: stringifyJson({ ...args, runId }), + body: stringifyJson({ ...body, runId }), signal, }), catch: (cause) => new LoomsClientError('Failed to start event stream', cause), @@ -300,6 +417,7 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { body: stringifyJson(forkOptions?.runId === undefined ? {} : { runId: forkOptions.runId }), }), workerCallback, + workerSecrets, listRunSummaries: () => request('/runs/summaries', RunSummariesSchema), listDefinitions: () => request('/definitions', DefinitionCatalogSchema), replayTo: (runId: string, seq: number) => @@ -329,3 +447,201 @@ export function createLoomsClient(options: LoomsClientOptions = {}) { } export type LoomsClient = ReturnType + +export interface SubscribeEventsOptions { + readonly transport?: SubscriptionTransport + readonly reconnectDelayMs?: number + readonly threadId?: string + readonly includeEphemeral?: boolean + readonly webSocket?: SubscriptionSocketConstructor +} + +export interface SubscriptionSocket { + addEventListener(type: string, listener: (event: { readonly data?: unknown }) => void): void + close(code?: number, reason?: string): void +} + +export interface SubscriptionSocketConstructor { + new (url: string): SubscriptionSocket +} + +interface SubscriptionDelivery { + readonly data: string + readonly runId: string + readonly currentSeq: () => number + readonly setSeq: (seq: number) => void + readonly onEvent: (event: EventEnvelope) => void + readonly onResync: () => void +} + +interface WebSocketSubscription { + readonly baseUrl: string + readonly runId: string + readonly settings: ResolvedSubscription + readonly signal: AbortSignal + readonly currentSeq: () => number + readonly setSeq: (seq: number) => void + readonly onEvent: (event: EventEnvelope) => void +} + +interface ResolvedSubscription extends SubscribeEventsOptions { + readonly transport: SubscriptionTransport + readonly reconnectDelayMs: number + readonly grant?: string +} + +function subscriptionSettings( + third: number | SubscribeEventsOptions, + grant: string | undefined, +): ResolvedSubscription { + if (Predicate.isNumber(third)) { + return { transport: 'sse', reconnectDelayMs: third, grant } + } + + return { + transport: third.transport ?? 'sse', + reconnectDelayMs: third.reconnectDelayMs ?? 1_000, + threadId: third.threadId, + includeEphemeral: third.includeEphemeral, + webSocket: third.webSocket, + grant, + } +} + +function useWebSocket(settings: ResolvedSubscription): boolean { + if (settings.transport !== 'websocket' && settings.transport !== 'auto') { + return false + } + + return settings.webSocket !== undefined || globalWebSocket() !== undefined +} + +function eventsQuery(runId: string, fromSeq: number, settings: ResolvedSubscription): string { + const params = new URLSearchParams({ fromSeq: String(fromSeq) }) + + if (settings.threadId) { + params.set('threadId', settings.threadId) + } + + if (settings.includeEphemeral === false) { + params.set('includeEphemeral', 'false') + } + + if (settings.grant) { + params.set('grant', settings.grant) + } + + return `/runs/${encodeURIComponent(runId)}/events?${params.toString()}` +} + +function webSocketUrl(baseUrl: string, path: string): string { + const url = new URL(path, baseUrl) + url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:' + return url.toString() +} + +function globalWebSocket(): SubscriptionSocketConstructor | undefined { + if (!('WebSocket' in globalThis) || !Predicate.isFunction(globalThis.WebSocket)) { + return undefined + } + + return globalThis.WebSocket +} + +function consumeWebSocket( + subscription: WebSocketSubscription, +): Effect.Effect { + const Socket = subscription.settings.webSocket ?? globalWebSocket() + + if (!Socket) { + return Effect.void + } + + const url = webSocketUrl( + subscription.baseUrl, + eventsQuery(subscription.runId, subscription.currentSeq(), subscription.settings), + ) + + return Effect.callback((resume) => { + const socket = new Socket(url) + + const finish = () => { + resume(Effect.void) + } + + if (subscription.signal.aborted) { + socket.close() + finish() + return Effect.void + } + + const onAbort = () => { + socket.close() + finish() + } + + subscription.signal.addEventListener('abort', onAbort) + + socket.addEventListener('message', (event) => { + if (!Predicate.isString(event.data)) { + return + } + + deliverSubscriptionData({ + data: event.data, + runId: subscription.runId, + currentSeq: subscription.currentSeq, + setSeq: subscription.setSeq, + onEvent: subscription.onEvent, + onResync: () => { + socket.close() + }, + }) + }) + + socket.addEventListener('close', finish) + socket.addEventListener('error', finish) + + return Effect.sync(() => { + subscription.signal.removeEventListener('abort', onAbort) + socket.close() + }) + }) +} + +function deliverSubscriptionData(delivery: SubscriptionDelivery): void { + const read = readSubscriptionData(delivery.data, delivery.runId) + + switch (read.kind) { + case 'events': { + const fromSeq = delivery.currentSeq() + let next = fromSeq + + for (const event of read.events) { + if (event.seq < fromSeq) { + continue + } + + delivery.onEvent(event) + next = Math.max(next, event.seq + 1) + } + + delivery.setSeq(next) + return + } + + case 'resync': + delivery.setSeq(read.head) + delivery.onResync() + return + + case 'hello': + case 'ignore': + return + + default: { + const unexpected: never = read + return unexpected + } + } +} diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index fe3be90..a1ef0b2 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -4,6 +4,9 @@ export { type LoomsClientOptions, type StartResult, type StreamHandle, + type SubscribeEventsOptions, + type SubscriptionSocket, + type SubscriptionSocketConstructor, } from './client' export { consumeSseStream, diff --git a/packages/client/src/sse.ts b/packages/client/src/sse.ts index 6b0e656..99f3758 100644 --- a/packages/client/src/sse.ts +++ b/packages/client/src/sse.ts @@ -58,7 +58,7 @@ export class SseStreamError extends Data.TaggedError('SseStreamError')<{ export function consumeSseStreamEffect( body: ReadableStream, - onFrame: (frame: SseFrame) => void, + onFrame: (frame: SseFrame) => void | boolean, ): Effect.Effect { const decoder = new TextDecoder() @@ -85,8 +85,8 @@ export function consumeSseStreamEffect( for (const chunk of chunks) { const frame = parseSseFrame(chunk) - if (frame) { - onFrame(frame) + if (frame && onFrame(frame) === false) { + return } } } diff --git a/packages/cloudflare/package.json b/packages/cloudflare/package.json index f2093ba..3e890c2 100644 --- a/packages/cloudflare/package.json +++ b/packages/cloudflare/package.json @@ -37,6 +37,7 @@ }, "devDependencies": { "@cloudflare/workers-types": "^5.20260914.1", + "@looms/client": "workspace:*", "@looms/schedule": "workspace:*", "@looms/workflow": "workspace:*", "@types/bun": "^1.3.9", diff --git a/packages/cloudflare/src/cloudflare.test.ts b/packages/cloudflare/src/cloudflare.test.ts index d9f5cf8..29e0418 100644 --- a/packages/cloudflare/src/cloudflare.test.ts +++ b/packages/cloudflare/src/cloudflare.test.ts @@ -4,7 +4,12 @@ import { describe, expect, mock, setSystemTime, test } from 'bun:test' import { Effect, Schema } from 'effect' import { createLocalActorHost } from '@looms/actor' -import { DefinitionCatalogSchema, emptyRunState, makeMemoryEventStore } from '@looms/core' +import { + DefinitionCatalogSchema, + emptyRunState, + makeMemoryEventStore, + type SubscriptionAttachment, +} from '@looms/core' import { defineSchedule, ensureSchedule, every, once, schedule } from '@looms/schedule' import { defineWorkflow, workflow } from '@looms/workflow' @@ -319,6 +324,104 @@ describe('@looms/cloudflare', () => { const fetchRes = await doInstance.fetch(new Request('http://localhost/health')) expect(fetchRes.status).toBe(200) }) + + test('LoomsDurableObject replays a run over a hibernating socket', async () => { + class Socket { + readonly sent: string[] = [] + bufferedAmount = 0 + private attachment: SubscriptionAttachment | undefined + + send(data: string): void { + this.sent.push(data) + } + + close(): void { + // The platform drops the socket. + } + + serializeAttachment(value: SubscriptionAttachment): void { + this.attachment = value + } + + deserializeAttachment(): SubscriptionAttachment | undefined { + return this.attachment + } + } + + let server: Socket | undefined + const hadPair = 'WebSocketPair' in globalThis + const previous = globalThis.WebSocketPair + + Object.defineProperty(globalThis, 'WebSocketPair', { + configurable: true, + writable: true, + value: class { + 0: Socket + 1: Socket + + constructor() { + this[0] = new Socket() + this[1] = new Socket() + server = this[1] + } + }, + }) + + try { + const accepted: Socket[] = [] + + const ctx = { + ...objectContext('run_ws', sqliteStorage()), + acceptWebSocket(ws: Socket) { + accepted.push(ws) + }, + getWebSockets() { + return accepted + }, + } + + const flow = defineWorkflow({ + name: 'ws-flow', + nodes: [{ id: 'work', run: () => ({ ok: true }) }], + }) + + class SocketObject extends LoomsDurableObject { + override configure(): LoomsDurableObjectConfig { + return { modules: [workflow({ definitions: [flow] })] } + } + } + + // SAFETY: the test stands in for a Durable Object state. + // @ts-expect-error mock socket context in unit test + const object = new SocketObject(ctx, {}) + + await object.startRun({ + runId: 'run_ws', + kind: 'workflow', + definitionName: 'ws-flow', + input: null, + }) + + const response = await object.fetch( + new Request('http://localhost/runs/run_ws/events', { headers: { upgrade: 'websocket' } }), + ) + + expect(response.status).toBe(101) + expect(server?.sent[0]).toContain('"type":"hello"') + expect(server?.sent.some((frame) => frame.includes('"batch"'))).toBe(true) + expect(accepted).toHaveLength(1) + } finally { + if (hadPair) { + Object.defineProperty(globalThis, 'WebSocketPair', { + configurable: true, + writable: true, + value: previous, + }) + } else { + Reflect.deleteProperty(globalThis, 'WebSocketPair') + } + } + }) }) const decodeCatalog = Schema.decodeUnknownSync(DefinitionCatalogSchema) diff --git a/packages/cloudflare/src/durable-object.ts b/packages/cloudflare/src/durable-object.ts index d5746df..b865a35 100644 --- a/packages/cloudflare/src/durable-object.ts +++ b/packages/cloudflare/src/durable-object.ts @@ -5,7 +5,9 @@ import { createActorCell, type ActorCell } from '@looms/actor' import { createRemoteRunLauncher, type AnyRuntimeModule, + type DefinitionResolver, type EventInput, + type BlobStore, type EventStoreTrimCoverage, type LaunchRunArgs, type RunLauncher, @@ -13,15 +15,26 @@ import { type RunStatusSnapshot, } from '@looms/core' import type { Projector } from '@looms/projectors' -import type { Authorize, StartResult, StartRunArgs } from '@looms/runtime' +import { + defaultAuthorize, + type Authorize, + type LoomsRuntime, + type StartResult, + type StartRunArgs, +} from '@looms/runtime' import { durableObjectAlarms } from './alarm-scheduler' import { durableObjectEventStore, type DurableObjectEventStore } from './event-store' import type { DurableObjectNamespaceLike } from './router' +import { attachSocketFanout, isRunSocketUpgrade, openRunSocket } from './websocket' export interface LoomsDurableObjectConfig { readonly modules: readonly AnyRuntimeModule[] + /** Loads definitions that were not registered on a module. See `CreateRuntimeOptions`. */ + readonly definitionResolver?: DefinitionResolver readonly projectors?: readonly Projector[] + /** Content-addressed store for event and snapshot JSON over the inline byte limits. */ + readonly blobs?: BlobStore readonly snapshotEvery?: number readonly maxWakeIterations?: number readonly maxPendingRunOperations?: number @@ -46,6 +59,7 @@ export interface LoomsDurableObjectConfig { export abstract class LoomsDurableObject extends DurableObject { private cellPromise?: Promise private store?: DurableObjectEventStore + private authorize: Authorize = defaultAuthorize abstract configure(env: Env): LoomsDurableObjectConfig | Promise @@ -87,15 +101,18 @@ export abstract class LoomsDurableObject extends DurableObject extends DurableObject { + if (isRunSocketUpgrade(req)) { + return this.getCell().then((cell) => this.acceptRunSocket(req, cell.runtime)) + } + return this.getCell().then((cell) => cell.fetch(req)) } + /** Client messages are ignored. Signals and other writes stay on HTTP. */ + override webSocketMessage(): void { + // A hibernated socket stays quiet. The platform delivers this only if a client writes. + } + + override webSocketClose(): void { + // The platform drops the socket. Nothing is retained for it. + } + + override webSocketError(): void { + // The platform drops the socket. Nothing is retained for it. + } + + private acceptRunSocket(req: Request, runtime: LoomsRuntime): Promise { + const store = this.store + const runId = this.ctx.id.name + + if (!store || !runId) { + return Promise.resolve(Response.json({ error: 'Run is not ready' }, { status: 500 })) + } + + return Effect.runPromise( + openRunSocket({ + ctx: this.ctx, + runtime, + store, + request: req, + runId, + authorize: this.authorize, + }), + ) + } + /** Commit this object's run and set its alarm. Returns before the run wakes. */ launch(args: LaunchRunArgs): Promise { return this.getCell().then((cell) => cell.launch(args)) @@ -133,11 +187,22 @@ export abstract class LoomsDurableObject extends DurableObject, - options?: { idempotencyKey?: string }, + options?: { idempotencyKey?: string; coalesce?: boolean }, ): Promise { return this.getCell().then((cell) => cell.signal(events, options)) } + accept( + events: ReadonlyArray, + options?: { idempotencyKey?: string; coalesce?: boolean }, + ): Promise { + return this.getCell().then((cell) => cell.accept(events, options)) + } + + cancel(): Promise { + return this.getCell().then((cell) => cell.cancel()) + } + getRun(): Promise { return this.getCell().then((cell) => cell.getRun()) } @@ -165,8 +230,13 @@ export interface DurableObjectRunStub { startRun(args: StartRunArgs): Promise signal( events: ReadonlyArray, - options?: { idempotencyKey?: string }, + options?: { idempotencyKey?: string; coalesce?: boolean }, + ): Promise + accept( + events: ReadonlyArray, + options?: { idempotencyKey?: string; coalesce?: boolean }, ): Promise + cancel(): Promise getRun(): Promise } @@ -181,6 +251,12 @@ export function durableObjectLauncher(namespace: { getByName(name: string): obje return createRemoteRunLauncher({ launch: (runId, args) => runs.getByName(runId).launch(args), status: (runId) => runs.getByName(runId).status(), + accept: (runId, events, options) => + runs + .getByName(runId) + .accept(events, options) + .then(() => undefined), + cancel: (runId) => runs.getByName(runId).cancel(), }) } diff --git a/packages/cloudflare/src/event-store.ts b/packages/cloudflare/src/event-store.ts index d706d9d..5263b7b 100644 --- a/packages/cloudflare/src/event-store.ts +++ b/packages/cloudflare/src/event-store.ts @@ -1,6 +1,6 @@ import type { Effect } from 'effect' -import type { EventStore } from '@looms/core' +import { snapshotStoreOf, withBlobOffload, type BlobStore, type EventStore } from '@looms/core' import { sqliteEventStore, sqlStorageExec } from '@looms/core/sqlite-store' import { createProjectorDelivery, @@ -13,6 +13,8 @@ import { export interface DurableObjectStoreOptions { readonly projectors?: readonly Projector[] readonly scheduleProjectorRetry?: (at: number) => Effect.Effect + /** Offloads oversized event and snapshot JSON. See `withBlobOffload`. */ + readonly blobs?: BlobStore } export interface DurableObjectEventStore extends EventStore { @@ -32,7 +34,12 @@ export function durableObjectEventStore( transaction: (fn) => ctx.storage.transactionSync(fn), }) - const store = sqliteEventStore({ exec }) + const sqlite = sqliteEventStore({ exec }) + const snapshots = snapshotStoreOf(sqlite) + + const store = options.blobs + ? withBlobOffload(sqlite, options.blobs, snapshots ? { snapshots } : undefined) + : sqlite if (options.projectors && options.projectors.length > 0) { const delivery = createProjectorDelivery( diff --git a/packages/cloudflare/src/index.ts b/packages/cloudflare/src/index.ts index 47e4c3c..eea4d2b 100644 --- a/packages/cloudflare/src/index.ts +++ b/packages/cloudflare/src/index.ts @@ -11,8 +11,52 @@ export { type DurableObjectRunStub, type LoomsDurableObjectConfig, } from './durable-object' +export { + r2BlobStore, + type R2BucketLike, + type R2ListResult, + type R2ObjectBodyLike, + type R2StoredObject, +} from './r2-blob-store' export { routeToDurableObject, type DurableObjectNamespaceLike, type DurableObjectStubLike, } from './router' +export { createMemoryQueue, type MemoryQueue, type MemoryQueueOptions } from './memory-queue' +export { + processQueueMessage, + queueConsumer, + QueueTaskError, + type QueueBatchLike, + type QueueConsumerOptions, + type QueueTask, + type QueueTaskContext, + type QueueTaskHandler, + type WorkerCallbackClient, +} from './queue-consumer' +export { + readQueueEffectMessage, + type QueueEffectMessage, + type QueueMessageLike, +} from './queue-message' +export { + cloudflareQueuePullSource, + createQueuePullConsumer, + QueuePullError, + type CloudflareQueuePullSourceOptions, + type PulledQueueMessage, + type QueueLeaseRetry, + type QueuePullConsumer, + type QueuePullConsumerOptions, + type QueuePullResult, + type QueuePullSource, +} from './queue-pull' +export { + cloudflareQueueWorker, + DEFAULT_MAX_QUEUE_MESSAGE_BYTES, + QueueDispatchError, + type CloudflareQueueWorkerOptions, + type QueueProducer, + type QueueRoute, +} from './queue-worker' diff --git a/packages/cloudflare/src/memory-queue.ts b/packages/cloudflare/src/memory-queue.ts new file mode 100644 index 0000000..16b3d5a --- /dev/null +++ b/packages/cloudflare/src/memory-queue.ts @@ -0,0 +1,105 @@ +import { Clock, Effect } from 'effect' + +import type { QueueBatchLike } from './queue-consumer' +import type { QueueEffectMessage } from './queue-message' +import type { PulledQueueMessage, QueuePullSource } from './queue-pull' +import type { QueueProducer } from './queue-worker' + +export interface MemoryQueueOptions { + /** Redeliveries before a message moves to the dead-letter list. Default 3, as on Cloudflare. */ + readonly maxRetries?: number +} + +/** A queue in process memory with Cloudflare's lease, retry, and dead-letter behavior. */ +export interface MemoryQueue extends QueueProducer, QueuePullSource { + /** Messages still held, leased or not. */ + size(): number + deadLetters(): ReadonlyArray + /** Lease visible messages as a push batch, the way a Worker's `queue` handler receives them. */ + batch(size?: number): Effect.Effect +} + +interface Entry { + readonly body: QueueEffectMessage + deliveries: number + visibleAt: number + leaseId: string | undefined +} + +export function createMemoryQueue(options: MemoryQueueOptions = {}): MemoryQueue { + const maxRetries = options.maxRetries ?? 3 + let entries: Entry[] = [] + const dead: QueueEffectMessage[] = [] + let nextLease = 0 + + const settleAt = ( + now: number, + acks: ReadonlyArray, + retries: ReadonlyArray<{ readonly leaseId: string; readonly delaySeconds: number }>, + ) => { + entries = entries.filter( + (entry) => entry.leaseId === undefined || !acks.includes(entry.leaseId), + ) + + for (const retry of retries) { + const entry = entries.find((candidate) => candidate.leaseId === retry.leaseId) + + if (entry) { + entry.leaseId = undefined + entry.visibleAt = now + retry.delaySeconds * 1_000 + } + } + } + + const pull = (batchSize: number, visibilityTimeoutMs: number) => + Effect.map(Clock.currentTimeMillis, (now) => { + const visible = entries.filter((entry) => entry.visibleAt <= now) + const exhausted = visible.filter((entry) => entry.deliveries > maxRetries) + + dead.push(...exhausted.map((entry) => entry.body)) + entries = entries.filter((entry) => !exhausted.includes(entry)) + + const leased = visible + .filter((entry) => entry.deliveries <= maxRetries) + .slice(0, batchSize) + .map((entry): PulledQueueMessage => { + nextLease += 1 + entry.deliveries += 1 + entry.visibleAt = now + visibilityTimeoutMs + entry.leaseId = `lease_${nextLease}` + + return { leaseId: entry.leaseId, body: entry.body, attempts: entry.deliveries } + }) + + return { messages: leased, backlog: entries.length } + }) + + const settle: QueuePullSource['settle'] = (acks, retries) => + Effect.map(Clock.currentTimeMillis, (now) => settleAt(now, acks, retries)) + + return { + send: (message) => { + entries.push({ body: message, deliveries: 0, visibleAt: 0, leaseId: undefined }) + + return Promise.resolve() + }, + pull, + settle, + size: () => entries.length, + deadLetters: () => [...dead], + batch: (size = 100) => + pull(size, 30_000).pipe( + Effect.map(({ messages }) => ({ + messages: messages.map((message) => ({ + body: message.body, + attempts: message.attempts, + ack: () => Effect.runSync(settle([message.leaseId], [])), + retry: (retry) => + Effect.runSync( + settle([], [{ leaseId: message.leaseId, delaySeconds: retry?.delaySeconds ?? 0 }]), + ), + })), + })), + ), + } +} diff --git a/packages/cloudflare/src/queue-consumer.ts b/packages/cloudflare/src/queue-consumer.ts new file mode 100644 index 0000000..74f62ae --- /dev/null +++ b/packages/cloudflare/src/queue-consumer.ts @@ -0,0 +1,240 @@ +import { Data, Effect } from 'effect' + +import { + resolveJson, + type BlobStore, + type EventInput, + type JsonValue, + type SecretContext, + type WorkerCallback, + type WorkerCallbackInput, + type WorkerCallbackResult, + type WorkerSecretsResult, +} from '@looms/core' + +import { + readQueueEffectMessage, + type QueueEffectMessage, + type QueueMessageLike, +} from './queue-message' + +/** The worker half of the Looms HTTP API. `createLoomsClient({ baseUrl, workerCallbackToken })` is one. */ +export interface WorkerCallbackClient { + workerCallback(runId: string, input: WorkerCallbackInput): Promise + workerSecrets(runId: string, attempt: WorkerCallback): Promise +} + +/** The attempt a handler runs, with its input read back from the blob store when it was offloaded. */ +export interface QueueTask extends Omit { + readonly input: JsonValue +} + +export interface QueueTaskContext { + /** Aborts once Looms asks the attempt to stop or no longer counts it as running. */ + readonly signal: AbortSignal + /** Report progress. A later attempt receives the last `details` as `task.details`. */ + heartbeat(details?: JsonValue): Promise + /** The run's secret context. Rejects once the attempt is no longer running. */ + secrets(): Promise +} + +export type QueueTaskHandler = ( + task: QueueTask, + ctx: QueueTaskContext, +) => Promise> | ReadonlyArray + +/** Throw from a handler to control how Looms treats the failed attempt. */ +export class QueueTaskError extends Data.TaggedError('QueueTaskError')<{ + readonly message: string + /** Fail the effect even when the retry policy has attempts left. */ + readonly nonRetryable?: boolean + /** Wait at least this long before the next attempt. */ + readonly retryAfterMs?: number +}> {} + +export interface QueueConsumerOptions { + readonly handlers: Readonly> + readonly callbacks: WorkerCallbackClient + /** Reads inputs the producer offloaded. Use the store the producer writes to. */ + readonly blobs?: BlobStore +} + +export interface QueueBatchLike { + readonly messages: ReadonlyArray +} + +class CallbackError extends Data.TaggedError('CallbackError')<{ readonly message: string }> {} + +/** + * Run one delivery. The message is acked as soon as Looms accepts `started`, and Looms owns every + * retry after that through heartbeat and start-to-close timeouts. A duplicate or stale delivery is + * acked without running. A message that cannot be started yet is retried by the queue, and one that + * never starts ends in the dead-letter queue, where the schedule-to-start timeout recovers it. + */ +export function processQueueMessage( + message: QueueMessageLike, + options: QueueConsumerOptions, +): Effect.Effect { + const task = readQueueEffectMessage(message) + const handler = task && options.handlers[task.type] + + const redeliver = Effect.sync(() => + message.retry({ delaySeconds: redeliveryDelay(message.attempts) }), + ) + + if (!task || !handler) { + return redeliver + } + + const attempt: WorkerCallback = { effectId: task.effectId, attempt: task.attempt } + + return callback(() => + options.callbacks.workerCallback(task.runId, { ...attempt, kind: 'started' }), + ).pipe( + Effect.matchEffect({ + onFailure: () => redeliver, + onSuccess: (started) => + Effect.sync(() => message.ack()).pipe( + Effect.andThen( + started.status === 'accepted' + ? runAttempt(task, attempt, handler, started.cancelRequested, options) + : Effect.void, + ), + ), + }), + ) +} + +/** A push consumer: call it from the Worker's `queue(batch)` handler. */ +export function queueConsumer( + options: QueueConsumerOptions, +): (batch: QueueBatchLike) => Promise { + return (batch) => + Effect.runPromise( + Effect.forEach(batch.messages, (message) => processQueueMessage(message, options), { + concurrency: 'unbounded', + discard: true, + }), + ) +} + +function runAttempt( + task: QueueEffectMessage, + attempt: WorkerCallback, + handler: QueueTaskHandler, + cancelRequested: boolean, + options: QueueConsumerOptions, +): Effect.Effect { + const controller = new AbortController() + + if (cancelRequested) { + controller.abort() + } + + const report = (input: WorkerCallbackInput) => + callback(() => options.callbacks.workerCallback(task.runId, input)).pipe(Effect.ignore) + + const ctx: QueueTaskContext = { + signal: controller.signal, + heartbeat: (details) => + Effect.runPromise( + callback(() => + options.callbacks.workerCallback( + task.runId, + details === undefined + ? { ...attempt, kind: 'heartbeat' } + : { ...attempt, kind: 'heartbeat', details }, + ), + ).pipe( + Effect.map((result) => { + if (result.status !== 'accepted' || result.cancelRequested) { + controller.abort() + } + }), + ), + ), + secrets: () => + Effect.runPromise( + callback(() => options.callbacks.workerSecrets(task.runId, attempt)).pipe( + Effect.flatMap((result) => { + if (result.status === 'accepted') { + return Effect.succeed(result.secrets) + } + + controller.abort() + + return Effect.fail(new CallbackError({ message: `the attempt is ${result.status}` })) + }), + ), + ), + } + + return taskInput(task, options.blobs).pipe( + Effect.flatMap((input) => + Effect.tryPromise({ + try: () => Promise.resolve(handler({ ...taskFields(task), input }, ctx)), + catch: (cause) => + cause instanceof QueueTaskError + ? cause + : new QueueTaskError({ + message: cause instanceof Error ? cause.message : String(cause), + }), + }), + ), + Effect.matchEffect({ + onSuccess: (events) => report({ ...attempt, kind: 'complete', events }), + onFailure: (error) => + report( + controller.signal.aborted + ? { ...attempt, kind: 'cancelled' } + : { + ...attempt, + kind: 'fail', + error: error.message, + nonRetryable: error.nonRetryable, + retryAfterMs: error.retryAfterMs, + }, + ), + }), + ) +} + +function taskFields(task: QueueEffectMessage): Omit { + const { v: _v, input: _input, inputRef: _inputRef, ...fields } = task + + return fields +} + +function taskInput( + task: QueueEffectMessage, + blobs: BlobStore | undefined, +): Effect.Effect { + if (!task.inputRef) { + return Effect.succeed(task.input ?? null) + } + + if (!blobs) { + return Effect.fail( + new QueueTaskError({ message: 'the input was offloaded and the consumer has no blob store' }), + ) + } + + return resolveJson(task.inputRef, blobs).pipe( + Effect.mapError( + (error) => new QueueTaskError({ message: `could not read the input: ${error.message}` }), + ), + ) +} + +function callback(call: () => Promise): Effect.Effect { + return Effect.tryPromise({ + try: call, + catch: (cause) => + new CallbackError({ message: cause instanceof Error ? cause.message : String(cause) }), + }) +} + +/** Seconds before the queue redelivers a message that could not start: 1, 2, 4, ... up to 60. */ +function redeliveryDelay(attempts: number): number { + return Math.min(60, 2 ** Math.max(0, attempts - 1)) +} diff --git a/packages/cloudflare/src/queue-message.ts b/packages/cloudflare/src/queue-message.ts new file mode 100644 index 0000000..7199b38 --- /dev/null +++ b/packages/cloudflare/src/queue-message.ts @@ -0,0 +1,74 @@ +import { Option, Schema } from 'effect' + +import { + isBlobRef, + TraceContextSchema, + type BlobRef, + type JsonValue, + type TraceContext, +} from '@looms/core' +import type { EffectWorkerDeadlines } from '@looms/runtime' + +const QueueEffectMessageSchema = Schema.Struct({ + v: Schema.Literal(1), + runId: Schema.String, + effectId: Schema.String, + attempt: Schema.Finite, + type: Schema.String, + input: Schema.optional(Schema.Json), + inputRef: Schema.optional(Schema.Json), + details: Schema.optional(Schema.Json), + trace: Schema.optional(TraceContextSchema), + deadlines: Schema.optional( + Schema.Struct({ + scheduleToStartAt: Schema.optional(Schema.Finite), + startToCloseTimeoutMs: Schema.optional(Schema.Finite), + heartbeatTimeoutMs: Schema.optional(Schema.Finite), + }), + ), +}) + +const decodeMessage = Schema.decodeUnknownOption(QueueEffectMessageSchema) + +/** + * One effect attempt on a Cloudflare Queue. It never carries the run's secrets; the consumer reads + * them through the worker callback API. Exactly one of `input` and `inputRef` is set. + */ +export interface QueueEffectMessage { + readonly v: 1 + readonly runId: string + readonly effectId: string + readonly attempt: number + readonly type: string + readonly input?: JsonValue + /** Set instead of `input` when the input would push the message over the queue's size limit. */ + readonly inputRef?: BlobRef + /** Progress from the previous attempt's last heartbeat. */ + readonly details?: JsonValue + readonly trace?: TraceContext + readonly deadlines?: EffectWorkerDeadlines +} + +/** A delivered message, shaped like a Cloudflare `Message`. */ +export interface QueueMessageLike { + readonly body: unknown + readonly attempts: number + ack(): void + retry(options?: { readonly delaySeconds?: number }): void +} + +/** `undefined` unless the body is a version 1 message with exactly one of `input` and `inputRef`. */ +export function readQueueEffectMessage(message: QueueMessageLike): QueueEffectMessage | undefined { + return Option.match(decodeMessage(message.body), { + onNone: () => undefined, + onSome: ({ inputRef, ...decoded }) => { + if (inputRef === undefined) { + return decoded.input === undefined ? undefined : decoded + } + + return decoded.input === undefined && isBlobRef(inputRef) + ? { ...decoded, inputRef } + : undefined + }, + }) +} diff --git a/packages/cloudflare/src/queue-pull.ts b/packages/cloudflare/src/queue-pull.ts new file mode 100644 index 0000000..b523bc9 --- /dev/null +++ b/packages/cloudflare/src/queue-pull.ts @@ -0,0 +1,281 @@ +import { + Data, + Deferred, + Effect, + Fiber, + FiberSet, + Option, + Predicate, + Schema, + Semaphore, +} from 'effect' + +import { stringifyJson, type JsonValue } from '@looms/core' + +import { processQueueMessage, type QueueConsumerOptions } from './queue-consumer' +import type { QueueMessageLike } from './queue-message' + +export interface PulledQueueMessage { + readonly leaseId: string + readonly body: unknown + /** Deliveries so far, including this one. */ + readonly attempts: number +} + +export interface QueuePullResult { + readonly messages: ReadonlyArray + /** Messages waiting in the queue, when the source reports it. */ + readonly backlog?: number +} + +export interface QueueLeaseRetry { + readonly leaseId: string + readonly delaySeconds: number +} + +export class QueuePullError extends Data.TaggedError('QueuePullError')<{ + readonly message: string + readonly status?: number +}> {} + +/** Where a pull consumer leases messages from. `cloudflareQueuePullSource` reads a Cloudflare Queue. */ +export interface QueuePullSource { + readonly pull: ( + batchSize: number, + visibilityTimeoutMs: number, + ) => Effect.Effect + readonly settle: ( + acks: ReadonlyArray, + retries: ReadonlyArray, + ) => Effect.Effect +} + +export interface QueuePullConsumerOptions extends QueueConsumerOptions { + readonly source: QueuePullSource + /** Attempts run at once. Default 8. */ + readonly concurrency?: number + /** How long a leased message stays hidden. Acks land long before; default 30 s. */ + readonly visibilityTimeoutMs?: number + /** The longest wait between pulls that come back empty or fail. Default 5 s. */ + readonly maxIdleMs?: number + /** How often buffered acks and retries are sent. Default 250 ms. */ + readonly ackIntervalMs?: number + /** A pull or ack request failed. The consumer backs off and keeps going. */ + readonly onError?: (error: QueuePullError) => void +} + +export interface QueuePullConsumer { + /** Messages waiting in the queue at the last pull, for autoscaling. */ + backlog(): number | undefined + inFlight(): number + /** Stop pulling, let running attempts finish, and send the last acks. Call it on SIGTERM. */ + stop(): Promise +} + +const MAX_PULL_BATCH = 100 +const MIN_IDLE_MS = 50 + +/** Lease messages from `source` and run them with the same handlers as {@link queueConsumer}. */ +export function createQueuePullConsumer(options: QueuePullConsumerOptions): QueuePullConsumer { + const concurrency = Math.max(1, options.concurrency ?? 8) + const visibilityTimeoutMs = options.visibilityTimeoutMs ?? 30_000 + const maxIdleMs = options.maxIdleMs ?? 5_000 + const slots = Semaphore.makeUnsafe(concurrency) + const stopping = Deferred.makeUnsafe() + let backlog: number | undefined + let inFlight = 0 + let acks: string[] = [] + let retries: QueueLeaseRetry[] = [] + + const reportError = (error: QueuePullError) => Effect.sync(() => options.onError?.(error)) + + const flush = Effect.suspend(() => { + if (acks.length === 0 && retries.length === 0) { + return Effect.void + } + + const settled = options.source.settle(acks, retries) + + acks = [] + retries = [] + + return settled.pipe(Effect.catchTag('QueuePullError', reportError)) + }) + + const lease = (message: PulledQueueMessage): QueueMessageLike => ({ + body: message.body, + attempts: message.attempts, + ack: () => { + acks.push(message.leaseId) + }, + retry: (retry) => { + retries.push({ leaseId: message.leaseId, delaySeconds: retry?.delaySeconds ?? 0 }) + }, + }) + + const untilStopped = (effect: Effect.Effect) => + Effect.raceFirst(effect, Deferred.await(stopping)) + + const run = Effect.gen(function* () { + const running = yield* FiberSet.make() + + const flusher = yield* Effect.forkScoped( + Effect.forever(Effect.sleep(options.ackIntervalMs ?? 250).pipe(Effect.andThen(flush))), + ) + + let idleMs = 0 + + while (!(yield* Deferred.isDone(stopping))) { + yield* untilStopped(Semaphore.withPermits(slots, 1)(Effect.void)) + + if (yield* Deferred.isDone(stopping)) { + break + } + + const pulled = yield* options.source + .pull(Math.min(concurrency - inFlight, MAX_PULL_BATCH), visibilityTimeoutMs) + .pipe( + Effect.catchTag('QueuePullError', (error) => + reportError(error).pipe(Effect.as({ messages: [] })), + ), + ) + + backlog = pulled.backlog ?? backlog + + for (const message of pulled.messages) { + inFlight += 1 + + yield* FiberSet.run( + running, + Semaphore.withPermits( + slots, + 1, + )(processQueueMessage(lease(message), options)).pipe( + Effect.ensuring( + Effect.sync(() => { + inFlight -= 1 + }), + ), + ), + ) + } + + idleMs = + pulled.messages.length > 0 ? 0 : Math.min(maxIdleMs, Math.max(MIN_IDLE_MS, idleMs * 2)) + + if (idleMs > 0) { + yield* untilStopped(Effect.sleep(idleMs)) + } + } + + yield* FiberSet.awaitEmpty(running) + yield* Fiber.interrupt(flusher) + yield* flush + }) + + const fiber = Effect.runFork(Effect.scoped(run)) + + return { + backlog: () => backlog, + inFlight: () => inFlight, + stop: () => + Effect.runPromise( + Deferred.succeed(stopping, undefined).pipe(Effect.andThen(Fiber.join(fiber))), + ), + } +} + +export interface CloudflareQueuePullSourceOptions { + readonly accountId: string + readonly queueId: string + /** An API token with Queues edit permission. */ + readonly token: string + readonly fetch?: typeof fetch +} + +const PullResponseSchema = Schema.Struct({ + result: Schema.Struct({ + message_backlog_count: Schema.optional(Schema.Finite), + messages: Schema.Array( + Schema.Struct({ + lease_id: Schema.String, + attempts: Schema.Finite, + body: Schema.Json, + }), + ), + }), +}) + +const decodePullResponse = Schema.decodeUnknownEffect(PullResponseSchema) +const decodeJsonText = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Json)) + +/** The Cloudflare Queues HTTP pull API. The producer must send `contentType: 'json'`. */ +export function cloudflareQueuePullSource( + options: CloudflareQueuePullSourceOptions, +): QueuePullSource { + const fetchImpl = options.fetch ?? fetch + + const endpoint = `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(options.accountId)}/queues/${encodeURIComponent(options.queueId)}/messages` + + const post = (path: string, body: JsonValue) => + Effect.tryPromise({ + try: () => + fetchImpl(`${endpoint}/${path}`, { + method: 'POST', + headers: { + authorization: `Bearer ${options.token}`, + 'content-type': 'application/json', + }, + body: stringifyJson(body), + }), + catch: (cause) => + new QueuePullError({ message: cause instanceof Error ? cause.message : String(cause) }), + }).pipe( + Effect.filterOrFail( + (response) => response.ok, + (response) => + new QueuePullError({ + message: `Cloudflare Queues ${path} returned ${response.status}`, + status: response.status, + }), + ), + ) + + return { + pull: (batchSize, visibilityTimeoutMs) => + post('pull', { batch_size: batchSize, visibility_timeout_ms: visibilityTimeoutMs }).pipe( + Effect.flatMap((response) => + Effect.tryPromise({ + try: () => response.json(), + catch: () => new QueuePullError({ message: 'the pull response is not JSON' }), + }), + ), + Effect.flatMap((json) => + decodePullResponse(json).pipe( + Effect.mapError(() => new QueuePullError({ message: 'unexpected pull response' })), + ), + ), + Effect.map(({ result }) => ({ + backlog: result.message_backlog_count, + messages: result.messages.map((message) => ({ + leaseId: message.lease_id, + attempts: message.attempts, + body: jsonBody(message.body), + })), + })), + ), + settle: (acks, retries) => + post('ack', { + acks: acks.map((leaseId) => ({ lease_id: leaseId })), + retries: retries.map((retry) => ({ + lease_id: retry.leaseId, + delay_seconds: retry.delaySeconds, + })), + }).pipe(Effect.asVoid), + } +} + +/** The pull API returns a JSON message's body as its JSON text. */ +function jsonBody(body: JsonValue): JsonValue { + return Predicate.isString(body) ? Option.getOrElse(decodeJsonText(body), () => body) : body +} diff --git a/packages/cloudflare/src/queue-worker.test.ts b/packages/cloudflare/src/queue-worker.test.ts new file mode 100644 index 0000000..ee6e938 --- /dev/null +++ b/packages/cloudflare/src/queue-worker.test.ts @@ -0,0 +1,489 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { createLoomsClient } from '@looms/client' +import { + complete, + defineEffect, + defineRuntimeModule, + defineThread, + invoke, + memoryBlobStore, + utf8JsonBytes, + type BlobStore, + type JsonValue, + type RetryPolicy, +} from '@looms/core' +import { bearerAuth, createLooms, type Looms } from '@looms/runtime' + +import { createMemoryQueue, type MemoryQueue } from './memory-queue' +import { queueConsumer, QueueTaskError, type QueueTaskHandler } from './queue-consumer' +import type { QueueEffectMessage } from './queue-message' +import { cloudflareQueuePullSource, createQueuePullConsumer } from './queue-pull' +import { cloudflareQueueWorker, type QueueProducer } from './queue-worker' + +const WORKER_TOKEN = 'worker-token' +const SECRET = 'sk_live_never_logged' +const definition = { kind: 'sandbox', name: 'sandbox', version: 'v1' } + +const FAST: RetryPolicy = { + maxAttempts: 3, + backoffMs: 5, + scheduleToStartTimeoutMs: 60_000, +} + +interface SandboxState { + readonly input: JsonValue + readonly output: JsonValue | null +} + +function sandboxModule(retry: RetryPolicy) { + const exec = defineEffect({ type: 'sandbox.exec', retry }) + + return defineRuntimeModule({ + namespace: 'sandbox', + protocolVersion: '1.0.0', + effects: { exec }, + threads: { + sandbox: defineThread({ + kind: 'sandbox', + initialState: (ctx): SandboxState => ({ input: ctx.input ?? null, output: null }), + step: (state, event) => + event.type === 'sandbox.done' ? { ...state, output: event.payload ?? true } : state, + effects: (state) => + state.output === null ? [invoke(exec, state.input, 'exec')] : [complete(state.output)], + }), + }, + definitions: [definition], + }) +} + +interface Harness { + readonly looms: Looms + readonly queue: MemoryQueue + readonly sent: QueueEffectMessage[] + readonly callbacks: ReturnType +} + +function harness( + options: { + retry?: RetryPolicy + producer?: (queue: MemoryQueue, sent: QueueEffectMessage[]) => QueueProducer + blobs?: BlobStore + maxRetries?: number + } = {}, +): Harness { + const queue = createMemoryQueue({ maxRetries: options.maxRetries }) + const sent: QueueEffectMessage[] = [] + + const producer: QueueProducer = options.producer?.(queue, sent) ?? { + send: (message, sendOptions) => { + sent.push(message) + + return queue.send(message, sendOptions) + }, + } + + const looms = createLooms({ + modules: [sandboxModule(options.retry ?? FAST)], + worker: cloudflareQueueWorker({ + routes: [{ handles: ['sandbox.exec'], queue: producer }], + blobs: options.blobs, + }), + authorize: bearerAuth({ worker: WORKER_TOKEN }), + }) + + const callbacks = createLoomsClient({ + baseUrl: 'http://looms.test', + workerCallbackToken: WORKER_TOKEN, + fetch: async (input, init) => + (await looms.fetch(new Request(input, init))) ?? new Response(null, { status: 404 }), + }) + + return { looms, queue, sent, callbacks } +} + +const done: QueueTaskHandler = (task) => [{ type: 'sandbox.done', payload: { ran: task.attempt } }] + +async function waitFor(condition: () => boolean | Promise, timeoutMs = 2_000) { + const deadline = Date.now() + timeoutMs + + while (!(await condition())) { + if (Date.now() > deadline) { + throw new Error('condition not met in time') + } + + await Bun.sleep(5) + } +} + +async function status(looms: Looms, runId: string) { + return (await looms.getRun(runId)).status +} + +async function eventTypes(looms: Looms, runId: string) { + return (await looms.getEvents(runId)).map((event) => event.type) +} + +describe('Cloudflare Queues effect transport', () => { + test('runs an attempt over the queue and reads secrets only through the callback', async () => { + const { looms, queue, sent, callbacks } = harness() + let secret: string | undefined + + const started = await looms.start( + definition, + { cmd: 'ls' }, + { + secrets: { token: SECRET }, + trace: { traceparent: '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01' }, + }, + ) + + const consumer = createQueuePullConsumer({ + source: queue, + callbacks, + handlers: { + 'sandbox.exec': async (task, ctx) => { + secret = (await ctx.secrets()).token + + return done(task, ctx) + }, + }, + }) + + await waitFor(async () => (await status(looms, started.runId)) === 'completed') + await consumer.stop() + + expect(secret).toBe(SECRET) + expect(JSON.stringify(sent)).not.toContain(SECRET) + expect(sent[0]?.input).toEqual({ cmd: 'ls' }) + expect(sent[0]?.trace?.traceparent).toContain('4bf92f3577b34da6a3ce929d0e0e4736') + expect(sent[0]?.deadlines?.scheduleToStartAt).toBeGreaterThan(0) + expect(queue.size()).toBe(0) + await looms.stop() + }) + + test('a duplicate delivery runs the handler once', async () => { + let runs = 0 + + const { looms, queue, callbacks } = harness({ + producer: (memory) => ({ + send: async (message, sendOptions) => { + await memory.send(message, sendOptions) + await memory.send(message, sendOptions) + }, + }), + }) + + const started = await looms.start(definition, null) + + await queueConsumer({ + callbacks, + handlers: { + 'sandbox.exec': (task, ctx) => { + runs += 1 + + return done(task, ctx) + }, + }, + })(await Effect.runPromise(queue.batch())) + + expect(runs).toBe(1) + expect(queue.size()).toBe(0) + expect(await status(looms, started.runId)).toBe('completed') + await looms.stop() + }) + + test('a consumer that dies before started leaves the message for redelivery', async () => { + const { looms, queue, callbacks } = harness() + const started = await looms.start(definition, null) + + await Effect.runPromise(queue.pull(10, 20)) + + const consumer = createQueuePullConsumer({ + source: queue, + callbacks, + handlers: { 'sandbox.exec': done }, + visibilityTimeoutMs: 20, + }) + + await waitFor(async () => (await status(looms, started.runId)) === 'completed') + await consumer.stop() + + expect(queue.size()).toBe(0) + await looms.stop() + }) + + test('an attempt that dies after started is retried by Looms, and its stale message is dropped', async () => { + const { looms, queue, sent, callbacks } = harness({ + retry: { ...FAST, heartbeatTimeoutMs: 40 }, + }) + + const ran: number[] = [] + + const consume = queueConsumer({ + callbacks, + handlers: { + 'sandbox.exec': (task, ctx) => { + ran.push(task.attempt) + + return task.attempt === 1 ? new Promise(() => undefined) : done(task, ctx) + }, + }, + }) + + const started = await looms.start(definition, null) + + void consume(await Effect.runPromise(queue.batch())) + + await waitFor(() => sent.length === 2) + await consume(await Effect.runPromise(queue.batch())) + + expect(await status(looms, started.runId)).toBe('completed') + + const first = sent[0] + + if (!first) { + throw new Error('missing first attempt') + } + + await queue.send(first) + await consume(await Effect.runPromise(queue.batch())) + + expect(ran).toEqual([1, 2]) + expect(queue.size()).toBe(0) + await looms.stop() + }) + + test('cancelling the run aborts the running attempt through its heartbeat', async () => { + const { looms, queue, callbacks } = harness() + let aborted = false + + const started = await looms.start(definition, null) + + const consumer = createQueuePullConsumer({ + source: queue, + callbacks, + handlers: { + 'sandbox.exec': async (_task, ctx) => { + while (!ctx.signal.aborted) { + await ctx.heartbeat({ at: Date.now() }) + await Bun.sleep(5) + } + + aborted = true + + throw new Error('stopped') + }, + }, + }) + + await waitFor(async () => + (await eventTypes(looms, started.runId)).includes('runtime.effect.heartbeat'), + ) + + await looms.cancel(started.runId) + await waitFor(() => aborted) + await consumer.stop() + + expect(await eventTypes(looms, started.runId)).toContain('runtime.effect.cancelled') + expect(await status(looms, started.runId)).toBe('cancelled') + await looms.stop() + }) + + test('a send the queue rejects is retried with backoff', async () => { + let rejected = 0 + + const { looms, queue, sent } = harness({ + producer: (memory, attempts) => ({ + send: (message, sendOptions) => { + attempts.push(message) + + if (rejected === 0) { + rejected += 1 + + return Promise.reject(new Error('429 Too Many Requests')) + } + + return memory.send(message, sendOptions) + }, + }), + }) + + const started = await looms.start(definition, null) + + await waitFor(() => queue.size() === 1) + + const types = await eventTypes(looms, started.runId) + + expect(types).toContain('runtime.effect.retry.scheduled') + expect(types).not.toContain('runtime.effect.ambiguous') + expect(sent.map((message) => message.attempt)).toEqual([1, 2]) + await looms.stop() + }) + + test('an input over the message limit travels as a blob reference', async () => { + const blobs = memoryBlobStore() + const { looms, queue, sent, callbacks } = harness({ blobs }) + const input = { file: 'x'.repeat(150_000) } + let length = 0 + + const started = await looms.start(definition, input) + + await queueConsumer({ + callbacks, + blobs, + handlers: { + 'sandbox.exec': (task, ctx) => { + length = JSON.stringify(task.input).length + + return done(task, ctx) + }, + }, + })(await Effect.runPromise(queue.batch())) + + expect(sent[0]?.input).toBeUndefined() + expect(sent[0]?.inputRef?.bytes).toBe(utf8JsonBytes(input)) + expect(utf8JsonBytes(sent[0] ?? null)).toBeLessThan(128 * 1024) + expect(length).toBe(JSON.stringify(input).length) + expect(await status(looms, started.runId)).toBe('completed') + await looms.stop() + }) + + test('a dead-lettered message is recovered by the schedule-to-start timeout', async () => { + const { looms, queue, sent, callbacks } = harness({ + retry: { ...FAST, scheduleToStartTimeoutMs: 60 }, + maxRetries: 0, + }) + + const started = await looms.start(definition, null) + + await Effect.runPromise(queue.pull(10, 1)) + await Bun.sleep(5) + await Effect.runPromise(queue.pull(10, 1)) + + expect(queue.deadLetters()).toHaveLength(1) + + await waitFor(() => sent.length === 2) + + await queueConsumer({ callbacks, handlers: { 'sandbox.exec': done } })( + await Effect.runPromise(queue.batch()), + ) + + expect(await status(looms, started.runId)).toBe('completed') + await looms.stop() + }) + + test('a non-retryable handler error fails the effect without another attempt', async () => { + const { looms, queue, sent, callbacks } = harness() + const started = await looms.start(definition, null) + + await queueConsumer({ + callbacks, + handlers: { + 'sandbox.exec': () => { + throw new QueueTaskError({ message: 'bad input', nonRetryable: true }) + }, + }, + })(await Effect.runPromise(queue.batch())) + + expect(await status(looms, started.runId)).toBe('failed') + expect(sent).toHaveLength(1) + await looms.stop() + }) + + test('stop lets running attempts finish, sends their acks, and pulls nothing more', async () => { + const { looms, queue, callbacks } = harness() + const gate = Promise.withResolvers() + + const first = await looms.start(definition, null) + + const consumer = createQueuePullConsumer({ + source: queue, + callbacks, + handlers: { 'sandbox.exec': async (task, ctx) => gate.promise.then(() => done(task, ctx)) }, + }) + + await waitFor(() => consumer.inFlight() === 1) + + const stopped = consumer.stop() + + gate.resolve() + await stopped + + expect(await status(looms, first.runId)).toBe('completed') + expect(queue.size()).toBe(0) + + await looms.start(definition, null) + await Bun.sleep(20) + + expect(queue.size()).toBe(1) + await looms.stop() + }) +}) + +describe('cloudflareQueuePullSource', () => { + test('pulls and acks through the Cloudflare Queues HTTP API', async () => { + const requests: Array<{ url: string; auth: string | null; body: string }> = [] + + const source = cloudflareQueuePullSource({ + accountId: 'acct', + queueId: 'q1', + token: 'cf-token', + fetch: async (input, init) => { + const request = new Request(input, init) + + requests.push({ + url: request.url, + auth: request.headers.get('authorization'), + body: await request.text(), + }) + + return request.url.endsWith('/pull') + ? Response.json({ + result: { + message_backlog_count: 7, + messages: [{ lease_id: 'l1', attempts: 2, body: '{"v":1}' }], + }, + }) + : Response.json({ success: true }) + }, + }) + + const pulled = await Effect.runPromise(source.pull(10, 30_000)) + + await Effect.runPromise(source.settle(['l1'], [{ leaseId: 'l2', delaySeconds: 4 }])) + + expect(pulled).toEqual({ + backlog: 7, + messages: [{ leaseId: 'l1', attempts: 2, body: { v: 1 } }], + }) + + expect(requests).toEqual([ + { + url: 'https://api.cloudflare.com/client/v4/accounts/acct/queues/q1/messages/pull', + auth: 'Bearer cf-token', + body: '{"batch_size":10,"visibility_timeout_ms":30000}', + }, + { + url: 'https://api.cloudflare.com/client/v4/accounts/acct/queues/q1/messages/ack', + auth: 'Bearer cf-token', + body: '{"acks":[{"lease_id":"l1"}],"retries":[{"lease_id":"l2","delay_seconds":4}]}', + }, + ]) + }) + + test('a rate-limited pull fails with its status', async () => { + const source = cloudflareQueuePullSource({ + accountId: 'acct', + queueId: 'q1', + token: 'cf-token', + fetch: async () => new Response(null, { status: 429 }), + }) + + const error = await Effect.runPromise(Effect.flip(source.pull(10, 30_000))) + + expect(error.status).toBe(429) + }) +}) diff --git a/packages/cloudflare/src/queue-worker.ts b/packages/cloudflare/src/queue-worker.ts new file mode 100644 index 0000000..86baf2e --- /dev/null +++ b/packages/cloudflare/src/queue-worker.ts @@ -0,0 +1,123 @@ +import { Data, Effect, Predicate, type Types } from 'effect' + +import { claimJson, utf8JsonBytes, type BlobStore } from '@looms/core' +import type { EffectWorker, EffectWorkerTask } from '@looms/runtime' + +import type { QueueEffectMessage } from './queue-message' + +/** Leaves room under Cloudflare's 128 KB message limit. */ +export const DEFAULT_MAX_QUEUE_MESSAGE_BYTES = 100_000 + +/** A Cloudflare Queue producer binding, or anything that sends the same way. */ +export interface QueueProducer { + send(message: QueueEffectMessage, options?: { readonly contentType?: 'json' }): Promise +} + +export interface QueueRoute { + readonly handles: ReadonlyArray | ((type: string) => boolean) + readonly queue: QueueProducer +} + +export interface CloudflareQueueWorkerOptions { + /** The first route that handles an effect type gets its attempts. */ + readonly routes: ReadonlyArray + /** Holds an input that would push a message past `maxMessageBytes`. Without it, that dispatch fails. */ + readonly blobs?: BlobStore + readonly maxMessageBytes?: number +} + +export class QueueDispatchError extends Data.TaggedError('QueueDispatchError')<{ + readonly message: string +}> {} + +/** + * An {@link EffectWorker} that sends each attempt to a Cloudflare Queue. A send the queue rejects + * (a 429, for one) fails the attempt under the effect's retry policy. A message the queue accepts + * and later dead-letters is recovered by the effect's schedule-to-start timeout, which every routed + * effect must set. Cancellation reaches a running attempt through its callback responses, since a + * queued message cannot be recalled. + */ +export function cloudflareQueueWorker(options: CloudflareQueueWorkerOptions): EffectWorker { + const routeFor = (type: string) => + options.routes.find((route) => + Predicate.isFunction(route.handles) ? route.handles(type) : route.handles.includes(type), + ) + + return { + dispatchSemantics: 'acknowledged', + handles: (type) => routeFor(type) !== undefined, + dispatch: (task) => Effect.runPromise(sendTask(task, routeFor(task.type), options)), + } +} + +function sendTask( + task: EffectWorkerTask, + route: QueueRoute | undefined, + options: CloudflareQueueWorkerOptions, +): Effect.Effect { + if (!route) { + return Effect.fail(new QueueDispatchError({ message: `no queue route handles "${task.type}"` })) + } + + return queueMessage(task, options).pipe( + Effect.flatMap((message) => + Effect.tryPromise({ + try: () => route.queue.send(message, { contentType: 'json' }), + catch: (cause) => + new QueueDispatchError({ + message: `queue send failed: ${cause instanceof Error ? cause.message : String(cause)}`, + }), + }), + ), + ) +} + +function queueMessage( + task: EffectWorkerTask, + options: CloudflareQueueWorkerOptions, +): Effect.Effect { + const message: Types.Mutable = { + v: 1, + runId: task.runId, + effectId: task.effectId, + attempt: task.attempt, + type: task.type, + } + + if (task.details !== undefined) { + message.details = task.details + } + + if (task.trace) { + message.trace = task.trace + } + + if (task.deadlines) { + message.deadlines = task.deadlines + } + + const inline: QueueEffectMessage = { ...message, input: task.input } + const limit = options.maxMessageBytes ?? DEFAULT_MAX_QUEUE_MESSAGE_BYTES + + if (utf8JsonBytes(inline) <= limit) { + return Effect.succeed(inline) + } + + const { blobs } = options + + if (!blobs) { + return Effect.fail( + new QueueDispatchError({ + message: `the "${task.type}" message is over ${limit} bytes and no blob store is configured`, + }), + ) + } + + return claimJson(task.input, blobs).pipe( + Effect.tap((ref) => blobs.retain(task.runId, [ref.sha256])), + Effect.map((inputRef): QueueEffectMessage => ({ ...message, inputRef })), + Effect.mapError( + (error) => new QueueDispatchError({ message: `could not store the input: ${error.message}` }), + ), + ) +} diff --git a/packages/cloudflare/src/r2-blob-store.test.ts b/packages/cloudflare/src/r2-blob-store.test.ts new file mode 100644 index 0000000..b2b4207 --- /dev/null +++ b/packages/cloudflare/src/r2-blob-store.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { isBlobRef, isJsonObject, offloadJson, resolveJson } from '@looms/core' + +import { r2BlobStore, type R2BucketLike, type R2ListResult } from './r2-blob-store' + +function fakeBucket(): R2BucketLike & { readonly keys: () => string[] } { + const objects = new Map() + + return { + keys: () => [...objects.keys()], + + put: (key, value) => { + objects.set(key, value.slice()) + return Promise.resolve({ key }) + }, + + get: (key) => { + const body = objects.get(key) + + if (!body) { + return Promise.resolve(null) + } + + const copy = body.slice() + + return Promise.resolve({ + arrayBuffer: () => + Promise.resolve(copy.buffer.slice(copy.byteOffset, copy.byteOffset + copy.byteLength)), + }) + }, + + head: (key) => Promise.resolve(objects.has(key) ? { key } : null), + + delete: (key) => { + objects.delete(key) + return Promise.resolve(undefined) + }, + + list: (options) => { + const prefix = options?.prefix ?? '' + const keys = [...objects.keys()].filter((key) => key.startsWith(prefix)).toSorted() + const start = options?.cursor === undefined ? 0 : Number(options.cursor) + const page = keys.slice(start, start + 1) + const next = start + page.length + const truncated = next < keys.length + + const result: R2ListResult = { + objects: page.map((key) => ({ key })), + truncated, + } + + if (truncated) { + return Promise.resolve({ ...result, cursor: String(next) }) + } + + return Promise.resolve(result) + }, + } +} + +describe('r2BlobStore', () => { + test('pages the bucket and resolves an offloaded value', async () => { + const bucket = fakeBucket() + const blobs = r2BlobStore(bucket) + const value = { text: 'q'.repeat(80) } + const stored = await Effect.runPromise(offloadJson(value, blobs, { thresholdBytes: 16 })) + const text = isJsonObject(stored) ? stored.text : undefined + + if (text === undefined || !isBlobRef(text)) { + throw new Error('expected the string to be stored by hash') + } + + await Effect.runPromise(blobs.retain('run-a', [text.sha256])) + await Effect.runPromise(blobs.retain('run-b', [text.sha256])) + await Effect.runPromise(blobs.replaceRefs('run-a', new Set())) + + const resolved = await Effect.runPromise(resolveJson(stored, blobs)) + + expect(resolved).toEqual(value) + + await Effect.runPromise(blobs.replaceRefs('run-b', new Set())) + + expect(bucket.keys().filter((key) => key.includes(text.sha256))).toEqual([]) + }) +}) diff --git a/packages/cloudflare/src/r2-blob-store.ts b/packages/cloudflare/src/r2-blob-store.ts new file mode 100644 index 0000000..af6fb1f --- /dev/null +++ b/packages/cloudflare/src/r2-blob-store.ts @@ -0,0 +1,110 @@ +import { Effect } from 'effect' + +import { blobStoreFromBytes, BlobStoreError, type BlobBytes, type BlobStore } from '@looms/core' + +const PREFIX = 'looms/blobs/' + +export interface R2ObjectBodyLike { + arrayBuffer(): Promise +} + +export interface R2ListResult { + readonly objects: ReadonlyArray<{ readonly key: string }> + readonly truncated: boolean + readonly cursor?: string +} + +/** + * Structural subset of a Cloudflare `R2Bucket` binding. An `R2Bucket` satisfies it. + */ +export interface R2StoredObject { + readonly key: string +} + +export interface R2BucketLike { + put(key: string, value: Uint8Array): Promise + get(key: string): Promise + head(key: string): Promise + delete(key: string): Promise + list(options?: { prefix?: string; cursor?: string }): Promise +} + +/** + * Content-addressed blob store in R2 under `looms/blobs/`. Every run's Durable Object can + * share one bucket: each run writes its own reference markers, and a body is deleted only + * after no marker for it remains. + */ +export function r2BlobStore(bucket: R2BucketLike): BlobStore { + return blobStoreFromBytes(r2BlobBytes(bucket)) +} + +function r2BlobBytes(bucket: R2BucketLike): BlobBytes { + const keyFor = (key: string) => `${PREFIX}${key}` + + return { + read: (key) => + Effect.tryPromise({ + try: () => bucket.get(keyFor(key)), + catch: (cause) => new BlobStoreError(`Unable to read blob key ${key}`, cause), + }).pipe( + Effect.flatMap((object) => { + if (!object) { + return Effect.succeed(null) + } + + return Effect.tryPromise({ + try: () => object.arrayBuffer(), + catch: (cause) => new BlobStoreError(`Unable to read blob key ${key}`, cause), + }).pipe(Effect.map((buffer) => new Uint8Array(buffer))) + }), + ), + + write: (key, body) => + Effect.tryPromise({ + try: () => bucket.put(keyFor(key), body), + catch: (cause) => new BlobStoreError(`Unable to write blob key ${key}`, cause), + }).pipe(Effect.asVoid), + + remove: (key) => + Effect.tryPromise({ + try: () => bucket.delete(keyFor(key)), + catch: (cause) => new BlobStoreError(`Unable to delete blob key ${key}`, cause), + }).pipe(Effect.asVoid), + + exists: (key) => + Effect.tryPromise({ + try: () => bucket.head(keyFor(key)), + catch: (cause) => new BlobStoreError(`Unable to check blob key ${key}`, cause), + }).pipe(Effect.map((object) => object !== null)), + + list: (prefix) => listPrefixed(bucket, `${PREFIX}${prefix}`), + } +} + +function listPrefixed( + bucket: R2BucketLike, + prefix: string, +): Effect.Effect, BlobStoreError> { + return Effect.gen(function* () { + const keys: string[] = [] + let cursor: string | undefined + + for (;;) { + const page = yield* Effect.tryPromise({ + try: () => + cursor === undefined ? bucket.list({ prefix }) : bucket.list({ prefix, cursor }), + catch: (cause) => new BlobStoreError(`Unable to list blob keys ${prefix}`, cause), + }) + + for (const object of page.objects) { + keys.push(object.key.slice(PREFIX.length)) + } + + if (!page.truncated || page.cursor === undefined || page.cursor === cursor) { + return keys + } + + cursor = page.cursor + } + }) +} diff --git a/packages/cloudflare/src/websocket.test.ts b/packages/cloudflare/src/websocket.test.ts new file mode 100644 index 0000000..dc7a538 --- /dev/null +++ b/packages/cloudflare/src/websocket.test.ts @@ -0,0 +1,337 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { + createEvent, + emptyRunState, + makeMemoryEventStore, + makeMemorySnapshotStore, + type EventStore, + type SubscriptionAttachment, +} from '@looms/core' +import { authAllowed, createRuntime, type RouteInfo } from '@looms/runtime' + +import { attachSocketFanout, openRunSocket, type HibernatingSocketState } from './websocket' + +const runtime = createRuntime({ modules: [] }) + +class FakeSocket { + readonly sent: string[] = [] + closed: { code: number; reason: string } | undefined + bufferedAmount = 0 + private attachment: SubscriptionAttachment | undefined + + send(data: string): void { + this.sent.push(data) + } + + close(code?: number, reason?: string): void { + this.closed = { code: code ?? 0, reason: reason ?? '' } + } + + serializeAttachment(value: SubscriptionAttachment): void { + this.attachment = value + } + + deserializeAttachment(): SubscriptionAttachment | undefined { + return this.attachment + } +} + +class FakeState implements HibernatingSocketState { + readonly accepted: FakeSocket[] = [] + autoResponse: { request: string; response: string } | undefined + + acceptWebSocket(ws: WebSocket): void { + if (ws instanceof FakeSocket) { + this.accepted.push(ws) + } + } + + getWebSockets(): readonly WebSocket[] { + return this.accepted.map(asSocket) + } + + setWebSocketAutoResponse(pair: WebSocketRequestResponsePair): void { + this.autoResponse = { request: pair.request, response: pair.response } + } +} + +function pairFrom(server: FakeSocket, client: FakeSocket) { + return () => ({ client: asSocket(client), server: asSocket(server) }) +} + +function asSocket(socket: FakeSocket): WebSocket { + // SAFETY: the test socket implements send, close, bufferedAmount, and attachment. + // oxlint-disable-next-line typescript/no-unsafe-type-assertion + return socket as WebSocket +} + +async function seededStore(): Promise { + const store = await Effect.runPromise(makeMemoryEventStore) + + await Effect.runPromise( + store.append('run_1', [threadEvent('thr_a'), threadEvent('thr_b')], { expectedTail: 0 }), + ) + + return store +} + +function threadEvent(threadId: string) { + return createEvent('run_1', { + type: 'runtime.thread.started', + payload: { + threadId, + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v1', + input: null, + parentThreadId: null, + }, + threadId, + origin: { type: 'system' }, + }) +} + +function upgrade(url: string): Request { + return new Request(url, { headers: { upgrade: 'websocket' } }) +} + +describe('hibernating run sockets', () => { + test('replays the log, then fans out a later append', async () => { + const store = await seededStore() + const ctx = new FakeState() + const server = new FakeSocket() + const broadcasting = attachSocketFanout(ctx, store, 'run_1') + + const response = await Effect.runPromise( + openRunSocket({ + runtime, + ctx, + store, + request: upgrade('http://looms.test/runs/run_1/events'), + runId: 'run_1', + createPair: pairFrom(server, new FakeSocket()), + }), + ) + + expect(response.status).toBe(101) + const frames = server.sent.map((line) => JSON.parse(line)) + expect(frames[0]).toEqual({ type: 'hello', head: 1, tail: 2 }) + expect(frames[1]?.batch).toHaveLength(2) + + expect(frames[1]?.batch.map((item: { threadId: string }) => item.threadId)).toEqual([ + 'thr_a', + 'thr_b', + ]) + + await Effect.runPromise( + broadcasting.append('run_1', [threadEvent('thr_a')], { expectedTail: 2 }), + ) + + const live = server.sent.map((line) => JSON.parse(line)) + expect(live.at(-1)).toMatchObject({ batch: [expect.objectContaining({ seq: 3 })] }) + }) + + test('resumes after fromSeq and drops other threads', async () => { + const store = await seededStore() + const server = new FakeSocket() + + await Effect.runPromise( + openRunSocket({ + runtime, + ctx: new FakeState(), + store, + request: upgrade('http://looms.test/runs/run_1/events?fromSeq=2&threadId=thr_b'), + runId: 'run_1', + createPair: pairFrom(server, new FakeSocket()), + }), + ) + + const frames = server.sent.map((line) => JSON.parse(line)) + const batches = frames.filter((frame) => 'batch' in frame) + expect(batches).toHaveLength(1) + expect(batches[0].batch[0].threadId).toBe('thr_b') + expect(batches[0].batch[0].seq).toBe(2) + }) + + test('a trimmed cursor closes with resync', async () => { + const store = await seededStore() + + if (!store.trim) { + throw new Error('memory store does not trim') + } + + const snapshotStore = await Effect.runPromise(makeMemorySnapshotStore) + + await Effect.runPromise( + snapshotStore.save({ + runId: 'run_1', + cursor: 2, + stateHash: 'test', + takenAt: 0, + state: emptyRunState('run_1'), + }), + ) + + await Effect.runPromise(store.trim('run_1', 2)) + const server = new FakeSocket() + + await Effect.runPromise( + openRunSocket({ + runtime: createRuntime({ modules: [], snapshotStore }), + ctx: new FakeState(), + store, + request: upgrade('http://looms.test/runs/run_1/events?fromSeq=1'), + runId: 'run_1', + createPair: pairFrom(server, new FakeSocket()), + }), + ) + + expect(JSON.parse(server.sent[0] ?? '{}')).toEqual({ + type: 'resync', + head: 2, + reason: 'trimmed', + }) + + expect(server.closed).toEqual({ code: 4000, reason: 'trimmed' }) + }) + + test('backpressure closes the socket', async () => { + const store = await seededStore() + const server = new FakeSocket() + server.bufferedAmount = 2 * 1024 * 1024 + + await Effect.runPromise( + openRunSocket({ + runtime, + ctx: new FakeState(), + store, + request: upgrade('http://looms.test/runs/run_1/events'), + runId: 'run_1', + createPair: pairFrom(server, new FakeSocket()), + }), + ) + + expect(server.closed?.reason).toBe('backpressure') + + expect(server.sent.map((line) => JSON.parse(line))).toContainEqual({ + type: 'resync', + head: 1, + reason: 'backpressure', + }) + }) + + test('refuses a socket the authorizer rejected', async () => { + const store = await seededStore() + const ctx = new FakeState() + + const response = await Effect.runPromise( + openRunSocket({ + runtime, + ctx, + store, + request: upgrade('http://looms.test/runs/run_1/events'), + runId: 'run_1', + authorize: () => Effect.succeed({ allowed: false, status: 401, error: 'no' }), + }), + ) + + expect(response.status).toBe(401) + expect(ctx.accepted).toHaveLength(0) + }) + + test('authorizes with the run attributes like every other run route', async () => { + const store = await Effect.runPromise(makeMemoryEventStore) + const routes: RouteInfo[] = [] + + const started = createEvent('run_1', { + type: 'runtime.run.started', + payload: { + rootThreadId: 'thr_a', + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v1', + input: null, + requestedThreadId: null, + attributes: { tenant: 'acme' }, + }, + threadId: 'thr_a', + origin: { type: 'system' }, + }) + + await Effect.runPromise(store.append('run_1', [started], { expectedTail: 0 })) + + const response = await Effect.runPromise( + openRunSocket({ + runtime, + ctx: new FakeState(), + store, + request: upgrade('http://looms.test/runs/run_1/events'), + runId: 'run_1', + createPair: pairFrom(new FakeSocket(), new FakeSocket()), + authorize: (_request, route) => { + routes.push(route) + return Effect.succeed(authAllowed) + }, + }), + ) + + expect(response.status).toBe(101) + + expect(routes).toEqual([ + { access: 'read', name: 'runs.events', runId: 'run_1', attributes: { tenant: 'acme' } }, + ]) + }) + + test('echoes a grant offered as a WebSocket subprotocol', async () => { + const store = await seededStore() + + const response = await Effect.runPromise( + openRunSocket({ + runtime, + ctx: new FakeState(), + store, + request: new Request('http://looms.test/runs/run_1/events', { + headers: { upgrade: 'websocket', 'sec-websocket-protocol': 'chat, v1.abc.def' }, + }), + runId: 'run_1', + createPair: pairFrom(new FakeSocket(), new FakeSocket()), + }), + ) + + expect(response.status).toBe(101) + expect(response.headers.get('sec-websocket-protocol')).toBe('v1.abc.def') + }) + + test('rejects an invalid Last-Event-ID and a mismatched run', async () => { + const store = await seededStore() + + const invalid = await Effect.runPromise( + openRunSocket({ + runtime, + ctx: new FakeState(), + store, + request: new Request('http://looms.test/runs/run_1/events', { + headers: { upgrade: 'websocket', 'last-event-id': 'nope' }, + }), + runId: 'run_1', + }), + ) + + expect(invalid.status).toBe(400) + + const missing = await Effect.runPromise( + openRunSocket({ + runtime, + ctx: new FakeState(), + store, + request: upgrade('http://looms.test/runs/other/events'), + runId: 'run_1', + }), + ) + + expect(missing.status).toBe(404) + }) +}) diff --git a/packages/cloudflare/src/websocket.ts b/packages/cloudflare/src/websocket.ts new file mode 100644 index 0000000..83e0537 --- /dev/null +++ b/packages/cloudflare/src/websocket.ts @@ -0,0 +1,342 @@ +import { Effect, Option, Predicate } from 'effect' + +import { + attachmentFromFilter, + fanoutSubscription, + openSubscription, + subscriptionFilter, + subscriptionFromSeq, + type AttachedSubscriptionSocket, + type EventEnvelope, + type EventStore, + type SubscriptionAttachment, +} from '@looms/core' +import { + authResponse, + defaultAuthorize, + grantSubprotocol, + loadSubscriptionOpening, + readSubscriptionBounds, + runRouteInfo, + type Authorize, + type LoomsRuntime, +} from '@looms/runtime' + +const EVENTS_PATH = /^\/runs\/([^/]+)\/events\/?$/ + +export interface HibernatingSocketState { + acceptWebSocket(ws: WebSocket, tags?: string[]): void + getWebSockets?(tag?: string): readonly WebSocket[] + setWebSocketAutoResponse?(pair: WebSocketRequestResponsePair): void +} + +export interface OpenRunSocketOptions { + readonly ctx: HibernatingSocketState + readonly runtime: LoomsRuntime + readonly store: EventStore + readonly request: Request + readonly runId: string + readonly authorize?: Authorize + readonly createPair?: () => { readonly client: WebSocket; readonly server: WebSocket } | undefined +} + +const autoResponseInstalled = new WeakSet() + +export function isRunSocketUpgrade(request: Request): boolean { + if (request.method !== 'GET') { + return false + } + + const upgrade = request.headers.get('upgrade') + + if (upgrade === null || upgrade.toLowerCase() !== 'websocket') { + return false + } + + return runEventsPath(new URL(request.url)) !== undefined +} + +/** + * Wakes on append, not on a socket subscription, so a hibernated object can sleep + * while its WebSockets stay accepted. + */ +export function attachSocketFanout( + ctx: HibernatingSocketState, + store: EventStore, + runId: string, +): EventStore { + return tapAppend(store, (events) => + readSubscriptionBounds(store, runId).pipe( + Effect.map((bounds) => { + fanoutSubscription(acceptedSockets(ctx, runId), events, { head: bounds.head }) + }), + Effect.orElseSucceed(() => { + fanoutSubscription(acceptedSockets(ctx, runId), events, { head: 1 }) + }), + ), + ) +} + +export function openRunSocket(options: OpenRunSocketOptions): Effect.Effect { + const url = new URL(options.request.url) + const requested = runEventsPath(url) + + if (requested !== options.runId) { + return Effect.succeed(Response.json({ error: 'Run not found' }, { status: 404 })) + } + + const authorize = options.authorize ?? defaultAuthorize + + return Effect.gen(function* () { + const route = yield* runRouteInfo(options.runtime, options.store, { + access: 'read', + name: 'runs.events', + runId: options.runId, + }).pipe(Effect.option) + + if (Option.isNone(route)) { + return Response.json({ error: 'Event log is unavailable' }, { status: 500 }) + } + + const decision = yield* authorize(options.request, route.value) + + const denied = authResponse(decision) + + if (denied) { + return denied + } + + const cursor = subscriptionFromSeq(url, options.request.headers.get('last-event-id')) + + if (!cursor.ok) { + return Response.json({ error: cursor.message }, { status: 400 }) + } + + const createPair = options.createPair ?? createWebSocketPair + const pair = createPair() + + if (!pair) { + return new Response('WebSocket upgrade is not available', { status: 426 }) + } + + installAutoResponse(options.ctx) + options.ctx.acceptWebSocket(pair.server, [options.runId]) + + const filter = subscriptionFilter(url) + const attachment = attachmentFromFilter(filter, cursor.fromSeq, null) + const socket = wrapSocket(pair.server) + socket.setAttachment(attachment) + + const loaded = yield* loadSubscriptionOpening( + options.store, + options.runId, + cursor.fromSeq, + ).pipe(Effect.option) + + if (Option.isNone(loaded)) { + return Response.json({ error: 'Event log is unavailable' }, { status: 500 }) + } + + const current = socket.attachment() ?? attachment + const next = openSubscription(socket, current, loaded.value) + + if (next) { + socket.setAttachment(next) + } + + const protocol = grantSubprotocol(options.request) + + return new Response(null, { + status: 101, + webSocket: pair.client, + headers: protocol ? { 'sec-websocket-protocol': protocol } : undefined, + }) + }) +} + +function tapAppend( + store: EventStore, + onAppend: (events: readonly EventEnvelope[]) => Effect.Effect, +): EventStore { + return { + ...store, + append: (runId, events, appendOptions) => + Effect.gen(function* () { + const result = yield* store.append(runId, events, appendOptions) + const first = result.sequences[0] + + if (first === undefined) { + return result + } + + yield* Effect.gen(function* () { + const appended = yield* store.read(runId, { + fromSeq: first, + limit: result.sequences.length, + }) + + if (appended.length === 0) { + return + } + + yield* onAppend(appended) + }).pipe(Effect.ignore) + + return result + }), + } +} + +function acceptedSockets( + ctx: HibernatingSocketState, + runId: string, +): readonly AttachedSubscriptionSocket[] { + if (!ctx.getWebSockets) { + return [] + } + + return ctx.getWebSockets(runId).map(wrapSocket) +} + +function wrapSocket(ws: WebSocket): AttachedSubscriptionSocket { + return { + send: (data) => { + ws.send(data) + }, + close: (code, reason) => { + ws.close(code, reason) + }, + get bufferedAmount() { + return Predicate.isNumber(ws.bufferedAmount) ? ws.bufferedAmount : 0 + }, + attachment: () => readAttachment(ws), + setAttachment: (attachment) => { + writeAttachment(ws, attachment) + }, + } +} + +function readAttachment(ws: WebSocket): SubscriptionAttachment | undefined { + if (!('deserializeAttachment' in ws)) { + return undefined + } + + // oxlint-disable-next-line typescript/unbound-method -- hibernation methods are own properties of the platform socket + const read: unknown = ws.deserializeAttachment + + if (!Predicate.isFunction(read)) { + return undefined + } + + const value: unknown = read.call(ws) + + if (!Predicate.isObject(value) || !('cursor' in value) || !Predicate.isNumber(value.cursor)) { + return undefined + } + + const principal = + 'principal' in value && (value.principal === null || Predicate.isString(value.principal)) + ? value.principal + : null + + const threadId = 'threadId' in value && Predicate.isString(value.threadId) ? value.threadId : null + + const includeEphemeral = !('includeEphemeral' in value) || value.includeEphemeral !== false + + return { + principal, + cursor: value.cursor, + threadId, + includeEphemeral, + } +} + +function writeAttachment(ws: WebSocket, attachment: SubscriptionAttachment): void { + if (!('serializeAttachment' in ws)) { + return + } + + // oxlint-disable-next-line typescript/unbound-method -- hibernation methods are own properties of the platform socket + const write: unknown = ws.serializeAttachment + + if (!Predicate.isFunction(write)) { + return + } + + write.call(ws, attachment) +} + +function installAutoResponse(ctx: HibernatingSocketState): void { + if (autoResponseInstalled.has(ctx) || !ctx.setWebSocketAutoResponse) { + return + } + + try { + ctx.setWebSocketAutoResponse(new WebSocketRequestResponsePair('ping', 'pong')) + autoResponseInstalled.add(ctx) + } catch { + // Hosts without the hibernation helper still accept the socket. + } +} + +interface WorkersSocketPair { + readonly 0: WebSocket + readonly 1: WebSocket +} + +interface WorkersGlobals { + readonly WebSocketPair?: new () => WorkersSocketPair +} + +function createWebSocketPair(): { client: WebSocket; server: WebSocket } | undefined { + // SAFETY: Workers adds WebSocketPair to the global object. Other hosts leave it absent. + // oxlint-disable-next-line typescript/no-unsafe-type-assertion + const candidate = (globalThis as WorkersGlobals).WebSocketPair + + if (!Predicate.isFunction(candidate)) { + return undefined + } + + let pair: unknown + + try { + pair = Reflect.construct(candidate, []) + } catch { + return undefined + } + + if (!isSocketPair(pair)) { + return undefined + } + + return { client: pair[0], server: pair[1] } +} + +function isSocket(value: unknown): value is WebSocket { + return ( + Predicate.isObject(value) && + 'send' in value && + Predicate.isFunction(value.send) && + 'close' in value && + Predicate.isFunction(value.close) + ) +} + +function isSocketPair(value: unknown): value is { 0: WebSocket; 1: WebSocket } { + return Predicate.isObject(value) && isSocket(value[0]) && isSocket(value[1]) +} + +function runEventsPath(url: URL): string | undefined { + const match = EVENTS_PATH.exec(url.pathname) + const raw = match?.[1] + + if (!raw) { + return undefined + } + + try { + return decodeURIComponent(raw) + } catch { + return undefined + } +} diff --git a/packages/core/src/blob-offload.test.ts b/packages/core/src/blob-offload.test.ts new file mode 100644 index 0000000..8a2090a --- /dev/null +++ b/packages/core/src/blob-offload.test.ts @@ -0,0 +1,257 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect, Option, Stream } from 'effect' + +import { withBlobOffload } from './blob-offload' +import { BlobNotFoundError, memoryBlobStore } from './blob-store' +import { collectBlobDigests, expandBlobDigests, isBlobRef } from './blobs' +import { createEvent } from './envelope' +import { makeMemorySnapshotStore, snapshotStoreOf } from './snapshot-store' +import { emptyRunState } from './state' +import { makeMemoryEventStore } from './store' +import { isJsonObject, type JsonValue } from './types' + +function started(runId: string, payload: JsonValue) { + return createEvent(runId, { + type: 'runtime.run.started', + payload, + threadId: null, + origin: { type: 'system' }, + }) +} + +function firstDigest(value: JsonValue): string | undefined { + if (isBlobRef(value)) { + return value.sha256 + } + + if (Array.isArray(value)) { + for (const item of value) { + const found = firstDigest(item) + + if (found) { + return found + } + } + + return undefined + } + + if (isJsonObject(value)) { + for (const child of Object.values(value)) { + const found = firstDigest(child) + + if (found) { + return found + } + } + } + + return undefined +} + +describe('blob offload', () => { + test('keeps small JSON inline and resolves values over the threshold', async () => { + const inner = await Effect.runPromise(makeMemoryEventStore) + const blobs = memoryBlobStore() + const store = withBlobOffload(inner, blobs, { thresholdBytes: 32 }) + const large = 'x'.repeat(80) + + await Effect.runPromise(store.append('run_small', [started('run_small', { n: 1 })])) + + await Effect.runPromise(store.append('run_large', [started('run_large', { blob: large })])) + + const small = await Effect.runPromise(inner.read('run_small')) + expect(small[0]?.payload).toEqual({ n: 1 }) + expect(JSON.stringify(small)).not.toContain('"$looms"') + + const raw = await Effect.runPromise(inner.read('run_large')) + expect(JSON.stringify(raw)).toContain('"$looms":"blob"') + expect(JSON.stringify(raw)).not.toContain(large) + + const resolved = await Effect.runPromise(store.read('run_large')) + expect(resolved[0]?.payload).toEqual({ blob: large }) + + const live = await Effect.runPromise( + store.subscribe('run_large', { fromSeq: 1 }).pipe(Stream.take(1), Stream.runCollect), + ) + + expect(live[0]?.payload).toEqual({ blob: large }) + }) + + test('stores one body for the same bytes and deletes it only after every run releases it', async () => { + const inner = await Effect.runPromise(makeMemoryEventStore) + const blobs = memoryBlobStore() + const store = withBlobOffload(inner, blobs, { thresholdBytes: 32 }) + const payload = { blob: 'x'.repeat(80) } + + await Effect.runPromise(store.append('run_a', [started('run_a', payload)])) + await Effect.runPromise(store.append('run_b', [started('run_b', payload)])) + + const raw = await Effect.runPromise(inner.read('run_a')) + const digest = firstDigest(raw[0]?.payload ?? null) + + expect(digest).toBeDefined() + + await Effect.runPromise(store.trim!('run_a', 10)) + + const surviving = await Effect.runPromise(store.read('run_b')) + expect(surviving[0]?.payload).toEqual(payload) + + await Effect.runPromise(store.trim!('run_b', 10)) + + const missing = await Effect.runPromise( + blobs.get(digest!).pipe( + Effect.match({ + onFailure: (error) => error, + onSuccess: () => null, + }), + ), + ) + + expect(missing).toBeInstanceOf(BlobNotFoundError) + }) + + test('deletes nested bodies when the parent reference is trimmed', async () => { + const inner = await Effect.runPromise(makeMemoryEventStore) + const blobs = memoryBlobStore() + const store = withBlobOffload(inner, blobs, { thresholdBytes: 32, maxEventBytes: 200 }) + const text = 'y'.repeat(80) + const payload = [text, text, text, text] + + await Effect.runPromise(store.append('run_nested', [started('run_nested', payload)])) + + const raw = await Effect.runPromise(inner.read('run_nested')) + const seeds = new Set() + collectBlobDigests(raw[0]?.payload ?? null, seeds) + + const expanded = await Effect.runPromise(expandBlobDigests(blobs, seeds)) + expect(expanded.size).toBeGreaterThan(1) + expect(isBlobRef(raw[0]?.payload ?? null)).toBe(true) + + const resolved = await Effect.runPromise(store.read('run_nested')) + expect(resolved[0]?.payload).toEqual(payload) + + await Effect.runPromise(store.trim!('run_nested', 10)) + + for (const digest of expanded) { + const missing = await Effect.runPromise( + blobs.get(digest).pipe( + Effect.match({ + onFailure: (error) => error, + onSuccess: () => null, + }), + ), + ) + + expect(missing).toBeInstanceOf(BlobNotFoundError) + } + }) + + test('fails a read that names a missing blob and ignores a lookalike object', async () => { + const inner = await Effect.runPromise(makeMemoryEventStore) + const blobs = memoryBlobStore() + const store = withBlobOffload(inner, blobs, { thresholdBytes: 32 }) + + const lookalike = { + $looms: 'blob', + sha256: 'b'.repeat(64), + bytes: 2, + mediaType: 'application/json', + extra: true, + } + + await Effect.runPromise( + inner.append('run_missing', [ + started('run_missing', { + $looms: 'blob', + sha256: 'a'.repeat(64), + bytes: 2, + mediaType: 'application/json', + }), + ]), + ) + + await Effect.runPromise(store.append('run_lookalike', [started('run_lookalike', lookalike)])) + + const missing = await Effect.runPromise( + store.read('run_missing').pipe( + Effect.match({ + onFailure: (error) => error, + onSuccess: () => null, + }), + ), + ) + + expect(missing).toBeInstanceOf(Error) + expect(String(missing)).toContain('was not found') + + const kept = await Effect.runPromise(store.read('run_lookalike')) + expect(kept[0]?.payload).toEqual(lookalike) + }) + + test('offloads snapshot state and keeps those blobs when the log is trimmed', async () => { + const inner = await Effect.runPromise(makeMemoryEventStore) + const snapshots = await Effect.runPromise(makeMemorySnapshotStore) + const blobs = memoryBlobStore() + const store = withBlobOffload(inner, blobs, { snapshots, thresholdBytes: 32 }) + const saved = snapshotStoreOf(store) + const large = 'z'.repeat(200) + + const state = { + ...emptyRunState('run_snap'), + startIdentity: { + kind: 'workflow', + definitionName: 'blob', + definitionVersion: 'v1', + input: { blob: large }, + requestedThreadId: null, + idempotencyKey: null, + rootThreadId: 'thr_root', + launchedBy: null, + correlationId: null, + }, + } + + expect(saved).toBeDefined() + + await Effect.runPromise( + saved!.save({ + runId: 'run_snap', + cursor: 1, + stateHash: 'hash', + takenAt: 1, + state, + }), + ) + + const raw = await Effect.runPromise(snapshots.loadLatest('run_snap')) + expect(Option.isSome(raw)).toBe(true) + + if (Option.isSome(raw)) { + const encoded = JSON.stringify(raw.value.state) + expect(encoded).toContain('"$looms":"blob"') + expect(encoded).not.toContain(large) + } + + const loaded = await Effect.runPromise(saved!.loadLatest('run_snap')) + expect(Option.isSome(loaded)).toBe(true) + + if (Option.isSome(loaded)) { + expect(loaded.value.state.startIdentity?.input).toEqual({ blob: large }) + } + + await Effect.runPromise( + store.append('run_snap', [started('run_snap', { other: 'q'.repeat(80) })]), + ) + + await Effect.runPromise(store.trim!('run_snap', 10)) + + const afterTrim = await Effect.runPromise(saved!.loadLatest('run_snap')) + expect(Option.isSome(afterTrim)).toBe(true) + + if (Option.isSome(afterTrim)) { + expect(afterTrim.value.state.startIdentity?.input).toEqual({ blob: large }) + } + }) +}) diff --git a/packages/core/src/blob-offload.ts b/packages/core/src/blob-offload.ts new file mode 100644 index 0000000..680da6b --- /dev/null +++ b/packages/core/src/blob-offload.ts @@ -0,0 +1,282 @@ +import { Effect, Option, Schema, Stream } from 'effect' + +import { BlobStoreError, type BlobStore, type BlobStoreFailure } from './blob-store' +import { + DEFAULT_BLOB_THRESHOLD_BYTES, + claimJson, + collectBlobDigests, + expandBlobDigests, + offloadJson, + resolveJson, +} from './blobs' +import type { AppendableEvent, EventEnvelope } from './envelope' +import { DEFAULT_MAX_EVENT_BYTES, utf8JsonBytes } from './limits' +import { SnapshotStoreError, withSnapshotStore, type SnapshotStore } from './snapshot-store' +import { RunStateSchema, type RunState } from './state' +import { EventStoreError, type EventStore } from './store' +import type { JsonValue } from './types' + +const decodeJsonText = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Json)) +const decodeRunState = Schema.decodeUnknownEffect(RunStateSchema) + +export interface BlobOffloadOptions { + /** Snapshot store to offload and resolve. Trim keeps digests referenced by its latest snapshot. */ + readonly snapshots?: SnapshotStore + /** UTF-8 JSON size above which a string is stored by hash. Default 64 KiB. */ + readonly thresholdBytes?: number + /** Serialized event size that claims the whole payload. Default 1 MiB. */ + readonly maxEventBytes?: number +} + +/** + * Store oversized JSON outside the event log. + * + * Append replaces values over the threshold with a content-addressed reference before the + * event byte limit runs. `read`, `readStream`, and `subscribe` resolve those references, so + * folds, projections, and callers see the original values. The stored log keeps the references. + * When `snapshots` is set, snapshot save and load do the same, and trim deletes a body only + * once no retained event and no latest snapshot in any run still needs it. + */ +export function withBlobOffload( + store: EventStore, + blobs: BlobStore, + options?: BlobOffloadOptions, +): EventStore { + const thresholdBytes = options?.thresholdBytes ?? DEFAULT_BLOB_THRESHOLD_BYTES + const maxEventBytes = options?.maxEventBytes ?? DEFAULT_MAX_EVENT_BYTES + const snapshots = options?.snapshots + + const wrappedSnapshots = snapshots + ? wrapSnapshots(snapshots, store, blobs, thresholdBytes) + : undefined + + const release = (runId: string): Effect.Effect => + keepDigests(store, snapshots, blobs, runId).pipe( + Effect.flatMap((keep) => blobs.replaceRefs(runId, keep)), + Effect.mapError(asEventError), + ) + + const wrapped: EventStore = { + append: (runId, events, appendOptions) => + Effect.gen(function* () { + const stored = yield* Effect.forEach(events, (event) => + offloadAppendable(runId, event, blobs, thresholdBytes, maxEventBytes), + ).pipe(Effect.mapError(asEventError)) + + const seeds = digestsOf(stored.map((event) => event.payload)) + + const expanded = yield* expandBlobDigests(blobs, seeds).pipe(Effect.mapError(asEventError)) + + yield* blobs.retain(runId, [...expanded]).pipe(Effect.mapError(asEventError)) + + return yield* store.append(runId, stored, appendOptions) + }), + + read: (runId, readOptions) => + store + .read(runId, readOptions) + .pipe( + Effect.flatMap((events) => + Effect.forEach(events, (event) => + resolveEnvelope(event, blobs).pipe(Effect.mapError(asEventError)), + ), + ), + ), + + readStream: store.readStream + ? (runId, readOptions) => + store.readStream!(runId, readOptions).pipe( + Stream.mapEffect((event) => + resolveEnvelope(event, blobs).pipe(Effect.mapError(asEventError)), + ), + ) + : undefined, + + tail: (runId) => store.tail(runId), + + bounds: store.bounds ? (runId) => store.bounds!(runId) : undefined, + + trim: store.trim + ? (runId, beforeSeq) => store.trim!(runId, beforeSeq).pipe(Effect.andThen(release(runId))) + : undefined, + + subscribe: (runId, subscribeOptions) => + store + .subscribe(runId, subscribeOptions) + .pipe( + Stream.mapEffect((event) => + resolveEnvelope(event, blobs).pipe(Effect.mapError(asEventError)), + ), + ), + + listRuns: store.listRuns, + } + + if (!wrappedSnapshots) { + return wrapped + } + + return withSnapshotStore(wrapped, wrappedSnapshots) +} + +function offloadAppendable( + runId: string, + event: AppendableEvent, + blobs: BlobStore, + thresholdBytes: number, + maxEventBytes: number, +): Effect.Effect { + return Effect.gen(function* () { + const payload = yield* offloadJson(event.payload, blobs, { thresholdBytes }) + const offloaded = { ...event, payload } + + if (eventBytes(runId, offloaded) <= maxEventBytes) { + return offloaded + } + + const ref = yield* claimJson(payload, blobs) + + return { ...event, payload: ref } + }) +} + +function eventBytes(runId: string, event: AppendableEvent): number { + return utf8JsonBytes({ ...event, runId, seq: event.seq ?? 0 }) +} + +function resolveEnvelope( + event: EventEnvelope, + blobs: BlobStore, +): Effect.Effect { + return resolveJson(event.payload, blobs).pipe(Effect.map((payload) => ({ ...event, payload }))) +} + +function digestsOf(values: ReadonlyArray): Set { + const seeds = new Set() + + for (const value of values) { + collectBlobDigests(value, seeds) + } + + return seeds +} + +function keepDigests( + store: EventStore, + snapshots: SnapshotStore | undefined, + blobs: BlobStore, + runId: string, +): Effect.Effect, EventStoreError | BlobStoreFailure> { + return Effect.gen(function* () { + const events = yield* store.read(runId) + const seeds = digestsOf(events.map((event) => event.payload)) + + if (snapshots) { + const latest = yield* snapshots + .loadLatest(runId) + .pipe(Effect.mapError((cause) => new EventStoreError(cause.message, cause))) + + if (Option.isSome(latest)) { + const encoded = yield* jsonValue(latest.value.state).pipe( + Effect.mapError((cause) => new EventStoreError(cause.message, cause)), + ) + + collectBlobDigests(encoded, seeds) + } + } + + return yield* expandBlobDigests(blobs, seeds) + }) +} + +function wrapSnapshots( + snapshots: SnapshotStore, + store: EventStore, + blobs: BlobStore, + thresholdBytes: number, +): SnapshotStore { + const release = (runId: string): Effect.Effect => + keepDigests(store, snapshots, blobs, runId).pipe( + Effect.flatMap((keep) => blobs.replaceRefs(runId, keep)), + Effect.mapError((cause) => new SnapshotStoreError(cause.message, cause)), + ) + + return { + loadLatest: (runId) => + Effect.gen(function* () { + const loaded = yield* snapshots.loadLatest(runId) + + if (Option.isNone(loaded)) { + return loaded + } + + const encoded = yield* jsonValue(loaded.value.state).pipe(Effect.mapError(asSnapshotError)) + const resolved = yield* resolveJson(encoded, blobs).pipe(Effect.mapError(asSnapshotError)) + + const state = yield* decodeRunState(resolved).pipe( + Effect.mapError( + (cause) => new SnapshotStoreError('snapshot state did not match RunState', cause), + ), + ) + + return Option.some({ ...loaded.value, state }) + }), + + save: (snapshot) => + Effect.gen(function* () { + const encoded = yield* jsonValue(snapshot.state).pipe(Effect.mapError(asSnapshotError)) + + const offloaded = yield* offloadJson(encoded, blobs, { + thresholdBytes, + claimRoot: false, + }).pipe(Effect.mapError(asSnapshotError)) + + const state = yield* decodeRunState(offloaded).pipe( + Effect.mapError( + (cause) => new SnapshotStoreError('snapshot state did not match RunState', cause), + ), + ) + + const seeds = digestsOf([offloaded]) + + const expanded = yield* expandBlobDigests(blobs, seeds).pipe( + Effect.mapError(asSnapshotError), + ) + + yield* blobs.retain(snapshot.runId, [...expanded]).pipe(Effect.mapError(asSnapshotError)) + + return yield* snapshots.save({ + runId: snapshot.runId, + cursor: snapshot.cursor, + stateHash: snapshot.stateHash, + takenAt: snapshot.takenAt, + state, + }) + }), + + listCursors: snapshots.listCursors ? (runId) => snapshots.listCursors!(runId) : undefined, + + prune: snapshots.prune + ? (runId, keepLatest) => + snapshots.prune!(runId, keepLatest).pipe(Effect.andThen(release(runId))) + : undefined, + + saveHeader: (header) => snapshots.saveHeader(header), + + listHeaders: snapshots.listHeaders, + } +} + +function jsonValue(state: RunState): Effect.Effect { + return decodeJsonText(JSON.stringify(state)).pipe( + Effect.mapError((cause) => new BlobStoreError('snapshot state is not JSON', cause)), + ) +} + +function asEventError(cause: BlobStoreFailure | EventStoreError): EventStoreError { + return new EventStoreError(cause.message, cause) +} + +function asSnapshotError(cause: BlobStoreFailure | EventStoreError): SnapshotStoreError { + return new SnapshotStoreError(cause.message, cause) +} diff --git a/packages/core/src/blob-store.test.ts b/packages/core/src/blob-store.test.ts new file mode 100644 index 0000000..f18eedd --- /dev/null +++ b/packages/core/src/blob-store.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { blobStoreFromBytes, sha256Hex, type BlobBytes } from './blob-store' + +interface Recorded { + readonly bytes: BlobBytes + readonly objects: Map + readonly reads: string[] + onRemove?: (key: string) => void +} + +function recordedBytes(): Recorded { + const objects = new Map() + const reads: string[] = [] + + const recorded: Recorded = { + objects, + reads, + bytes: { + read: (key) => + Effect.sync(() => { + reads.push(key) + return objects.get(key) ?? null + }), + write: (key, body) => + Effect.sync(() => { + objects.set(key, body) + }), + remove: (key) => + Effect.sync(() => { + objects.delete(key) + recorded.onRemove?.(key) + }), + exists: (key) => Effect.sync(() => objects.has(key)), + list: (prefix) => + Effect.sync(() => [...objects.keys()].filter((key) => key.startsWith(prefix))), + }, + } + + return recorded +} + +const body = new TextEncoder().encode('"a large string"') +const digest = await Effect.runPromise(sha256Hex(body)) +const objectKey = `objects/${digest}` + +describe('blob references', () => { + test('a shared body survives until the last run drops it', async () => { + const recorded = recordedBytes() + const blobs = blobStoreFromBytes(recorded.bytes) + + await Effect.runPromise(blobs.put(digest, body)) + await Effect.runPromise(blobs.retain('run-a', [digest])) + await Effect.runPromise(blobs.retain('run-b', [digest])) + await Effect.runPromise(blobs.replaceRefs('run-a', new Set())) + + expect(recorded.objects.has(objectKey)).toBe(true) + + await Effect.runPromise(blobs.replaceRefs('run-b', new Set())) + + expect([...recorded.objects.keys()].filter((key) => key.includes(digest))).toEqual([]) + }) + + test('dropping a reference never reads another run', async () => { + const recorded = recordedBytes() + const blobs = blobStoreFromBytes(recorded.bytes) + + await Effect.runPromise(blobs.put(digest, body)) + + for (const runId of ['run-a', 'run-b', 'run-c', 'run-d']) { + await Effect.runPromise(blobs.retain(runId, [digest])) + } + + recorded.reads.length = 0 + await Effect.runPromise(blobs.replaceRefs('run-a', new Set())) + + expect(recorded.reads).toEqual(['runs/run-a']) + }) + + test('a reference recorded after the body was deleted fails so the append can retry', async () => { + const recorded = recordedBytes() + const blobs = blobStoreFromBytes(recorded.bytes) + + await Effect.runPromise(blobs.put(digest, body)) + await Effect.runPromise(blobs.retain('run-a', [digest])) + await Effect.runPromise(blobs.put(digest, body)) + await Effect.runPromise(blobs.replaceRefs('run-a', new Set())) + + const failed = await Effect.runPromise(Effect.flip(blobs.retain('run-b', [digest]))) + + expect(failed.message).toContain('retry the append') + + await Effect.runPromise(blobs.put(digest, body)) + await Effect.runPromise(blobs.retain('run-b', [digest])) + + expect(recorded.objects.has(objectKey)).toBe(true) + }) + + test('a body deleted while another run references it is put back', async () => { + const recorded = recordedBytes() + const blobs = blobStoreFromBytes(recorded.bytes) + + await Effect.runPromise(blobs.put(digest, body)) + await Effect.runPromise(blobs.retain('run-a', [digest])) + + recorded.onRemove = (key) => { + if (key === objectKey) { + recorded.objects.set(`refs/${digest}/run-b`, new Uint8Array(0)) + } + } + + await Effect.runPromise(blobs.replaceRefs('run-a', new Set())) + + expect(recorded.objects.get(objectKey)).toEqual(body) + }) +}) diff --git a/packages/core/src/blob-store.ts b/packages/core/src/blob-store.ts new file mode 100644 index 0000000..f9c54ec --- /dev/null +++ b/packages/core/src/blob-store.ts @@ -0,0 +1,294 @@ +import { Data, Effect, Schema } from 'effect' + +const SHA256 = /^[a-f0-9]{64}$/ + +const decodeDigestList = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Array(Schema.String)), +) + +export class BlobStoreError extends Data.TaggedError('BlobStoreError')<{ + readonly message: string + readonly cause?: unknown +}> { + override readonly cause?: unknown + + constructor(message: string, cause?: unknown) { + super({ message, cause }) + this.name = 'BlobStoreError' + this.cause = cause + } +} + +export class BlobNotFoundError extends Data.TaggedError('BlobNotFoundError')<{ + readonly digest: string + readonly message: string +}> { + constructor(digest: string) { + super({ digest, message: `blob ${digest} was not found` }) + this.name = 'BlobNotFoundError' + } +} + +export type BlobStoreFailure = BlobStoreError | BlobNotFoundError + +/** + * Content-addressed bytes. `put` is idempotent: the same digest stores one body. + * `retain` and `replaceRefs` record which digests a run still needs. A body is deleted only + * when no run references it. Many runs may share one store, each writing its own references. + */ +export interface BlobStore { + readonly put: (digest: string, body: Uint8Array) => Effect.Effect + readonly get: (digest: string) => Effect.Effect + readonly retain: ( + runId: string, + digests: ReadonlyArray, + ) => Effect.Effect + readonly replaceRefs: ( + runId: string, + keep: ReadonlySet, + ) => Effect.Effect +} + +export function isSha256Digest(value: string): boolean { + return SHA256.test(value) +} + +/** Byte operations behind {@link blobStoreFromBytes}. Keys are store-relative, not run ids. */ +export interface BlobBytes { + readonly read: (key: string) => Effect.Effect + readonly write: (key: string, body: Uint8Array) => Effect.Effect + readonly remove: (key: string) => Effect.Effect + readonly exists: (key: string) => Effect.Effect + readonly list: (prefix: string) => Effect.Effect, BlobStoreError> +} + +const EMPTY = new Uint8Array(0) + +export function memoryBlobStore(): BlobStore { + return blobStoreFromBytes(memoryBlobBytes()) +} + +export function blobStoreFromBytes(bytes: BlobBytes): BlobStore { + return { + put: (digest, body) => + Effect.gen(function* () { + yield* assertDigest(digest) + + const actual = yield* sha256Hex(body) + + if (actual !== digest) { + return yield* new BlobStoreError(`blob body does not match digest ${digest}`) + } + + const existing = yield* bytes.read(objectKey(digest)) + + if (existing) { + return yield* Effect.void + } + + return yield* bytes.write(objectKey(digest), body) + }), + + get: (digest) => + Effect.gen(function* () { + yield* assertDigest(digest) + + const body = yield* bytes.read(objectKey(digest)) + + if (!body) { + return yield* new BlobNotFoundError(digest) + } + + const actual = yield* sha256Hex(body) + + if (actual !== digest) { + return yield* new BlobStoreError(`blob ${digest} failed its integrity check`) + } + + return body + }), + + // Markers go down before the existence check, and `collect` lists markers again after + // deleting. Whichever side runs second sees the other, so a referenced body is never lost: + // either `collect` restores it or this append fails and its retry uploads the body again. + retain: (runId, digests) => + Effect.gen(function* () { + const current = new Set(yield* readRunDigests(bytes, runId)) + const added = [...new Set(digests)].filter((digest) => !current.has(digest)) + + if (added.length === 0) { + return yield* Effect.void + } + + yield* Effect.forEach(added, assertDigest, { discard: true }) + + yield* Effect.forEach(added, (digest) => bytes.write(refKey(digest, runId), EMPTY), { + concurrency: REF_CONCURRENCY, + discard: true, + }) + + yield* writeRunDigests(bytes, runId, [...current, ...added]) + + const present = yield* Effect.forEach(added, (digest) => bytes.exists(objectKey(digest)), { + concurrency: REF_CONCURRENCY, + }) + + const missing = added.filter((_, index) => !present[index]) + + if (missing.length > 0) { + return yield* new BlobStoreError( + `blobs ${missing.join(', ')} were deleted before run ${runId} referenced them; retry the append`, + ) + } + + return yield* Effect.void + }), + + replaceRefs: (runId, keep) => + Effect.gen(function* () { + const previous = yield* readRunDigests(bytes, runId) + const dropped = previous.filter((digest) => !keep.has(digest)) + + if (dropped.length === 0) { + return + } + + yield* Effect.forEach( + dropped, + (digest) => + bytes.remove(refKey(digest, runId)).pipe(Effect.andThen(collect(bytes, digest))), + { concurrency: REF_CONCURRENCY, discard: true }, + ) + + yield* writeRunDigests( + bytes, + runId, + previous.filter((digest) => keep.has(digest)), + ) + }), + } +} + +const REF_CONCURRENCY = 8 + +/** Deletes a body no run references, and puts it back if a run referenced it meanwhile. */ +function collect(bytes: BlobBytes, digest: string): Effect.Effect { + const referenced = bytes.list(refPrefix(digest)).pipe(Effect.map((keys) => keys.length > 0)) + + return Effect.gen(function* () { + if (yield* referenced) { + return + } + + const body = yield* bytes.read(objectKey(digest)) + + if (!body) { + return + } + + yield* bytes.remove(objectKey(digest)) + + if (yield* referenced) { + yield* bytes.write(objectKey(digest), body) + } + }) +} + +function memoryBlobBytes(): BlobBytes { + const objects = new Map() + + return { + read: (key) => + Effect.sync(() => { + const body = objects.get(key) + + return body ? body.slice() : null + }), + + write: (key, body) => + Effect.sync(() => { + objects.set(key, body.slice()) + }), + + remove: (key) => + Effect.sync(() => { + objects.delete(key) + }), + + exists: (key) => Effect.sync(() => objects.has(key)), + + list: (prefix) => + Effect.sync(() => [...objects.keys()].filter((key) => key.startsWith(prefix))), + } +} + +function objectKey(digest: string): string { + return `objects/${digest}` +} + +function runKey(runId: string): string { + return `runs/${encodeURIComponent(runId)}` +} + +function refPrefix(digest: string): string { + return `refs/${digest}/` +} + +function refKey(digest: string, runId: string): string { + return `${refPrefix(digest)}${encodeURIComponent(runId)}` +} + +function assertDigest(digest: string): Effect.Effect { + if (isSha256Digest(digest)) { + return Effect.void + } + + return Effect.fail(new BlobStoreError(`invalid blob digest ${digest}`)) +} + +export function sha256Hex(body: Uint8Array): Effect.Effect { + const copy = new Uint8Array(body.byteLength) + copy.set(body) + + return Effect.tryPromise({ + try: () => crypto.subtle.digest('SHA-256', copy), + catch: (cause) => new BlobStoreError('SHA-256 failed', cause), + }).pipe(Effect.map((buffer) => hex(new Uint8Array(buffer)))) +} + +function hex(bytes: Uint8Array): string { + let encoded = '' + + for (const byte of bytes) { + encoded += byte.toString(16).padStart(2, '0') + } + + return encoded +} + +function readRunDigests( + bytes: BlobBytes, + runId: string, +): Effect.Effect, BlobStoreError> { + return bytes.read(runKey(runId)).pipe( + Effect.flatMap((body) => { + if (!body) { + return Effect.succeed([]) + } + + return decodeDigestList(new TextDecoder().decode(body)).pipe( + Effect.mapError( + (cause) => new BlobStoreError(`blob references for ${runId} are unreadable`, cause), + ), + ) + }), + ) +} + +function writeRunDigests( + bytes: BlobBytes, + runId: string, + digests: ReadonlyArray, +): Effect.Effect { + return bytes.write(runKey(runId), new TextEncoder().encode(JSON.stringify(digests))) +} diff --git a/packages/core/src/blobs.ts b/packages/core/src/blobs.ts new file mode 100644 index 0000000..1558950 --- /dev/null +++ b/packages/core/src/blobs.ts @@ -0,0 +1,283 @@ +import { Effect, Predicate, Schema } from 'effect' + +import { + BlobStoreError, + isSha256Digest, + sha256Hex, + type BlobStore, + type BlobStoreFailure, +} from './blob-store' +import { utf8JsonBytes } from './limits' +import { isJsonObject, type JsonValue } from './types' + +export const DEFAULT_BLOB_THRESHOLD_BYTES = 64 * 1024 + +const BLOB_KIND = 'blob' +const BLOB_MEDIA_TYPE = 'application/json' +const decodeJsonText = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Json)) + +/** + * Reserved stored shape. A JSON object with exactly these fields is a claim check, + * not caller data. `bytes` is the UTF-8 length of the JSON body named by `sha256`. + */ +export interface BlobRef { + readonly $looms: typeof BLOB_KIND + readonly sha256: string + readonly bytes: number + readonly mediaType: typeof BLOB_MEDIA_TYPE + readonly [key: string]: JsonValue +} + +export function isBlobRef(value: JsonValue): value is BlobRef { + if (!isJsonObject(value) || Object.keys(value).length !== 4) { + return false + } + + const kind = value.$looms + const sha256 = value.sha256 + const bytes = value.bytes + const mediaType = value.mediaType + + return ( + kind === BLOB_KIND && + Predicate.isString(sha256) && + isSha256Digest(sha256) && + Predicate.isNumber(bytes) && + Number.isInteger(bytes) && + bytes >= 0 && + mediaType === BLOB_MEDIA_TYPE + ) +} + +export interface OffloadJsonOptions { + readonly thresholdBytes?: number + /** When false, the root value stays inline even if it is over the threshold. Children are still claimed. */ + readonly claimRoot?: boolean +} + +/** Store `value` and return a reference, even when it is under the offload threshold. */ +export function claimJson( + value: JsonValue, + blobs: BlobStore, +): Effect.Effect { + return putJson(value, blobs) +} + +/** Replace values larger than `thresholdBytes` with {@link BlobRef}s. Bodies are stored by hash. */ +export function offloadJson( + value: JsonValue, + blobs: BlobStore, + options?: OffloadJsonOptions, +): Effect.Effect { + const thresholdBytes = options?.thresholdBytes ?? DEFAULT_BLOB_THRESHOLD_BYTES + + return offloadNode(value, blobs, thresholdBytes, options?.claimRoot ?? true) +} + +/** Replace {@link BlobRef}s with the JSON they name, including references nested inside a body. */ +export function resolveJson( + value: JsonValue, + blobs: BlobStore, +): Effect.Effect { + return resolveNode(value, blobs, new Set()) +} + +/** + * Digests named by `value`, plus digests nested inside those bodies. + * A parent blob that itself contains references still keeps those bodies alive. + */ +export function expandBlobDigests( + blobs: BlobStore, + seeds: ReadonlySet, +): Effect.Effect, BlobStoreFailure> { + if (seeds.size === 0) { + return Effect.succeed(new Set()) + } + + return Effect.gen(function* () { + const all = new Set() + const pending = [...seeds] + + while (pending.length > 0) { + const digest = pending.pop() + + if (digest === undefined || all.has(digest)) { + continue + } + + all.add(digest) + + const body = yield* blobs.get(digest) + const json = yield* decodeBody(body) + const nested = new Set() + + collectBlobDigests(json, nested) + + for (const child of nested) { + if (!all.has(child)) { + pending.push(child) + } + } + } + + return all + }) +} + +/** Record every blob digest embedded in `value`. Does not read blob bodies. */ +export function collectBlobDigests(value: JsonValue, into: Set): void { + if (isBlobRef(value)) { + into.add(value.sha256) + return + } + + if (Array.isArray(value)) { + for (const item of value) { + collectBlobDigests(item, into) + } + + return + } + + if (isJsonObject(value)) { + for (const child of Object.values(value)) { + collectBlobDigests(child, into) + } + } +} + +function offloadNode( + value: JsonValue, + blobs: BlobStore, + thresholdBytes: number, + claim: boolean, +): Effect.Effect { + if (isBlobRef(value)) { + return Effect.succeed(value) + } + + if (Array.isArray(value)) { + return Effect.forEach(value, (item) => offloadNode(item, blobs, thresholdBytes, true)).pipe( + Effect.flatMap((items) => storeIfOver(items, blobs, thresholdBytes, claim)), + ) + } + + if (isJsonObject(value)) { + return offloadObject(value, blobs, thresholdBytes, claim) + } + + return storeIfOver(value, blobs, thresholdBytes, claim) +} + +function offloadObject( + value: { readonly [key: string]: JsonValue }, + blobs: BlobStore, + thresholdBytes: number, + claim: boolean, +): Effect.Effect { + return Effect.gen(function* () { + const next: { [key: string]: JsonValue } = {} + + for (const key of Object.keys(value)) { + const child = value[key] + + if (child === undefined) { + continue + } + + next[key] = yield* offloadNode(child, blobs, thresholdBytes, true) + } + + return yield* storeIfOver(next, blobs, thresholdBytes, claim) + }) +} + +function storeIfOver( + value: JsonValue, + blobs: BlobStore, + thresholdBytes: number, + claim: boolean, +): Effect.Effect { + // Strings only. Objects and arrays stay in place so schema-shaped snapshot state still decodes. + // An event that is still over the byte limit claims its whole payload afterward. + if (!claim || !Predicate.isString(value) || utf8JsonBytes(value) <= thresholdBytes) { + return Effect.succeed(value) + } + + return putJson(value, blobs) +} + +function putJson(value: JsonValue, blobs: BlobStore): Effect.Effect { + const body = new TextEncoder().encode(JSON.stringify(value)) + + return sha256Hex(body).pipe( + Effect.flatMap((sha256) => + blobs.put(sha256, body).pipe( + Effect.map((): BlobRef => ({ + $looms: BLOB_KIND, + sha256, + bytes: body.byteLength, + mediaType: BLOB_MEDIA_TYPE, + })), + ), + ), + ) +} + +function resolveNode( + value: JsonValue, + blobs: BlobStore, + seen: ReadonlySet, +): Effect.Effect { + if (isBlobRef(value)) { + if (seen.has(value.sha256)) { + return Effect.fail(new BlobStoreError(`blob ${value.sha256} refers to itself`)) + } + + const next = new Set(seen) + next.add(value.sha256) + + return blobs.get(value.sha256).pipe( + Effect.flatMap((body) => decodeBody(body)), + Effect.flatMap((decoded) => resolveNode(decoded, blobs, next)), + ) + } + + if (Array.isArray(value)) { + return Effect.forEach(value, (item) => resolveNode(item, blobs, seen)) + } + + if (isJsonObject(value)) { + return resolveObject(value, blobs, seen) + } + + return Effect.succeed(value) +} + +function resolveObject( + value: { readonly [key: string]: JsonValue }, + blobs: BlobStore, + seen: ReadonlySet, +): Effect.Effect { + return Effect.gen(function* () { + const next: { [key: string]: JsonValue } = {} + + for (const key of Object.keys(value)) { + const child = value[key] + + if (child === undefined) { + continue + } + + next[key] = yield* resolveNode(child, blobs, seen) + } + + return next + }) +} + +function decodeBody(body: Uint8Array): Effect.Effect { + return decodeJsonText(new TextDecoder().decode(body)).pipe( + Effect.mapError((cause) => new BlobStoreError('blob body is not JSON', cause)), + ) +} diff --git a/packages/core/src/definition-resolver.test.ts b/packages/core/src/definition-resolver.test.ts new file mode 100644 index 0000000..dceebaa --- /dev/null +++ b/packages/core/src/definition-resolver.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { + cachedDefinitionResolver, + chainDefinitionResolvers, + pinnedDefinitionRefs, + staticDefinitionResolver, +} from './definition-resolver' +import { makeDefinitionStore, type VersionedDefinition } from './definition-store' +import { createEvent } from './envelope' + +const checkout = { + kind: 'workflow', + name: 'checkout', + version: 'v1', +} as const + +const refund = { + kind: 'workflow', + name: 'refund', + version: 'v2', +} as const + +function started(type: string, payload: Record) { + return { + ...createEvent('run_1', { + type, + payload, + origin: { type: 'system' as const }, + }), + seq: 1, + } +} + +describe('definition resolver', () => { + test('static lookup returns the registered object and misses as undefined', async () => { + const resolver = staticDefinitionResolver([checkout]) + const found = await Effect.runPromise(resolver.resolve('workflow', 'checkout', 'v1')) + + expect(found).toBe(checkout) + expect(resolver.peek('workflow', 'refund', 'v2')).toBeUndefined() + expect(await Effect.runPromise(resolver.resolve('workflow', 'refund', 'v2'))).toBeUndefined() + }) + + test('chain stops at the first definition', async () => { + let calls = 0 + + const second = { + peek: () => undefined, + resolve: () => + Effect.sync(() => { + calls += 1 + return refund + }), + } + + const resolver = chainDefinitionResolvers([staticDefinitionResolver([checkout]), second]) + const found = await Effect.runPromise(resolver.resolve('workflow', 'checkout', 'v1')) + + expect(found).toBe(checkout) + expect(calls).toBe(0) + expect(await Effect.runPromise(resolver.resolve('workflow', 'refund', 'v2'))).toBe(refund) + expect(calls).toBe(1) + }) + + test('cache returns a miss without fetching again and evicts the oldest entry', async () => { + const fetches: string[] = [] + + const inner = { + peek: () => undefined, + resolve: (kind: string, name: string, version: string) => + Effect.sync(() => { + fetches.push(`${kind}:${name}@${version}`) + + if (name === 'missing') { + return undefined + } + + return { kind, name, version } + }), + } + + const resolver = cachedDefinitionResolver(inner, { capacity: 1 }) + + expect(await Effect.runPromise(resolver.resolve('workflow', 'missing', 'v1'))).toBeUndefined() + expect(await Effect.runPromise(resolver.resolve('workflow', 'missing', 'v1'))).toBeUndefined() + expect(fetches).toEqual(['workflow:missing@v1']) + expect(resolver.peek('workflow', 'missing', 'v1')).toBeUndefined() + + await Effect.runPromise(resolver.resolve('workflow', 'checkout', 'v1')) + expect(resolver.peek('workflow', 'missing', 'v1')).toBeUndefined() + expect(resolver.peek('workflow', 'checkout', 'v1')?.name).toBe('checkout') + + await Effect.runPromise(resolver.resolve('workflow', 'missing', 'v1')) + expect(fetches).toEqual(['workflow:missing@v1', 'workflow:checkout@v1', 'workflow:missing@v1']) + }) + + test('a store peeks a primed resolver and keeps its own definitions first', async () => { + const dynamic = { kind: 'workflow', name: 'dynamic', version: 'v9', nodes: ['a'] } + + const resolver = cachedDefinitionResolver({ + peek: () => undefined, + resolve: (kind: string, name: string, version: string) => + Effect.succeed( + kind === 'workflow' && name === 'dynamic' && version === 'v9' ? dynamic : undefined, + ), + }) + + const store = makeDefinitionStore('workflow', [checkout]) + store.useResolver(resolver) + + expect(store.get('checkout', 'v1')).toBe(checkout) + expect(store.get('dynamic', 'v9')).toBeUndefined() + expect(await Effect.runPromise(store.resolve('dynamic', 'v9'))).toBe(dynamic) + expect(store.get('dynamic', 'v9')).toMatchObject({ nodes: ['a'] }) + }) + + test('pinned refs follow the start events and default a missing version', () => { + const refs = pinnedDefinitionRefs([ + started('runtime.run.started', { + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v2', + }), + started('runtime.thread.started', { + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v2', + }), + started('runtime.thread.started', { kind: 'agent', definitionName: 'helper' }), + started('workflow.node.started', { nodeId: 'a' }), + ]) + + expect(refs).toEqual([ + { kind: 'workflow', name: 'checkout', version: 'v2' }, + { kind: 'agent', name: 'helper', version: 'v1' }, + ]) + }) +}) diff --git a/packages/core/src/definition-resolver.ts b/packages/core/src/definition-resolver.ts new file mode 100644 index 0000000..5c46b08 --- /dev/null +++ b/packages/core/src/definition-resolver.ts @@ -0,0 +1,207 @@ +import { Data, Effect, Predicate } from 'effect' + +import type { VersionedDefinition } from './definition-store' +import type { EventEnvelope } from './envelope' +import { DEFAULT_DEFINITION_VERSION, definitionKey } from './module' +import type { JsonValue } from './types' + +export class DefinitionResolveError extends Data.TaggedError('DefinitionResolveError')<{ + readonly kind: string + readonly name: string + readonly version: string + readonly message: string +}> { + constructor(kind: string, name: string, version: string, message: string) { + super({ kind, name, version, message }) + this.name = 'DefinitionResolveError' + } +} + +/** + * Loads one definition by the identity pinned on a run. + * `peek` is synchronous and only returns a definition this resolver already holds. + * `resolve` may fetch, and it caches through {@link cachedDefinitionResolver}. + */ +export interface DefinitionResolver { + readonly peek: (kind: string, name: string, version: string) => VersionedDefinition | undefined + readonly resolve: ( + kind: string, + name: string, + version: string, + ) => Effect.Effect +} + +export const DEFAULT_DEFINITION_CACHE_CAPACITY = 256 + +/** Definitions registered on a module. A miss is `undefined`, not an error. */ +export function staticDefinitionResolver( + definitions: ReadonlyArray, +): DefinitionResolver { + const byKey = new Map( + definitions.map( + (definition) => + [definitionKey(definition.kind, definition.name, definition.version), definition] as const, + ), + ) + + const lookup = (kind: string, name: string, version: string) => + byKey.get(definitionKey(kind, name, version)) + + return { + peek: lookup, + resolve: (kind, name, version) => Effect.succeed(lookup(kind, name, version)), + } +} + +/** First resolver that returns a definition wins. Later resolvers are not called. */ +export function chainDefinitionResolvers( + resolvers: ReadonlyArray, +): DefinitionResolver { + return { + peek(kind, name, version) { + for (const resolver of resolvers) { + const found = resolver.peek(kind, name, version) + + if (found) { + return found + } + } + + return undefined + }, + resolve(kind, name, version) { + return Effect.gen(function* () { + for (const resolver of resolvers) { + const found = yield* resolver.resolve(kind, name, version) + + if (found) { + return found + } + } + + return undefined + }) + }, + } +} + +/** + * Remembers successful lookups and misses. A miss stays cached so recovery does not + * fetch a deleted definition on every wake. Failures are not cached. + */ +export function cachedDefinitionResolver( + inner: DefinitionResolver, + options?: { readonly capacity?: number }, +): DefinitionResolver { + const capacity = options?.capacity ?? DEFAULT_DEFINITION_CACHE_CAPACITY + const entries = new Map() + + const remember = (key: string, value: VersionedDefinition | undefined): void => { + entries.delete(key) + entries.set(key, value) + + while (entries.size > capacity) { + const oldest = entries.keys().next().value + + if (oldest === undefined) { + return + } + + entries.delete(oldest) + } + } + + return { + peek(kind, name, version) { + const key = definitionKey(kind, name, version) + + if (!entries.has(key)) { + return inner.peek(kind, name, version) + } + + const value = entries.get(key) + remember(key, value) + return value + }, + resolve(kind, name, version) { + const key = definitionKey(kind, name, version) + + if (entries.has(key)) { + const value = entries.get(key) + remember(key, value) + return Effect.succeed(value) + } + + return inner.resolve(kind, name, version).pipe( + Effect.tap((value) => + Effect.sync(() => { + remember(key, value) + }), + ), + ) + }, + } +} + +/** Identities pinned by `runtime.run.started` and `runtime.thread.started`. */ +export function pinnedDefinitionRefs( + events: readonly EventEnvelope[], +): readonly VersionedDefinition[] { + const refs: VersionedDefinition[] = [] + const seen = new Set() + + for (const event of events) { + if (event.type !== 'runtime.run.started' && event.type !== 'runtime.thread.started') { + continue + } + + const pinned = readPinned(event.payload) + + if (!pinned) { + continue + } + + const key = definitionKey(pinned.kind, pinned.name, pinned.version) + + if (seen.has(key)) { + continue + } + + seen.add(key) + refs.push(pinned) + } + + return refs +} + +/** Resolves every pinned identity so a later synchronous lookup can see it. */ +export function primePinnedDefinitions( + resolver: DefinitionResolver, + events: readonly EventEnvelope[], +): Effect.Effect { + return Effect.gen(function* () { + for (const ref of pinnedDefinitionRefs(events)) { + yield* resolver.resolve(ref.kind, ref.name, ref.version) + } + }) +} + +function readPinned(payload: JsonValue): VersionedDefinition | undefined { + if (!Predicate.isObject(payload) || !('kind' in payload) || !('definitionName' in payload)) { + return undefined + } + + const kind = payload.kind + const name = payload.definitionName + const version = 'definitionVersion' in payload ? payload.definitionVersion : undefined + + if (!Predicate.isString(kind) || !Predicate.isString(name)) { + return undefined + } + + return { + kind, + name, + version: Predicate.isString(version) ? version : DEFAULT_DEFINITION_VERSION, + } +} diff --git a/packages/core/src/definition-store.ts b/packages/core/src/definition-store.ts index c79cfe6..e617e75 100644 --- a/packages/core/src/definition-store.ts +++ b/packages/core/src/definition-store.ts @@ -1,3 +1,6 @@ +import { Effect } from 'effect' + +import type { DefinitionResolveError, DefinitionResolver } from './definition-resolver' import { definitionKey } from './module' export interface VersionedDefinition { @@ -9,6 +12,13 @@ export interface VersionedDefinition { /** Lookup of one definition kind by name and version. */ export interface DefinitionStore { get(name: string, version: string): TDefinition | undefined + /** Static definition, or a definition the bound resolver has already loaded. */ + resolve( + name: string, + version: string, + ): Effect.Effect + /** Called by the runtime before the module's services are built. */ + useResolver(resolver: DefinitionResolver): void } /** Indexes definitions of one kind by `definitionKey`. */ @@ -23,5 +33,57 @@ export function makeDefinitionStore( ), ) - return { get: (name, version) => byKey.get(definitionKey(kind, name, version)) } + let resolver: DefinitionResolver | undefined + + const get = (name: string, version: string): TDefinition | undefined => { + const stored = byKey.get(definitionKey(kind, name, version)) + + if (stored) { + return stored + } + + const peeked = resolver?.peek(kind, name, version) + + if (peeked && isStoredDefinition(kind, peeked)) { + return peeked + } + + return undefined + } + + return { + get, + useResolver(next) { + resolver = next + }, + resolve(name, version) { + const stored = get(name, version) + + if (stored) { + return Effect.succeed(stored) + } + + if (!resolver) { + // Effect.void is Effect, which is not a missing definition. + // oxlint-disable-next-line effecttsgo/effect-succeed-with-void + return Effect.succeed(undefined) + } + + return resolver + .resolve(kind, name, version) + .pipe( + Effect.map((found) => + found && isStoredDefinition(kind, found) ? found : undefined, + ), + ) + }, + } +} + +// oxlint-disable-next-line typescript/no-unnecessary-type-parameters -- the predicate returns the store's definition type +function isStoredDefinition( + kind: string, + value: VersionedDefinition, +): value is TDefinition { + return value.kind === kind } diff --git a/packages/core/src/effects.test.ts b/packages/core/src/effects.test.ts index a545e9d..8f075cf 100644 --- a/packages/core/src/effects.test.ts +++ b/packages/core/src/effects.test.ts @@ -10,6 +10,8 @@ const ctx: EffectContext = { runId: 'run_1', threadId: 'thr_1', causingEventId: 'evt_1', + attributes: {}, + secrets: {}, signal: new AbortController().signal, emit: async () => undefined, } diff --git a/packages/core/src/effects.ts b/packages/core/src/effects.ts index 57035c6..28636ff 100644 --- a/packages/core/src/effects.ts +++ b/packages/core/src/effects.ts @@ -2,12 +2,14 @@ import { Data, Effect, Predicate, Schema, type Context } from 'effect' import { EventInputSchema, type EventInput } from './envelope' import { DEFAULT_DEFINITION_VERSION } from './module' +import type { RunAttributes, SecretContext } from './run-context' import { validateInputEffect, type InferDefinedSchema, type InvalidInputError, type SchemaInput, } from './schema' +import type { TraceContext } from './trace' import { asJson, type JsonValue } from './types' export type WaitOnEvent = { @@ -60,6 +62,23 @@ export type CancelEffect = { tag?: string } +export type LaunchEffect = { + type: 'runtime.launch' + runId: string + kind: string + definitionName: string + definitionVersion: string + input: JsonValue + close: 'terminate' | 'wait' + tag?: string +} + +export type CancelRunEffect = { + type: 'runtime.cancelRun' + runId: string + tag?: string +} + export interface RetryPolicy { readonly maxAttempts: number readonly backoffMs?: number @@ -86,6 +105,8 @@ export type PrimitiveEffect = | CompleteEffect | FailEffect | CancelEffect + | LaunchEffect + | CancelRunEffect export type RuntimeEffect = PrimitiveEffect | InvokeEffect @@ -132,6 +153,25 @@ export const RuntimeEffectSchema = Schema.Union([ threadId: Schema.String, tag: Schema.optional(Schema.String), }), + Schema.Struct({ + type: Schema.Literal('runtime.launch'), + runId: Schema.String, + kind: Schema.String, + definitionName: Schema.String, + definitionVersion: Schema.String.pipe( + Schema.withDecodingDefaultKey(Effect.succeed(DEFAULT_DEFINITION_VERSION)), + ), + input: Schema.Json, + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('terminate' as const)), + ), + tag: Schema.optional(Schema.String), + }), + Schema.Struct({ + type: Schema.Literal('runtime.cancelRun'), + runId: Schema.String, + tag: Schema.optional(Schema.String), + }), Schema.Struct({ type: Schema.String, input: Schema.Json, @@ -148,6 +188,8 @@ export function isPrimitiveEffect(effect: RuntimeEffect): effect is PrimitiveEff case 'runtime.complete': case 'runtime.fail': case 'runtime.cancel': + case 'runtime.launch': + case 'runtime.cancelRun': return true default: return false @@ -187,6 +229,12 @@ export interface EffectContext { readonly threadId: string readonly causingEventId: string readonly signal: AbortSignal + /** Labels safe for every reader of the run. */ + readonly attributes: RunAttributes + /** Bearer material for this run. Never copy it into an emitted event. */ + readonly secrets: SecretContext + /** The W3C trace context of the caller that started the run, to parent spans the handler starts. */ + readonly trace: TraceContext | null emit(event: EventInput): Promise } @@ -196,6 +244,9 @@ export interface ScopedEffectContext { readonly threadId: string readonly causingEventId: string readonly signal: AbortSignal + readonly attributes: RunAttributes + readonly secrets: SecretContext + readonly trace: TraceContext | null emit(event: E): Promise } @@ -215,6 +266,52 @@ export class EffectHandlerError extends Data.TaggedError('EffectHandlerError')<{ } } +export interface EffectFailureOptions { + readonly error: string + /** Fail the attempt even when the retry policy has attempts remaining. */ + readonly nonRetryable?: boolean + /** Wait at least this long before the next attempt. Does not shorten the policy backoff. */ + readonly retryAfterMs?: number +} + +/** A handler's deliberate attempt failure, distinct from an unexpected thrown error. */ +export class EffectFailure extends Data.TaggedError('EffectFailure')<{ + readonly error: string + readonly nonRetryable: boolean + readonly retryAfterMs?: number +}> { + constructor(options: EffectFailureOptions) { + const error = options.error + const nonRetryable = options.nonRetryable === true + + if (options.retryAfterMs === undefined) { + super({ error, nonRetryable }) + } else { + super({ error, nonRetryable, retryAfterMs: options.retryAfterMs }) + } + + this.name = 'EffectFailure' + } +} + +/** Fail the current attempt. The retry policy decides whether another attempt is scheduled. */ +export function failEffect(options: EffectFailureOptions): Effect.Effect { + return Effect.fail(new EffectFailure(options)) +} + +/** Unwrap a handler failure, including one wrapped by `EffectHandlerError`. */ +export function readEffectFailure(cause: unknown): EffectFailure | null { + if (cause instanceof EffectFailure) { + return cause + } + + if (cause instanceof EffectHandlerError) { + return readEffectFailure(cause.cause) + } + + return null +} + export type EffectHandlerResult< R = never, E extends EventInput = EventInput, @@ -266,6 +363,18 @@ function liftHandlerResult( return Effect.succeed(result) } +function invokeHandler( + handler: (input: JsonValue, ctx: EffectContext) => EffectHandlerResult, + input: JsonValue, + ctx: EffectContext, +): Effect.Effect, EffectHandlerError, R> { + try { + return liftHandlerResult(handler(input, ctx)) + } catch (cause) { + return Effect.fail(new EffectHandlerError(cause)) + } +} + /** * Define an effect. With `execute`, a run's actor cell runs the handler at * most once while it is alive; if the cell dies mid-execution, the replacement @@ -327,7 +436,12 @@ export function defineEffect(def: { Effect.gen(function* () { const input = schema ? yield* validateInputEffect(schema, raw) : raw - return handler ? yield* liftHandlerResult(handler(input, ctx)) : noOutput + if (!handler) { + return noOutput + } + + // invokeHandler turns a synchronous throw into EffectHandlerError. + return yield* invokeHandler(handler, input, ctx) }), } } @@ -413,6 +527,38 @@ export function cancel(threadId: string, tag?: string): CancelEffect { : { type: 'runtime.cancel', threadId, tag } } +export function launch(args: { + runId: string + kind: string + definitionName: string + definitionVersion?: string + input: JsonValue + close?: 'terminate' | 'wait' + tag?: string +}): LaunchEffect { + const effect: LaunchEffect = { + type: 'runtime.launch', + runId: args.runId, + kind: args.kind, + definitionName: args.definitionName, + definitionVersion: args.definitionVersion ?? DEFAULT_DEFINITION_VERSION, + input: args.input, + close: args.close ?? 'terminate', + } + + if (args.tag !== undefined) { + effect.tag = args.tag + } + + return effect +} + +export function cancelRun(runId: string, tag?: string): CancelRunEffect { + return tag === undefined + ? { type: 'runtime.cancelRun', runId } + : { type: 'runtime.cancelRun', runId, tag } +} + export interface InvokeOptions { /** Wait until this epoch millisecond before the host runs the effect. */ readonly at?: number diff --git a/packages/core/src/fold.test.ts b/packages/core/src/fold.test.ts index 74b3654..86f00e8 100644 --- a/packages/core/src/fold.test.ts +++ b/packages/core/src/fold.test.ts @@ -97,10 +97,12 @@ describe('foldRun', () => { definitionVersion: 'v1', input: null, requestedThreadId: null, + attributes: {}, idempotencyKey: null, rootThreadId: threadId, launchedBy: null, correlationId: null, + trace: null, }) expect(state.threads[threadId]?.status).toBe('running') diff --git a/packages/core/src/fold.ts b/packages/core/src/fold.ts index 535df71..55e9fd6 100644 --- a/packages/core/src/fold.ts +++ b/packages/core/src/fold.ts @@ -5,9 +5,11 @@ import type { EventEnvelope } from './envelope' import { createEffectId } from './ids' import { readRunLaunchSource } from './launcher' import { DEFAULT_DEFINITION_VERSION } from './module' +import { readAttributes } from './run-context' import type { ThreadRecord, OutstandingEffect, RunState, WaitRecord } from './state' import { completedRunStatus, emptyRunState } from './state' import { isTerminalStatus, type ThreadContext, type ThreadDefinition } from './thread' +import { readStoredTrace } from './trace' import type { JsonValue } from './types' export interface FoldRegistry { @@ -35,6 +37,28 @@ function readString(obj: { [key: string]: JsonValue }, key: string): string | un return Predicate.isString(value) ? value : undefined } +/** Heartbeat progress already recorded for this effect, if any. */ +function priorHeartbeatDetails(state: RunState, effectId: string): JsonValue | null { + return state.effectExecutions[effectId]?.heartbeatDetails ?? null +} + +/** + * A heartbeat that carries `details` replaces the saved progress. Every other execution event + * keeps it, including the retry that starts the next attempt. + */ +function nextHeartbeatDetails( + state: RunState, + effectId: string, + event: EventEnvelope, + payload: { [key: string]: JsonValue }, +): JsonValue | null { + if (event.type === 'runtime.effect.heartbeat' && Object.hasOwn(payload, 'details')) { + return payload.details ?? null + } + + return priorHeartbeatDetails(state, effectId) +} + function cloneState(state: RunState): RunState { return { runId: state.runId, @@ -50,6 +74,10 @@ function cloneState(state: RunState): RunState { processedIdempotencyKeys: state.processedIdempotencyKeys ? [...state.processedIdempotencyKeys] : [], + launchedRuns: state.launchedRuns ? [...state.launchedRuns] : [], + closeWaiting: state.closeWaiting ?? false, + childCloseSent: state.childCloseSent ?? false, + attributes: state.attributes ?? {}, } } @@ -127,6 +155,8 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi rootThreadId, launchedBy: readRunLaunchSource(payload.launchedBy), correlationId: event.correlationId ?? null, + attributes: readAttributes(payload.attributes), + trace: readStoredTrace(payload.trace), } : null @@ -135,6 +165,7 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi runId: event.runId, rootThreadId: rootThreadId ?? null, startIdentity, + attributes: startIdentity?.attributes ?? {}, status: 'running', } } @@ -290,6 +321,7 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi deadlineAt: null, startedAt: event.ts, lastHeartbeatAt: null, + heartbeatDetails: priorHeartbeatDetails(state, effectId), lastError: null, }, }, @@ -352,6 +384,7 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi : sameAttempt ? previous.lastHeartbeatAt : null, + heartbeatDetails: nextHeartbeatDetails(state, effectId, event, payload), lastError: readString(payload, 'error') ?? null, }, }, @@ -384,6 +417,7 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi deadlineAt: nextAttemptAt, startedAt: null, lastHeartbeatAt: null, + heartbeatDetails: priorHeartbeatDetails(state, effectId), lastError: readString(payload, 'error') ?? 'effect failed', }, }, @@ -464,7 +498,45 @@ function applyProtocol(state: RunState, event: EventEnvelope, registry: FoldRegi return next } - case 'runtime.run.launched': + case 'runtime.run.launched': { + const childRunId = readString(payload, 'runId') + + if (!childRunId || state.launchedRuns.some((run) => run.runId === childRunId)) { + return state + } + + const close = readString(payload, 'close') + + if (close !== 'terminate' && close !== 'wait') { + return state + } + + return { + ...state, + launchedRuns: [...state.launchedRuns, { runId: childRunId, close }], + } + } + + case 'runtime.child.settled': { + const childRunId = readString(payload, 'runId') + + if (!childRunId) { + return state + } + + return { + ...state, + launchedRuns: state.launchedRuns.filter((run) => run.runId !== childRunId), + } + } + + case 'runtime.child.close.requested': + return { + ...state, + childCloseSent: true, + closeWaiting: readString(payload, 'policy') === 'wait' ? true : state.closeWaiting, + } + case 'runtime.snapshot.taken': case 'runtime.thread.cancel.requested': case 'runtime.timer.set': @@ -500,6 +572,18 @@ export function stableEffectId(threadId: string, effect: RuntimeEffect): string return createEffectId(threadId, `cancel_${effect.threadId}`) } + if (effect.type === 'runtime.launch' && 'runId' in effect && Predicate.isString(effect.runId)) { + return createEffectId(threadId, `launch_${effect.runId}`) + } + + if ( + effect.type === 'runtime.cancelRun' && + 'runId' in effect && + Predicate.isString(effect.runId) + ) { + return createEffectId(threadId, `cancel_${effect.runId}`) + } + return undefined } diff --git a/packages/core/src/ids.ts b/packages/core/src/ids.ts index be3709e..c404df2 100644 --- a/packages/core/src/ids.ts +++ b/packages/core/src/ids.ts @@ -34,6 +34,11 @@ function fnv1a32(value: string, seed: number): string { return (hash >>> 0).toString(16).padStart(8, '0') } +/** A stable `run_` id derived from `parts`, so a retry launches the same child run. */ +export function deterministicRunId(...parts: readonly (string | number)[]): string { + return `run_${deterministicThreadId(...parts).replaceAll('-', '')}` +} + /** * A UUID-shaped id derived only from `parts`, so a reducer that replays the same events names the same * child thread every time. diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c1ee11b..c561c4c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -16,6 +16,7 @@ export * from './envelope' export * from './catalog' export * from './effects' export * from './protocol' +export * from './subscription' export * from './thread' export * from './state' export * from './match' @@ -38,7 +39,13 @@ export * from './serialize' export * from './coalesce' export * from './event-index' export * from './limits' +export * from './blob-store' +export * from './blobs' +export * from './blob-offload' export * from './definition-store' +export * from './definition-resolver' export * from './launcher' +export * from './trace' +export * from './run-context' export * from './published' export type { DebuggerHost, DebuggerRequest } from './debugger-host' diff --git a/packages/core/src/launcher.ts b/packages/core/src/launcher.ts index 70d7370..f64fd48 100644 --- a/packages/core/src/launcher.ts +++ b/packages/core/src/launcher.ts @@ -1,7 +1,14 @@ import { Context, Data, Effect, Option, Schema } from 'effect' +import type { EventInput } from './envelope' +import type { RunAttributes, SecretContext } from './run-context' +import type { TraceContext } from './trace' import type { JsonValue, RunStatus } from './types' +export const ChildCloseSchema = Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]) + +export type ChildClose = Schema.Schema.Type + /** The effect that launched a run, recorded on the launched run's `runtime.run.started`. */ export const RunLaunchSourceSchema = Schema.Struct({ runId: Schema.String, @@ -36,6 +43,15 @@ export interface LaunchRunArgs { * correlation id, or to the launching run's id when that run has none. */ readonly correlationId?: string + /** + * What the launching run does with this child when the launching run closes. + * `terminate` cancels the child. `wait` lets the child finish cancelling first. + */ + readonly close?: ChildClose + readonly attributes?: RunAttributes + readonly secrets?: SecretContext + /** Defaults to the launching run's trace context. */ + readonly trace?: TraceContext } /** Enough of a run to decide whether a previous fire is still in progress. */ @@ -50,6 +66,16 @@ export interface RunStatusSnapshot { export interface RunLauncher { readonly launch: (args: LaunchRunArgs) => Effect.Effect readonly status: (runId: string) => Effect.Effect + /** + * Append events to a run and wake it. When that run is already waking, the wake coalesces + * instead of waiting, so a child can notify its parent from inside the parent's wake. + */ + readonly signal: ( + runId: string, + events: readonly EventInput[], + options?: { readonly idempotencyKey?: string }, + ) => Effect.Effect + readonly cancel: (runId: string) => Effect.Effect } export class RunLauncherTag extends Context.Service()( @@ -85,12 +111,20 @@ export class RunLauncherUnavailableError extends Data.TaggedError('RunLauncherUn export const unavailableRunLauncher: RunLauncher = { launch: () => Effect.fail(new RunLauncherUnavailableError()), status: () => Effect.fail(new RunLauncherUnavailableError()), + signal: () => Effect.fail(new RunLauncherUnavailableError()), + cancel: () => Effect.fail(new RunLauncherUnavailableError()), } /** Adapts a per-`runId` RPC or cell lookup into a `RunLauncher`. */ export function createRemoteRunLauncher(target: { readonly launch: (runId: string, args: LaunchRunArgs) => Promise readonly status: (runId: string) => Promise + readonly accept: ( + runId: string, + events: readonly EventInput[], + options?: { readonly idempotencyKey?: string; readonly coalesce?: boolean }, + ) => Promise + readonly cancel: (runId: string) => Promise }): RunLauncher { return { launch: (args) => { @@ -110,5 +144,15 @@ export function createRemoteRunLauncher(target: { try: () => target.status(runId), catch: runLauncherError, }), + signal: (runId, events, options) => + Effect.tryPromise({ + try: () => target.accept(runId, events, { ...options, coalesce: true }), + catch: runLauncherError, + }), + cancel: (runId) => + Effect.tryPromise({ + try: () => target.cancel(runId), + catch: runLauncherError, + }), } } diff --git a/packages/core/src/module-scope.ts b/packages/core/src/module-scope.ts index 4ac8898..02c0660 100644 --- a/packages/core/src/module-scope.ts +++ b/packages/core/src/module-scope.ts @@ -24,6 +24,7 @@ import type { EventEnvelope, EventInput } from './envelope' import type { RunLauncherTag } from './launcher' import { type AnyRuntimeModule, + type DefinitionHost, type EffectMiddleware, type RuntimeModule, type DefinitionRef, @@ -255,6 +256,7 @@ export function defineModule< readonly definitions?: readonly DefinitionRef[] readonly middleware?: readonly EffectMiddleware[] readonly interruptsOnSignal?: SignalInterruptRule + readonly bindHost?: (host: DefinitionHost) => void } & ModuleServicesConfig>, ): RuntimeModule, EventCatalog>, FX, TH, PR> & { readonly effects: FX @@ -308,5 +310,9 @@ export function defineModule(options: any, setup: (scope: any) => any): any { module.interruptsOnSignal = members.interruptsOnSignal } + if (members.bindHost) { + module.bindHost = members.bindHost + } + return module } diff --git a/packages/core/src/module.ts b/packages/core/src/module.ts index 752010f..429d7da 100644 --- a/packages/core/src/module.ts +++ b/packages/core/src/module.ts @@ -1,6 +1,7 @@ import type { Context, Effect, Layer } from 'effect' import type { EventCatalog } from './catalog' +import type { DefinitionResolver } from './definition-resolver' import type { EffectContext, EffectDefinition, RuntimeEffect } from './effects' import type { EventInput } from './envelope' import type { ProjectionDefinition } from './projection' @@ -26,6 +27,11 @@ export type EffectMiddleware = ( */ export type SignalInterruptRule = (event: EventInput) => readonly string[] +/** Passed to a module once, before its services layer is constructed. */ +export interface DefinitionHost { + readonly definitionResolver: DefinitionResolver +} + export interface RuntimeModuleDependency { readonly namespace: string readonly protocolVersion?: string @@ -91,6 +97,11 @@ export interface RuntimeModule< readonly interruptsOnSignal?: SignalInterruptRule /** Host-side services this module's effect handlers need (an LLM, a definition lookup, a DB pool). */ readonly services?: () => Layer.Layer + /** + * Receives the host's definition resolver before `services()` runs. + * Modules that look definitions up during fold share one store with this resolver. + */ + readonly bindHost?: (host: DefinitionHost) => void } export function defineRuntimeModule< diff --git a/packages/core/src/projection.ts b/packages/core/src/projection.ts index e0fbf05..47defcf 100644 --- a/packages/core/src/projection.ts +++ b/packages/core/src/projection.ts @@ -296,12 +296,16 @@ export function toThreadTree(state: TreeBuildState): ThreadTree { rootThreadId: state.rootThreadId, startIdentity: null, forkedFrom: null, + attributes: {}, threads: state.records, waits: {}, outstandingEffects: [], effectExecutions: {}, completedEffectIds: [], processedIdempotencyKeys: [], + launchedRuns: [], + closeWaiting: false, + childCloseSent: false, }) } diff --git a/packages/core/src/protocol.ts b/packages/core/src/protocol.ts index 527ceaf..5bbf8b3 100644 --- a/packages/core/src/protocol.ts +++ b/packages/core/src/protocol.ts @@ -5,6 +5,8 @@ import { WaitConditionSchema, type WaitCondition } from './effects' import { JsonValueSchema } from './envelope' import { RunLaunchSourceSchema } from './launcher' import { DEFAULT_DEFINITION_VERSION } from './module' +import { RunAttributesSchema } from './run-context' +import { TraceContextSchema } from './trace' const DefinitionVersionSchema = Schema.String.pipe( Schema.withDecodingDefaultKey(Effect.succeed(DEFAULT_DEFINITION_VERSION)), @@ -19,6 +21,8 @@ export const protocolCatalog = defineEventCatalog('runtime', { input: JsonValueSchema, requestedThreadId: Schema.optional(Schema.NullOr(Schema.String)), launchedBy: Schema.optional(RunLaunchSourceSchema), + attributes: Schema.optional(RunAttributesSchema), + trace: Schema.optional(TraceContextSchema), }), 'run.completed': Schema.Struct({ output: Schema.NullOr(JsonValueSchema), @@ -34,6 +38,22 @@ export const protocolCatalog = defineEventCatalog('runtime', { definitionVersion: DefinitionVersionSchema, /** In the payload, not the envelope: an envelope `effectId` would complete the effect. */ effectId: Schema.String, + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('terminate' as const)), + ), + }), + 'child.settled': Schema.Struct({ + runId: Schema.String, + status: Schema.Union([ + Schema.Literal('completed'), + Schema.Literal('failed'), + Schema.Literal('cancelled'), + ]), + output: Schema.NullOr(JsonValueSchema), + error: Schema.NullOr(Schema.String), + }), + 'child.close.requested': Schema.Struct({ + policy: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]), }), 'run.forked': Schema.Struct({ sourceRunId: Schema.String, @@ -94,6 +114,8 @@ export const protocolCatalog = defineEventCatalog('runtime', { 'effect.failed': Schema.Struct({ effectId: Schema.String, error: Schema.String, + nonRetryable: Schema.optional(Schema.Boolean), + retryAfterMs: Schema.optional(Schema.Finite), }), 'effect.attempt.started': Schema.Struct({ effectId: Schema.String, @@ -119,6 +141,8 @@ export const protocolCatalog = defineEventCatalog('runtime', { attempt: Schema.Finite, heartbeatAt: Schema.Finite, deadlineAt: Schema.NullOr(Schema.Finite), + /** Progress the next attempt should resume from. Absent on heartbeats that only extend the deadline. */ + details: Schema.optional(JsonValueSchema), }), 'effect.cancel.requested': Schema.Struct({ effectId: Schema.String, @@ -167,6 +191,8 @@ export const PROTOCOL_TYPES = [ 'runtime.run.completed', 'runtime.run.cancelled', 'runtime.run.launched', + 'runtime.child.settled', + 'runtime.child.close.requested', 'runtime.run.forked', 'runtime.fork.elided', 'runtime.thread.started', diff --git a/packages/core/src/published.ts b/packages/core/src/published.ts index c9836ec..0e2e7f2 100644 --- a/packages/core/src/published.ts +++ b/packages/core/src/published.ts @@ -1,7 +1,8 @@ -import { Schema } from 'effect' +import { Effect, Schema } from 'effect' import { JsonValueSchema } from './envelope' import type { RegisteredDefinition } from './module' +import { RunAttributesSchema } from './run-context' import { jsonSchemaOfInput } from './schema' import { RunStateSchema, type RunState } from './state' @@ -40,6 +41,7 @@ export const RunSummarySchema = Schema.Struct({ status: RunStateSchema.fields.status, kind: Schema.NullOr(Schema.String), definitionName: Schema.NullOr(Schema.String), + attributes: RunAttributesSchema.pipe(Schema.withDecodingDefaultKey(Effect.succeed({}))), }) export type RunSummary = Schema.Schema.Type @@ -50,6 +52,7 @@ export function summarizeRun(state: RunState): RunSummary { status: state.status, kind: state.startIdentity?.kind ?? null, definitionName: state.startIdentity?.definitionName ?? null, + attributes: state.attributes ?? {}, } } diff --git a/packages/core/src/run-context.ts b/packages/core/src/run-context.ts new file mode 100644 index 0000000..6ab8fcd --- /dev/null +++ b/packages/core/src/run-context.ts @@ -0,0 +1,54 @@ +import { Option, Schema, type Effect } from 'effect' + +import type { JsonValue } from './types' + +/** + * Immutable labels for a run, such as tenant, project, or org. + * They are safe for every reader of the run: events, snapshots, summaries, and authorization. + */ +export type RunAttributes = { readonly [key: string]: string } + +/** + * Bearer material for a run, such as an attenuated macaroon. + * It is stored beside the log and is never written to an event, snapshot, or HTTP response. + */ +export type SecretContext = { readonly [key: string]: string } + +export const RunAttributesSchema = Schema.Record(Schema.String, Schema.String) + +export const SecretContextSchema = Schema.Record(Schema.String, Schema.String) + +const decodeAttributes = Schema.decodeUnknownOption(RunAttributesSchema) +const decodeSecretContext = Schema.decodeUnknownOption(SecretContextSchema) + +export interface RunSecretStore { + readonly get: (runId: string) => Effect.Effect + readonly put: (runId: string, secrets: SecretContext) => Effect.Effect +} + +export function hasEntries(record: { readonly [key: string]: string }): boolean { + return Object.keys(record).length > 0 +} + +export function sameStringRecord( + left: { readonly [key: string]: string }, + right: { readonly [key: string]: string }, +): boolean { + const leftKeys = Object.keys(left).toSorted() + const rightKeys = Object.keys(right).toSorted() + + if (leftKeys.length !== rightKeys.length) { + return false + } + + return leftKeys.every((key, index) => key === rightKeys[index] && left[key] === right[key]) +} + +/** Reads a string map from an event payload. Anything else becomes an empty map. */ +export function readAttributes(value: JsonValue | undefined): RunAttributes { + return value === undefined ? {} : Option.getOrElse(decodeAttributes(value), () => ({})) +} + +export function readSecretContext(value: JsonValue): SecretContext { + return Option.getOrElse(decodeSecretContext(value), () => ({})) +} diff --git a/packages/core/src/sqlite-store.ts b/packages/core/src/sqlite-store.ts index 2a6621b..f86b0e7 100644 --- a/packages/core/src/sqlite-store.ts +++ b/packages/core/src/sqlite-store.ts @@ -8,6 +8,7 @@ import { SnapshotPayloadTooLargeError, } from './limits' import type { RunSummary } from './published' +import { RunAttributesSchema, type RunSecretStore } from './run-context' import { SnapshotStoreError, withSnapshotStore, @@ -59,7 +60,13 @@ CREATE TABLE IF NOT EXISTS looms_run_headers ( run_id TEXT PRIMARY KEY, status TEXT NOT NULL, kind TEXT, - definition_name TEXT + definition_name TEXT, + attributes TEXT NOT NULL DEFAULT '{}' +); + +CREATE TABLE IF NOT EXISTS looms_run_secrets ( + run_id TEXT PRIMARY KEY, + body TEXT NOT NULL ); `.trim() @@ -178,6 +185,13 @@ function initSchema(exec: SqliteExec): void { exec.run(sql) } } + + // Databases created before attributes were indexed already have the headers table. + try { + exec.run(`ALTER TABLE looms_run_headers ADD COLUMN attributes TEXT NOT NULL DEFAULT '{}'`) + } catch { + // The column is already present. + } } function parseEvent(body: string): EventEnvelope { @@ -201,9 +215,23 @@ const HeaderRowSchema = Schema.Struct({ status: RunStateSchema.fields.status, kind: Schema.NullOr(Schema.String), definition_name: Schema.NullOr(Schema.String), + attributes: Schema.String, }) const decodeHeaderRow = Schema.decodeUnknownSync(HeaderRowSchema) +const encodeAttributes = Schema.encodeSync(Schema.fromJsonString(RunAttributesSchema)) + +const encodeSecrets = Schema.encodeSync( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)), +) + +const decodeAttributesColumn = Schema.decodeUnknownOption( + Schema.fromJsonString(RunAttributesSchema), +) + +const decodeSecretsColumn = Schema.decodeUnknownOption( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)), +) function headerFromRow(row: Schema.Schema.Type): RunSummary { return { @@ -211,6 +239,7 @@ function headerFromRow(row: Schema.Schema.Type): RunSumm status: row.status, kind: row.kind, definitionName: row.definition_name, + attributes: Option.getOrElse(decodeAttributesColumn(row.attributes), () => ({})), } } @@ -289,13 +318,20 @@ function createSqliteSnapshotStore(exec: SqliteExec): SnapshotStore { Effect.try({ try: () => { exec.run( - `INSERT INTO looms_run_headers (run_id, status, kind, definition_name) - VALUES (?, ?, ?, ?) + `INSERT INTO looms_run_headers (run_id, status, kind, definition_name, attributes) + VALUES (?, ?, ?, ?, ?) ON CONFLICT (run_id) DO UPDATE SET status = excluded.status, kind = excluded.kind, - definition_name = excluded.definition_name`, - [header.runId, header.status, header.kind, header.definitionName], + definition_name = excluded.definition_name, + attributes = excluded.attributes`, + [ + header.runId, + header.status, + header.kind, + header.definitionName, + encodeAttributes(header.attributes), + ], ) }, catch: (cause) => @@ -305,13 +341,46 @@ function createSqliteSnapshotStore(exec: SqliteExec): SnapshotStore { listHeaders: Effect.try({ try: () => exec - .rows('SELECT run_id, status, kind, definition_name FROM looms_run_headers') + .rows('SELECT run_id, status, kind, definition_name, attributes FROM looms_run_headers') .map((row) => headerFromRow(decodeHeaderRow(row))), catch: (cause) => new SnapshotStoreError('Failed to list run headers', cause), }), } } +const SecretRowSchema = Schema.Struct({ body: Schema.String }) +const decodeSecretRow = Schema.decodeUnknownSync(SecretRowSchema) + +function createSqliteSecrets(exec: SqliteExec): RunSecretStore { + return { + get: (runId) => + Effect.try({ + try: () => { + const [row] = exec.rows('SELECT body FROM looms_run_secrets WHERE run_id = ?', [runId]) + + if (row === undefined) { + return {} + } + + return Option.getOrElse(decodeSecretsColumn(decodeSecretRow(row).body), () => ({})) + }, + catch: (cause) => new EventStoreError(`Failed to read secret context for ${runId}`, cause), + }), + put: (runId, secrets) => + Effect.try({ + try: () => { + exec.run( + `INSERT INTO looms_run_secrets (run_id, body) + VALUES (?, ?) + ON CONFLICT (run_id) DO UPDATE SET body = excluded.body`, + [runId, encodeSecrets(secrets)], + ) + }, + catch: (cause) => new EventStoreError(`Failed to store secret context for ${runId}`, cause), + }), + } +} + function createSqliteEventStore(exec: SqliteExec): EventStore { const waiters = new Map void>>() @@ -580,6 +649,7 @@ function createSqliteEventStore(exec: SqliteExec): EventStore { .map((row) => decodeRunIdRow(row).run_id), catch: (cause) => new EventStoreError('listRuns failed', cause), }), + runSecrets: createSqliteSecrets(exec), } return service diff --git a/packages/core/src/state.ts b/packages/core/src/state.ts index 58d370b..c86e658 100644 --- a/packages/core/src/state.ts +++ b/packages/core/src/state.ts @@ -7,9 +7,11 @@ import { type WaitCondition, } from './effects' import { JsonValueSchema } from './envelope' -import { RunLaunchSourceSchema, type RunLaunchSource } from './launcher' +import { RunLaunchSourceSchema, type ChildClose, type RunLaunchSource } from './launcher' import { DEFAULT_DEFINITION_VERSION } from './module' +import { RunAttributesSchema, type RunAttributes } from './run-context' import { isTerminalStatus } from './thread' +import { TraceContextSchema, type TraceContext } from './trace' import type { JsonValue, ThreadStatus, RunStatus } from './types' export interface ThreadRecord { @@ -58,6 +60,8 @@ export interface EffectExecutionRecord { /** When the current attempt started executing, locally or on a worker. */ startedAt: number | null lastHeartbeatAt: number | null + /** Progress reported by the latest heartbeat. Survives retries so the next attempt can resume. */ + heartbeatDetails: JsonValue | null lastError: string | null } @@ -71,6 +75,9 @@ export interface RunStartIdentity { rootThreadId: string launchedBy: RunLaunchSource | null correlationId: string | null + attributes: RunAttributes + /** The caller's W3C trace context. Effects, workers, and launched runs inherit it. */ + trace: TraceContext | null } /** The run and last sequence a fork copied its history from. */ @@ -79,18 +86,31 @@ export interface RunForkSource { seq: number } +export interface LaunchedRun { + runId: string + close: ChildClose +} + export interface RunState { runId: string status: RunStatus rootThreadId: string | null startIdentity: RunStartIdentity | null forkedFrom: RunForkSource | null + /** Immutable labels copied from `runtime.run.started`. Safe for every reader of the run. */ + attributes: RunAttributes threads: { [threadId: string]: ThreadRecord } waits: { [waitId: string]: WaitRecord } outstandingEffects: readonly OutstandingEffect[] effectExecutions: { [effectId: string]: EffectExecutionRecord } completedEffectIds: readonly string[] processedIdempotencyKeys: readonly string[] + /** Child runs this run launched and has not yet seen settle. */ + launchedRuns: readonly LaunchedRun[] + /** The run is cancelling and will not finish until `wait` children settle. */ + closeWaiting: boolean + /** Child runs were already asked to stop because this run closed. */ + childCloseSent: boolean } export const RunStateSchema = Schema.Struct({ @@ -117,8 +137,13 @@ export const RunStateSchema = Schema.Struct({ correlationId: Schema.NullOr(Schema.String).pipe( Schema.withDecodingDefaultKey(Effect.succeed(null)), ), + attributes: RunAttributesSchema.pipe(Schema.withDecodingDefaultKey(Effect.succeed({}))), + trace: Schema.NullOr(TraceContextSchema).pipe( + Schema.withDecodingDefaultKey(Effect.succeed(null)), + ), }), ).pipe(Schema.withDecodingDefaultKey(Effect.succeed(null))), + attributes: RunAttributesSchema.pipe(Schema.withDecodingDefaultKey(Effect.succeed({}))), forkedFrom: Schema.NullOr(Schema.Struct({ runId: Schema.String, seq: Schema.Finite })).pipe( Schema.withDecodingDefaultKey(Effect.succeed(null)), ), @@ -189,6 +214,9 @@ export const RunStateSchema = Schema.Struct({ lastHeartbeatAt: Schema.NullOr(Schema.Finite).pipe( Schema.withDecodingDefaultKey(Effect.succeed(null)), ), + heartbeatDetails: Schema.NullOr(JsonValueSchema).pipe( + Schema.withDecodingDefaultKey(Effect.succeed(null)), + ), lastError: Schema.NullOr(Schema.String), }), ).pipe(Schema.withDecodingDefaultKey(Effect.succeed({}))), @@ -198,6 +226,14 @@ export const RunStateSchema = Schema.Struct({ processedIdempotencyKeys: Schema.Array(Schema.String).pipe( Schema.withDecodingDefaultKey(Effect.succeed([])), ), + launchedRuns: Schema.Array( + Schema.Struct({ + runId: Schema.String, + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]), + }), + ).pipe(Schema.withDecodingDefaultKey(Effect.succeed([]))), + closeWaiting: Schema.Boolean.pipe(Schema.withDecodingDefaultKey(Effect.succeed(false))), + childCloseSent: Schema.Boolean.pipe(Schema.withDecodingDefaultKey(Effect.succeed(false))), }) export function emptyRunState(runId: string): RunState { @@ -207,12 +243,16 @@ export function emptyRunState(runId: string): RunState { rootThreadId: null, startIdentity: null, forkedFrom: null, + attributes: {}, threads: {}, waits: {}, outstandingEffects: [], effectExecutions: {}, completedEffectIds: [], processedIdempotencyKeys: [], + launchedRuns: [], + closeWaiting: false, + childCloseSent: false, } } diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index 7651bdd..cf46794 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -2,6 +2,7 @@ import { Context, Data, Effect, Layer, Option, Queue, Ref, Stream } from 'effect import { withAssignedSeq, type AppendableEvent, type EventEnvelope } from './envelope' import { assertEventByteLimits, EventPayloadTooLargeError } from './limits' +import { readSecretContext, type RunSecretStore, type SecretContext } from './run-context' export class EventStoreError extends Data.TaggedError('EventStoreError')<{ readonly message: string @@ -288,6 +289,11 @@ export interface EventStore { ) => Stream.Stream readonly listRuns: Effect.Effect + /** + * Run-scoped bearer material. Absent on stores that cannot keep it out of the event log. + * A start that supplies secrets fails closed when this is missing. + */ + readonly runSecrets?: RunSecretStore } /** @@ -321,6 +327,13 @@ interface RunLog { export const makeMemoryEventStore = Effect.gen(function* () { const logs = yield* Ref.make(new Map()) + const secrets = yield* Ref.make(new Map()) + + const runSecrets: RunSecretStore = { + get: (runId) => + Ref.get(secrets).pipe(Effect.map((map) => readSecretContext(map.get(runId) ?? {}))), + put: (runId, value) => Ref.update(secrets, (map) => new Map(map).set(runId, value)), + } const getOrCreate = (map: Map, runId: string): RunLog => { const existing = map.get(runId) @@ -552,6 +565,7 @@ export const makeMemoryEventStore = Effect.gen(function* () { ), listRuns: Ref.get(logs).pipe(Effect.map((map) => [...map.keys()])), + runSecrets, } return service diff --git a/packages/core/src/subscription.test.ts b/packages/core/src/subscription.test.ts new file mode 100644 index 0000000..2dad15a --- /dev/null +++ b/packages/core/src/subscription.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, test } from 'bun:test' + +import { createEvent } from './envelope' +import { + attachmentFromFilter, + decodeSubscriptionFrame, + encodeSubscriptionFrame, + eventMatchesSubscription, + fanoutSubscription, + openSubscription, + pushSubscriptionEvents, + readSubscriptionData, + subscriptionFilter, + subscriptionFromSeq, + subscriptionReplay, + type AttachedSubscriptionSocket, + type SubscriptionAttachment, +} from './subscription' + +function envelope(seq: number, threadId: string, ephemeral = false) { + return { + ...createEvent('run_1', { + type: 'runtime.thread.started', + payload: { + threadId, + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v1', + input: null, + parentThreadId: null, + }, + threadId, + origin: { type: 'system' }, + ephemeral, + }), + seq, + } +} + +class FakeSocket implements AttachedSubscriptionSocket { + readonly sent: string[] = [] + closed: { code: number; reason: string } | undefined + bufferedAmount = 0 + private current: SubscriptionAttachment | undefined + + constructor(attachment: SubscriptionAttachment) { + this.current = attachment + } + + send(data: string): void { + if (this.closed) { + throw new Error('closed') + } + + this.sent.push(data) + } + + close(code: number, reason: string): void { + this.closed = { code, reason } + } + + attachment(): SubscriptionAttachment | undefined { + return this.current + } + + setAttachment(attachment: SubscriptionAttachment): void { + this.current = attachment + } +} + +describe('subscription protocol', () => { + test('decodes hello, a typeless batch, and resync', () => { + expect(decodeSubscriptionFrame('{"type":"hello","head":1,"tail":4}')).toEqual({ + type: 'hello', + head: 1, + tail: 4, + }) + + const batch = decodeSubscriptionFrame('{"batch":[{"type":"runtime.run.started"}]}') + expect(batch?.type).toBe('batch') + + expect( + encodeSubscriptionFrame({ type: 'batch', batch: [{ type: 'runtime.run.started' }] }), + ).toBe('{"batch":[{"type":"runtime.run.started"}]}') + + expect(decodeSubscriptionFrame('{"type":"resync","head":3,"reason":"trimmed"}')).toEqual({ + type: 'resync', + head: 3, + reason: 'trimmed', + }) + + expect(decodeSubscriptionFrame('not-json')).toBeUndefined() + }) + + test('cursor and filter follow the query string', () => { + const url = new URL( + 'http://looms.test/runs/run_1/events?fromSeq=4&threadId=thr_a&includeEphemeral=0', + ) + + expect(subscriptionFromSeq(url, null)).toEqual({ ok: true, fromSeq: 4 }) + expect(subscriptionFromSeq(url, '2')).toEqual({ ok: true, fromSeq: 3 }) + expect(subscriptionFromSeq(url, 'nope').ok).toBe(false) + expect(subscriptionFilter(url)).toEqual({ threadId: 'thr_a', includeEphemeral: false }) + }) + + test('a cursor below the retained head asks for resync', () => { + expect(subscriptionReplay(1, 1)).toEqual({ kind: 'play', fromSeq: 1 }) + expect(subscriptionReplay(5, 4)).toEqual({ kind: 'resync', head: 5 }) + }) + + test('replay sends hello then the matching events and resumes the cursor', () => { + const socket = new FakeSocket( + attachmentFromFilter({ threadId: 'thr_a', includeEphemeral: false }, 1, null), + ) + + const next = openSubscription(socket, socket.attachment()!, { + kind: 'open', + hello: { type: 'hello', head: 1, tail: 3 }, + events: [envelope(1, 'thr_a'), envelope(2, 'thr_b'), envelope(3, 'thr_a', true)], + }) + + expect(socket.sent.map((line) => JSON.parse(line))).toEqual([ + { type: 'hello', head: 1, tail: 3 }, + { batch: [expect.objectContaining({ seq: 1, threadId: 'thr_a' })] }, + ]) + + expect(next?.cursor).toBe(3) + expect(socket.closed).toBeUndefined() + }) + + test('a trimmed cursor closes with resync', () => { + const socket = new FakeSocket( + attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, 'ada'), + ) + + const next = openSubscription(socket, socket.attachment()!, { kind: 'resync', head: 8 }) + + expect(next).toBeUndefined() + + expect(JSON.parse(socket.sent[0] ?? '{}')).toEqual({ + type: 'resync', + head: 8, + reason: 'trimmed', + }) + + expect(socket.closed).toEqual({ code: 4000, reason: 'trimmed' }) + }) + + test('backpressure closes the socket and a later append does not write to it', () => { + const slow = new FakeSocket( + attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, null), + ) + + slow.bufferedAmount = 32 + + const live = new FakeSocket( + attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, null), + ) + + fanoutSubscription([slow, live], [envelope(1, 'thr_a')], { bufferLimitBytes: 8, head: 4 }) + + expect(slow.closed?.reason).toBe('backpressure') + + expect(JSON.parse(slow.sent[0] ?? '{}')).toMatchObject({ + type: 'resync', + reason: 'backpressure', + head: 4, + }) + + expect(live.closed).toBeUndefined() + expect(live.attachment()?.cursor).toBe(1) + + fanoutSubscription([slow, live], [envelope(2, 'thr_a')], { bufferLimitBytes: 8, head: 4 }) + expect(live.attachment()?.cursor).toBe(2) + expect(slow.sent).toHaveLength(1) + }) + + test('readSubscriptionData keeps events and surfaces resync', () => { + const event = envelope(1, 'thr_a') + const encoded = encodeSubscriptionFrame({ type: 'batch', batch: [event] }) + const read = readSubscriptionData(encoded, 'run_1') + + expect(read.kind).toBe('events') + + if (read.kind === 'events') { + expect(read.events[0]?.seq).toBe(1) + } + + expect(readSubscriptionData('{"type":"resync","head":9,"reason":"trimmed"}', 'run_1')).toEqual({ + kind: 'resync', + head: 9, + reason: 'trimmed', + }) + }) + + test('filter drops other threads and ephemeral events', () => { + const filter = { threadId: 'thr_a', includeEphemeral: false } + + expect(eventMatchesSubscription(envelope(1, 'thr_a'), filter)).toBe(true) + expect(eventMatchesSubscription(envelope(2, 'thr_b'), filter)).toBe(false) + expect(eventMatchesSubscription(envelope(3, 'thr_a', true), filter)).toBe(false) + }) + + test('push does not resend a seq the attachment already passed', () => { + const attachment = { + ...attachmentFromFilter({ threadId: null, includeEphemeral: true }, 1, null), + cursor: 1, + } + + const socket = new FakeSocket(attachment) + + const next = pushSubscriptionEvents( + socket, + [envelope(1, 'thr_a'), envelope(2, 'thr_a')], + attachment, + ) + + expect(socket.sent).toHaveLength(1) + expect(next?.cursor).toBe(2) + }) +}) diff --git a/packages/core/src/subscription.ts b/packages/core/src/subscription.ts new file mode 100644 index 0000000..7cc5c0d --- /dev/null +++ b/packages/core/src/subscription.ts @@ -0,0 +1,390 @@ +import { Option, Predicate, Schema } from 'effect' + +import type { EventEnvelope } from './envelope' +import { decodeLoomsEvent, encodeLoomsEvent } from './event-codec' +import { stringifyJson, type JsonValue } from './types' + +/** A socket is closed with this code when the client must resume from `head`. */ +export const SUBSCRIPTION_RESYNC_CODE = 4000 + +/** Close a socket that has fallen this far behind instead of buffering without bound. */ +export const SUBSCRIPTION_BACKPRESSURE_BYTES = 1024 * 1024 + +export type SubscriptionTransport = 'websocket' | 'sse' | 'auto' + +export type SubscriptionResyncReason = 'trimmed' | 'backpressure' + +export interface SubscriptionHello { + readonly type: 'hello' + readonly head: number + readonly tail: number +} + +export interface SubscriptionBatch { + readonly type: 'batch' + readonly batch: readonly JsonValue[] +} + +export interface SubscriptionResync { + readonly type: 'resync' + readonly head: number + readonly reason: SubscriptionResyncReason +} + +export type SubscriptionFrame = SubscriptionHello | SubscriptionBatch | SubscriptionResync + +export interface SubscriptionFilter { + readonly threadId: string | null + readonly includeEphemeral: boolean +} + +/** State hibernated with a Durable Object WebSocket. Writes stay on HTTP. */ +export interface SubscriptionAttachment { + readonly principal: string | null + readonly cursor: number + readonly threadId: string | null + readonly includeEphemeral: boolean +} + +export interface SubscriptionSender { + send(data: string): void + close(code: number, reason: string): void + readonly bufferedAmount: number +} + +export interface AttachedSubscriptionSocket extends SubscriptionSender { + attachment(): SubscriptionAttachment | undefined + setAttachment(attachment: SubscriptionAttachment): void +} + +export interface SubscriptionPushOptions { + readonly bufferLimitBytes?: number + readonly head?: number +} + +export type SubscriptionOpening = + | { readonly kind: 'resync'; readonly head: number } + | { + readonly kind: 'open' + readonly hello: SubscriptionHello + readonly events: readonly EventEnvelope[] + } + +export type SubscriptionRead = + | { readonly kind: 'events'; readonly events: readonly EventEnvelope[] } + | { readonly kind: 'hello'; readonly head: number; readonly tail: number } + | { readonly kind: 'resync'; readonly head: number; readonly reason: SubscriptionResyncReason } + | { readonly kind: 'ignore' } + +const decodeJsonArray = Schema.decodeUnknownOption(Schema.Array(Schema.Json)) +const asJson = Schema.decodeUnknownOption(Schema.Json) + +export function encodeSubscriptionFrame(frame: SubscriptionFrame): string { + switch (frame.type) { + case 'hello': + return stringifyJson({ type: 'hello', head: frame.head, tail: frame.tail }) + case 'resync': + return stringifyJson({ type: 'resync', head: frame.head, reason: frame.reason }) + case 'batch': + return stringifyJson({ batch: frame.batch }) + + default: { + const unexpected: never = frame + return unexpected + } + } +} + +export function decodeSubscriptionFrame(data: string): SubscriptionFrame | undefined { + let parsed: unknown + + try { + parsed = JSON.parse(data) + } catch { + return undefined + } + + if (!Predicate.isObject(parsed)) { + return undefined + } + + if ('type' in parsed && parsed.type === 'hello') { + const head = + 'head' in parsed && Predicate.isNumber(parsed.head) && Number.isFinite(parsed.head) + ? parsed.head + : undefined + + const tail = + 'tail' in parsed && Predicate.isNumber(parsed.tail) && Number.isFinite(parsed.tail) + ? parsed.tail + : undefined + + if (head === undefined || tail === undefined || head < 1 || tail < 0) { + return undefined + } + + return { type: 'hello', head, tail } + } + + if ('type' in parsed && parsed.type === 'resync') { + const head = + 'head' in parsed && Predicate.isNumber(parsed.head) && Number.isFinite(parsed.head) + ? parsed.head + : undefined + + const reason = + 'reason' in parsed && (parsed.reason === 'trimmed' || parsed.reason === 'backpressure') + ? parsed.reason + : undefined + + if (head === undefined || reason === undefined || head < 1) { + return undefined + } + + return { type: 'resync', head, reason } + } + + if (!('batch' in parsed)) { + return undefined + } + + const batch = decodeJsonArray(parsed.batch) + + if (Option.isNone(batch)) { + return undefined + } + + return { type: 'batch', batch: batch.value } +} + +/** One cursor for SSE and WebSocket. `Last-Event-ID` is the last seen seq. */ +export function subscriptionFromSeq( + url: URL, + lastEventId: string | null, +): + | { readonly ok: true; readonly fromSeq: number } + | { readonly ok: false; readonly message: string } { + if (lastEventId !== null && lastEventId.trim() !== '') { + const value = Number(lastEventId) + + if (!Number.isSafeInteger(value) || value < 0) { + return { ok: false, message: 'Last-Event-ID must be a non-negative integer' } + } + + return { ok: true, fromSeq: value + 1 } + } + + const raw = url.searchParams.get('fromSeq') + + if (raw === null) { + return { ok: true, fromSeq: 1 } + } + + const fromSeq = Number(raw) + + if (!Number.isSafeInteger(fromSeq) || fromSeq < 1) { + return { ok: false, message: 'fromSeq must be an integer greater than or equal to 1' } + } + + return { ok: true, fromSeq } +} + +export function subscriptionFilter(url: URL): SubscriptionFilter { + const threadId = url.searchParams.get('threadId') + const includeEphemeral = url.searchParams.get('includeEphemeral') + + return { + threadId: threadId && threadId.length > 0 ? threadId : null, + includeEphemeral: includeEphemeral !== 'false' && includeEphemeral !== '0', + } +} + +export function subscriptionReplay( + head: number, + fromSeq: number, +): + | { readonly kind: 'resync'; readonly head: number } + | { readonly kind: 'play'; readonly fromSeq: number } { + if (fromSeq < head) { + return { kind: 'resync', head } + } + + return { kind: 'play', fromSeq } +} + +export function eventMatchesSubscription( + event: EventEnvelope, + filter: SubscriptionFilter, +): boolean { + if (!filter.includeEphemeral && event.ephemeral) { + return false + } + + if (filter.threadId !== null && event.threadId !== filter.threadId) { + return false + } + + return true +} + +export function attachmentFromFilter( + filter: SubscriptionFilter, + fromSeq: number, + principal: string | null, +): SubscriptionAttachment { + return { + principal, + cursor: fromSeq - 1, + threadId: filter.threadId, + includeEphemeral: filter.includeEphemeral, + } +} + +export function readSubscriptionData(data: string, runId: string): SubscriptionRead { + const frame = decodeSubscriptionFrame(data) + + if (!frame) { + return { kind: 'ignore' } + } + + switch (frame.type) { + case 'hello': + return { kind: 'hello', head: frame.head, tail: frame.tail } + case 'resync': + return { kind: 'resync', head: frame.head, reason: frame.reason } + + case 'batch': { + const events: EventEnvelope[] = [] + + for (const item of frame.batch) { + try { + events.push(decodeLoomsEvent(item, runId)) + } catch { + // A bad item does not drop the rest of the batch. + } + } + + return { kind: 'events', events } + } + + default: { + const unexpected: never = frame + return unexpected + } + } +} + +/** + * Sends the events this attachment still needs. Returns the advanced attachment, + * or `undefined` after the socket is closed for resync. + */ +export function pushSubscriptionEvents( + socket: SubscriptionSender, + events: readonly EventEnvelope[], + attachment: SubscriptionAttachment, + options?: SubscriptionPushOptions, +): SubscriptionAttachment | undefined { + const limit = options?.bufferLimitBytes ?? SUBSCRIPTION_BACKPRESSURE_BYTES + + if (socket.bufferedAmount > limit) { + closeForResync(socket, options?.head ?? 1, 'backpressure') + return undefined + } + + const filter: SubscriptionFilter = { + threadId: attachment.threadId, + includeEphemeral: attachment.includeEphemeral, + } + + let cursor = attachment.cursor + const batch: JsonValue[] = [] + + for (const event of events) { + if (event.seq <= cursor) { + continue + } + + cursor = event.seq + + if (eventMatchesSubscription(event, filter)) { + const parsed: unknown = JSON.parse(stringifyJson(encodeLoomsEvent(event))) + const encoded = asJson(parsed) + + if (Option.isSome(encoded)) { + batch.push(encoded.value) + } + } + } + + if (batch.length > 0 && !sendFrame(socket, { type: 'batch', batch })) { + closeForResync(socket, options?.head ?? 1, 'backpressure') + return undefined + } + + return { ...attachment, cursor } +} + +export function openSubscription( + socket: SubscriptionSender, + attachment: SubscriptionAttachment, + opening: SubscriptionOpening, + options?: SubscriptionPushOptions, +): SubscriptionAttachment | undefined { + if (opening.kind === 'resync') { + closeForResync(socket, opening.head, 'trimmed') + return undefined + } + + if (!sendFrame(socket, opening.hello)) { + closeForResync(socket, opening.hello.head, 'backpressure') + return undefined + } + + return pushSubscriptionEvents(socket, opening.events, attachment, { + ...options, + head: opening.hello.head, + }) +} + +export function fanoutSubscription( + sockets: readonly AttachedSubscriptionSocket[], + events: readonly EventEnvelope[], + options?: SubscriptionPushOptions, +): void { + for (const socket of sockets) { + const current = socket.attachment() + + if (!current) { + continue + } + + const next = pushSubscriptionEvents(socket, events, current, options) + + if (next) { + socket.setAttachment(next) + } + } +} + +function closeForResync( + socket: SubscriptionSender, + head: number, + reason: SubscriptionResyncReason, +): void { + sendFrame(socket, { type: 'resync', head, reason }) + + try { + socket.close(SUBSCRIPTION_RESYNC_CODE, reason) + } catch { + // The platform may already have dropped the socket. + } +} + +function sendFrame(socket: SubscriptionSender, frame: SubscriptionFrame): boolean { + try { + socket.send(encodeSubscriptionFrame(frame)) + return true + } catch { + return false + } +} diff --git a/packages/core/src/trace.test.ts b/packages/core/src/trace.test.ts new file mode 100644 index 0000000..072b252 --- /dev/null +++ b/packages/core/src/trace.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, test } from 'bun:test' + +import { readTraceContext, readTraceHeaders, traceHeaders } from './trace' + +const TRACEPARENT = '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01' + +describe('trace context', () => { + test('reads a version-00 traceparent and its tracestate', () => { + expect(readTraceContext(` ${TRACEPARENT} `, 'congo=t61rcWkgMzE')).toEqual({ + traceparent: TRACEPARENT, + tracestate: 'congo=t61rcWkgMzE', + }) + }) + + test('drops values that are not a W3C traceparent', () => { + expect(readTraceContext(undefined)).toBeUndefined() + expect(readTraceContext('not-a-trace')).toBeUndefined() + expect(readTraceContext(TRACEPARENT.toUpperCase())).toBeUndefined() + expect(readTraceContext(TRACEPARENT.replace(/^00/, '01'))).toBeUndefined() + expect(readTraceContext(`00-${'0'.repeat(32)}-00f067aa0ba902b7-01`)).toBeUndefined() + + expect(readTraceContext(`00-4bf92f3577b34da6a3ce929d0e0e4736-${'0'.repeat(16)}-01`)).toBe( + undefined, + ) + }) + + test('drops a tracestate over the W3C length limit but keeps the traceparent', () => { + expect(readTraceContext(TRACEPARENT, `a=${'b'.repeat(600)}`)).toEqual({ + traceparent: TRACEPARENT, + }) + }) + + test('round-trips through request headers', () => { + const trace = { traceparent: TRACEPARENT, tracestate: 'k=v' } + + expect(readTraceHeaders(new Headers(traceHeaders(trace)))).toEqual(trace) + expect(traceHeaders(undefined)).toEqual({}) + }) +}) diff --git a/packages/core/src/trace.ts b/packages/core/src/trace.ts new file mode 100644 index 0000000..152e81a --- /dev/null +++ b/packages/core/src/trace.ts @@ -0,0 +1,63 @@ +import { Option, Schema } from 'effect' + +import type { JsonValue } from './types' + +/** + * W3C trace context from the caller that started a run. The field names are the W3C header names, + * so the object is also a carrier for OpenTelemetry `propagation.extract`. + */ +export const TraceContextSchema = Schema.Struct({ + traceparent: Schema.String, + tracestate: Schema.optional(Schema.String), +}) + +export type TraceContext = Schema.Schema.Type + +const decodeTraceContext = Schema.decodeUnknownOption(TraceContextSchema) + +const TRACEPARENT = /^00-([0-9a-f]{32})-([0-9a-f]{16})-[0-9a-f]{2}$/ +const INVALID_TRACE_ID = '0'.repeat(32) +const INVALID_PARENT_ID = '0'.repeat(16) +const MAX_TRACESTATE_LENGTH = 512 + +/** + * A version-00 `traceparent`, with its `tracestate` when that is within the W3C length limit. + * Anything else is not a trace context, so callers drop it rather than store it. + */ +export function readTraceContext( + traceparent: string | null | undefined, + tracestate?: string | null, +): TraceContext | undefined { + const match = traceparent ? TRACEPARENT.exec(traceparent.trim()) : null + + if (!match || match[1] === INVALID_TRACE_ID || match[2] === INVALID_PARENT_ID) { + return undefined + } + + const parent = match[0] + const state = tracestate?.trim() + + return state && state.length <= MAX_TRACESTATE_LENGTH + ? { traceparent: parent, tracestate: state } + : { traceparent: parent } +} + +/** The trace context recorded on `runtime.run.started`, or null when the run has none. */ +export function readStoredTrace(value: JsonValue | undefined): TraceContext | null { + return value === undefined ? null : Option.getOrNull(decodeTraceContext(value)) +} + +export function readTraceHeaders(headers: Headers): TraceContext | undefined { + return readTraceContext(headers.get('traceparent'), headers.get('tracestate')) +} + +/** The W3C headers for an outgoing request. */ +export function traceHeaders(trace: TraceContext | undefined): Record { + if (!trace) { + return {} + } + + return trace.tracestate + ? { traceparent: trace.traceparent, tracestate: trace.tracestate } + : { traceparent: trace.traceparent } +} diff --git a/packages/core/src/worker-callback.ts b/packages/core/src/worker-callback.ts index f93722c..cb9a60d 100644 --- a/packages/core/src/worker-callback.ts +++ b/packages/core/src/worker-callback.ts @@ -1,4 +1,6 @@ import type { EventInput } from './envelope' +import type { SecretContext } from './run-context' +import type { JsonValue } from './types' /** Identifies one attempt of an effect executed by an external worker. */ export interface WorkerCallback { @@ -8,8 +10,30 @@ export interface WorkerCallback { /** What an external worker reports about an effect attempt. */ export type WorkerCallbackInput = - | (WorkerCallback & { readonly kind: 'started' | 'heartbeat' | 'cancelled' }) + | (WorkerCallback & { readonly kind: 'started' | 'cancelled' }) + | (WorkerCallback & { readonly kind: 'heartbeat'; readonly details?: JsonValue }) | (WorkerCallback & { readonly kind: 'complete'; readonly events: ReadonlyArray }) - | (WorkerCallback & { readonly kind: 'fail'; readonly error: string }) + | (WorkerCallback & { + readonly kind: 'fail' + readonly error: string + /** When set, the attempt is terminal even if the retry policy has attempts left. */ + readonly nonRetryable?: boolean + /** Extra delay before the next attempt. The policy backoff is never shortened. */ + readonly retryAfterMs?: number + }) export type WorkerCallbackKind = WorkerCallbackInput['kind'] + +/** + * What the host tells the worker about the callback it just sent. + * A queued message cannot be recalled, so `cancelRequested` is how an in-flight attempt learns to stop. + */ +export interface WorkerCallbackResult { + readonly status: 'accepted' | 'stale' | 'cancelled' + readonly cancelRequested: boolean +} + +/** The run's secret context, released only to the attempt that is currently running. */ +export type WorkerSecretsResult = + | { readonly status: 'accepted'; readonly secrets: SecretContext } + | { readonly status: 'stale' | 'cancelled' } diff --git a/packages/debugger/src/events/protocol-summarize.ts b/packages/debugger/src/events/protocol-summarize.ts index 5ea918e..6d7dc01 100644 --- a/packages/debugger/src/events/protocol-summarize.ts +++ b/packages/debugger/src/events/protocol-summarize.ts @@ -34,6 +34,13 @@ export function summarizeProtocolEvent(event: DebuggerEvent): EventSummary | und title: 'run launched', detail: `${jsonText(payload.kind) ?? 'run'}:${jsonText(payload.definitionName) ?? 'unknown'} · ${jsonText(payload.runId) ?? 'unknown'}`, } + case 'runtime.child.settled': + return { + title: 'child settled', + detail: `${jsonText(payload.status) ?? 'done'} · ${jsonText(payload.runId) ?? ''}`, + } + case 'runtime.child.close.requested': + return { title: 'child close requested', detail: jsonText(payload.policy) } case 'runtime.run.forked': return { title: 'run forked', diff --git a/packages/evaluator/src/definitions-store.ts b/packages/evaluator/src/definitions-store.ts index 2aa44f6..13e86f9 100644 --- a/packages/evaluator/src/definitions-store.ts +++ b/packages/evaluator/src/definitions-store.ts @@ -1,10 +1,10 @@ import { Context, Layer } from 'effect' +import { makeDefinitionStore, type DefinitionStore } from '@looms/core' + import type { EvaluatorDefinition } from './definitions' -export interface EvaluatorDefinitionStore { - get(name: string): EvaluatorDefinition | undefined -} +export type EvaluatorDefinitionStore = DefinitionStore export class EvaluatorDefinitionsTag extends Context.Service< EvaluatorDefinitionsTag, @@ -14,9 +14,8 @@ export class EvaluatorDefinitionsTag extends Context.Service< export function makeEvaluatorDefinitionStore( definitions: ReadonlyArray, ): EvaluatorDefinitionStore { - const map = new Map(definitions.map((def) => [def.name, def])) - return { get: (name) => map.get(name) } + return makeDefinitionStore('evaluator', definitions) } -export const EvaluatorDefinitionsLive = (definitions: ReadonlyArray) => - Layer.succeed(EvaluatorDefinitionsTag, makeEvaluatorDefinitionStore(definitions)) +export const EvaluatorDefinitionsLive = (definitions: EvaluatorDefinitionStore) => + Layer.succeed(EvaluatorDefinitionsTag, definitions) diff --git a/packages/evaluator/src/definitions.ts b/packages/evaluator/src/definitions.ts index 233773c..83da692 100644 --- a/packages/evaluator/src/definitions.ts +++ b/packages/evaluator/src/definitions.ts @@ -1,7 +1,7 @@ import type { StandardSchemaV1 } from '@standard-schema/spec' import type { Schema } from 'effect' -import { defineKind, type JsonValue } from '@looms/core' +import { DEFAULT_DEFINITION_VERSION, defineKind, type JsonValue } from '@looms/core' import type { EvaluatorQuestions, @@ -13,6 +13,7 @@ import type { export interface EvaluatorDefinition { readonly kind: 'evaluator' readonly name: TName + readonly version: string readonly description?: string /** Provider model id. Omit it to use the adapter default. */ readonly model?: string @@ -26,7 +27,12 @@ export interface EvaluatorDefinition( - def: Omit, 'kind'>, + def: Omit, 'kind' | 'version'> & { + readonly version?: string + }, ): EvaluatorDefinition { - return defineKind('evaluator', def) + return defineKind('evaluator', { + ...def, + version: def.version ?? DEFAULT_DEFINITION_VERSION, + }) } diff --git a/packages/evaluator/src/effects.ts b/packages/evaluator/src/effects.ts index f3c4bd6..83311f8 100644 --- a/packages/evaluator/src/effects.ts +++ b/packages/evaluator/src/effects.ts @@ -1,6 +1,12 @@ import { Effect, Schema } from 'effect' -import { stringifyJson, validateInputEffect, type EventInputOf, type JsonValue } from '@looms/core' +import { + DEFAULT_DEFINITION_VERSION, + stringifyJson, + validateInputEffect, + type EventInputOf, + type JsonValue, +} from '@looms/core' import { EvaluatorDefinitionsTag } from './definitions-store' import { EvaluatorTag } from './evaluator' @@ -10,6 +16,7 @@ import type { EvaluatorAnswers, EvaluatorEvaluateResult, ResolvedEvaluatorDecisi const EvaluateInput = Schema.Struct({ evaluationId: Schema.String, definitionName: Schema.optional(Schema.String), + definitionVersion: Schema.optional(Schema.String), input: Schema.optional(Schema.Json), }) @@ -47,7 +54,11 @@ export const evaluateEffect = evaluatorModule.effect({ const threadId = ctx.threadId const definitions = yield* EvaluatorDefinitionsTag const evaluator = yield* EvaluatorTag - const definition = input.definitionName ? definitions.get(input.definitionName) : undefined + const version = input.definitionVersion ?? DEFAULT_DEFINITION_VERSION + + const definition = input.definitionName + ? yield* definitions.resolve(input.definitionName, version) + : undefined if (!definition) { const name = input.definitionName ?? 'unknown' diff --git a/packages/evaluator/src/module.test.ts b/packages/evaluator/src/module.test.ts index 5f322b1..89088a0 100644 --- a/packages/evaluator/src/module.test.ts +++ b/packages/evaluator/src/module.test.ts @@ -76,6 +76,7 @@ describe('@looms/evaluator module', () => { test('defineEvaluator sets kind', () => { expect(scoreRefund.kind).toBe('evaluator') expect(scoreRefund.name).toBe('score-refund') + expect(scoreRefund.version).toBe('v1') }) test('composes and exposes evaluate handler', () => { diff --git a/packages/evaluator/src/module.ts b/packages/evaluator/src/module.ts index 96cdcbc..3004df0 100644 --- a/packages/evaluator/src/module.ts +++ b/packages/evaluator/src/module.ts @@ -1,6 +1,6 @@ import { Layer } from 'effect' -import { defineModule } from '@looms/core' +import { defineModule, makeDefinitionStore } from '@looms/core' import type { EvaluatorDefinition } from './definitions' import { EvaluatorDefinitionsLive } from './definitions-store' @@ -24,14 +24,15 @@ export interface EvaluatorModuleOptions { } export function evaluator(options: EvaluatorModuleOptions = {}) { - const definitions = options.definitions ?? [] + const listed = options.definitions ?? [] + const definitions = makeDefinitionStore('evaluator', listed) const evaluatorLayer = options.evaluator ? Layer.succeed(EvaluatorTag, evaluatorFromAdapter(options.evaluator)) : StubEvaluatorLive(options.evaluatorPolicy) return defineModule(evaluatorModule, () => ({ - definitions, + definitions: listed, threads: { evaluator: evaluatorThread }, effects: { evaluate: evaluateEffect, @@ -40,5 +41,8 @@ export function evaluator(options: EvaluatorModuleOptions = {}) { evaluations, }, services: () => Layer.merge(evaluatorLayer, EvaluatorDefinitionsLive(definitions)), + bindHost(host) { + definitions.useResolver(host.definitionResolver) + }, })) } diff --git a/packages/evaluator/src/threads.ts b/packages/evaluator/src/threads.ts index 737d8a4..8905e54 100644 --- a/packages/evaluator/src/threads.ts +++ b/packages/evaluator/src/threads.ts @@ -17,6 +17,7 @@ import { evaluatorModule } from './scope' export const EvaluatorThreadStateSchema = Schema.Struct({ definitionName: Schema.optional(Schema.String), + definitionVersion: Schema.optional(Schema.String), input: Schema.Json, evaluationId: Schema.String, dispatched: Schema.Boolean, @@ -42,6 +43,7 @@ export const evaluatorThread = evaluatorModule.thread({ shape: EvaluatorThreadStateSchema, initialState: (ctx) => ({ definitionName: ctx.definitionName, + definitionVersion: ctx.definitionVersion, input: ctx.input, evaluationId: `eval_${ctx.threadId}`, dispatched: false, @@ -59,6 +61,7 @@ export const evaluatorThread = evaluatorModule.thread({ return { ...state, definitionName: event.payload.definitionName || state.definitionName, + definitionVersion: event.payload.definitionVersion || state.definitionVersion, input: event.payload.input ?? state.input, evaluationId: `eval_${ctx.threadId}`, } @@ -139,6 +142,7 @@ export const evaluatorThread = evaluatorModule.thread({ { evaluationId: state.evaluationId, definitionName: state.definitionName, + definitionVersion: state.definitionVersion, input: state.input, }, `evaluate_${state.evaluationId}`, diff --git a/packages/react/src/materialize.test.ts b/packages/react/src/materialize.test.ts index 0cc08ab..58e9107 100644 --- a/packages/react/src/materialize.test.ts +++ b/packages/react/src/materialize.test.ts @@ -82,6 +82,7 @@ describe('materializeEvents', () => { kind: 'agent', definitionName: 'assistant', launchedBy, + attributes: {}, }) }) diff --git a/packages/react/src/materialize.ts b/packages/react/src/materialize.ts index 545242c..c92a3f3 100644 --- a/packages/react/src/materialize.ts +++ b/packages/react/src/materialize.ts @@ -2,6 +2,7 @@ import { completedRunStatus, isJsonObject, isJsonString, + readAttributes, readRunLaunchSource, type EventEnvelope, type JsonValue, @@ -29,6 +30,7 @@ function setRunStatus(tables: MaterializedTables, runId: string, status: RunStat kind: existing?.kind ?? null, definitionName: existing?.definitionName ?? null, launchedBy: existing?.launchedBy ?? null, + attributes: existing?.attributes ?? {}, }) } @@ -56,6 +58,7 @@ export function materializeEvents( kind: readString(payload, 'kind') ?? existing?.kind ?? null, definitionName: readString(payload, 'definitionName') ?? existing?.definitionName ?? null, launchedBy: readRunLaunchSource(payload.launchedBy) ?? existing?.launchedBy ?? null, + attributes: readAttributes(payload.attributes), } tables.runs.set(event.runId, next) diff --git a/packages/react/src/react.ts b/packages/react/src/react.ts index 89f2ae1..35bc5d0 100644 --- a/packages/react/src/react.ts +++ b/packages/react/src/react.ts @@ -13,13 +13,16 @@ import { import { isTerminalStatus, + sameStringRecord, threadTree, toThreadTree, type AnyEventEnvelope, type ProjectionDefinition, type RegisteredEvent, + type RunAttributes, type RunLaunchSource, type RunStatus, + type SubscriptionTransport, type ThreadTree, } from '@looms/core' @@ -35,11 +38,13 @@ export interface LoomsProviderProps { children: ReactNode endpoint?: string coalesce?: CoalesceOption + transport?: SubscriptionTransport } interface ProviderContextValue { endpoint: string coalesce?: CoalesceOption + transport?: SubscriptionTransport } const LoomsContext = createContext({ endpoint: '' }) @@ -53,8 +58,23 @@ interface RegistryEntry { const registry = new Map() -function registryKey(endpoint: string, runId: string): string { - return `${endpoint}\0${runId}` +interface StoreIdentity { + readonly endpoint: string + readonly reconnectDelayMs?: number + readonly coalesce?: CoalesceOption + readonly transport?: SubscriptionTransport + readonly threadId?: string + readonly includeEphemeral?: boolean +} + +function registryKey(runId: string, identity: StoreIdentity): string { + return [ + identity.endpoint, + runId, + identity.transport ?? '', + identity.threadId ?? '', + identity.includeEphemeral === false ? '0' : '1', + ].join('\0') } function disposeEntry(key: string): void { @@ -74,13 +94,8 @@ function scheduleDisposal(key: string): Fiber.Fiber { ) } -function peekStore( - runId: string, - endpoint: string, - reconnectDelayMs?: number, - coalesce?: CoalesceOption, -): LoomsClientStore { - const key = registryKey(endpoint, runId) +function peekStore(runId: string, identity: StoreIdentity): LoomsClientStore { + const key = registryKey(runId, identity) const existing = registry.get(key) if (existing) { @@ -89,9 +104,12 @@ function peekStore( const store = createLoomsStore({ storeId: runId, - endpoint, - reconnectDelayMs, - coalesce, + endpoint: identity.endpoint, + reconnectDelayMs: identity.reconnectDelayMs, + coalesce: identity.coalesce, + transport: identity.transport, + threadId: identity.threadId, + includeEphemeral: identity.includeEphemeral, }) registry.set(key, { @@ -103,14 +121,9 @@ function peekStore( return store } -function retainStore( - runId: string, - endpoint: string, - reconnectDelayMs?: number, - coalesce?: CoalesceOption, -): void { - const key = registryKey(endpoint, runId) - peekStore(runId, endpoint, reconnectDelayMs, coalesce) +function retainStore(runId: string, identity: StoreIdentity): void { + const key = registryKey(runId, identity) + peekStore(runId, identity) const entry = registry.get(key) if (!entry) { @@ -125,8 +138,8 @@ function retainStore( } } -function releaseStore(runId: string, endpoint: string): void { - const key = registryKey(endpoint, runId) +function releaseStore(runId: string, identity: StoreIdentity): void { + const key = registryKey(runId, identity) const entry = registry.get(key) if (!entry) { @@ -141,8 +154,13 @@ function releaseStore(runId: string, endpoint: string): void { } /** Root provider that supplies the Looms host endpoint to run stores. */ -export function LoomsProvider({ children, endpoint = '', coalesce }: LoomsProviderProps) { - const value = useMemo(() => ({ endpoint, coalesce }), [endpoint, coalesce]) +export function LoomsProvider({ + children, + endpoint = '', + coalesce, + transport, +}: LoomsProviderProps) { + const value = useMemo(() => ({ endpoint, coalesce, transport }), [endpoint, coalesce, transport]) return createElement(LoomsContext.Provider, { value, children }) } @@ -150,6 +168,9 @@ export interface UseRunStoreOptions { endpoint?: string reconnectDelayMs?: number coalesce?: CoalesceOption + transport?: SubscriptionTransport + threadId?: string + includeEphemeral?: boolean } /** @@ -162,12 +183,25 @@ export function useRunStore(runId: string, options?: UseRunStoreOptions): LoomsC const endpoint = options?.endpoint ?? context.endpoint const reconnectDelayMs = options?.reconnectDelayMs const coalesce = options?.coalesce ?? context.coalesce - const store = peekStore(runId, endpoint, reconnectDelayMs, coalesce) + const transport = options?.transport ?? context.transport + const threadId = options?.threadId + const includeEphemeral = options?.includeEphemeral + + const identity = { + endpoint, + reconnectDelayMs, + coalesce, + transport, + threadId, + includeEphemeral, + } + + const store = peekStore(runId, identity) useEffect(() => { - retainStore(runId, endpoint, reconnectDelayMs, coalesce) - return () => releaseStore(runId, endpoint) - }, [runId, endpoint, reconnectDelayMs, coalesce]) + retainStore(runId, identity) + return () => releaseStore(runId, identity) + }, [runId, endpoint, reconnectDelayMs, coalesce, transport, threadId, includeEphemeral]) return store } @@ -232,6 +266,7 @@ export interface RunSummary { readonly startedAt: number | undefined /** The effect in another run that launched this one. */ readonly launchedBy: RunLaunchSource | undefined + readonly attributes: RunAttributes readonly connection: LoomsConnectionStatus } @@ -243,6 +278,7 @@ function shallowEqualSummary(a: RunSummary, b: RunSummary): boolean { a.rootThreadId === b.rootThreadId && a.startedAt === b.startedAt && a.launchedBy === b.launchedBy && + sameStringRecord(a.attributes, b.attributes) && a.connection === b.connection ) } @@ -264,6 +300,7 @@ export function useRunSummary rootThreadId: run?.rootThreadId ?? undefined, startedAt: s.query.startedAt(), launchedBy: run?.launchedBy ?? undefined, + attributes: run?.attributes ?? {}, connection: s.status(), } }, []) diff --git a/packages/react/src/store.test.ts b/packages/react/src/store.test.ts index a277ef2..d977fc4 100644 --- a/packages/react/src/store.test.ts +++ b/packages/react/src/store.test.ts @@ -285,3 +285,65 @@ describe('createLoomsStore reconnect', () => { } }) }) + +describe('createLoomsStore websocket', () => { + test('applies batches and resumes from a resync head', async () => { + class FakeSocket { + static latest: FakeSocket | undefined + readonly url: string + private readonly listeners = new Map< + string, + Array<(event: { readonly data?: unknown }) => void> + >() + + constructor(url: string) { + this.url = url + FakeSocket.latest = this + } + + addEventListener(type: string, listener: (event: { readonly data?: unknown }) => void): void { + const current = this.listeners.get(type) ?? [] + current.push(listener) + this.listeners.set(type, current) + } + + emit(type: string, data?: string): void { + for (const listener of this.listeners.get(type) ?? []) { + listener({ data }) + } + } + + close(): void { + this.emit('close') + } + } + + const store = createLoomsStore({ + storeId: 'run_1', + endpoint: 'http://127.0.0.1:9', + transport: 'websocket', + reconnectDelayMs: 20, + webSocket: FakeSocket, + }) + + await waitFor(() => FakeSocket.latest !== undefined) + const first = FakeSocket.latest + + if (!first) { + throw new Error('missing socket') + } + + first.emit('message', JSON.stringify({ type: 'hello', head: 1, tail: 0 })) + expect(store.events()).toHaveLength(0) + + first.emit( + 'message', + JSON.stringify({ batch: [encodeLoomsEvent(event(1, 'runtime.run.started'))] }), + ) + + await waitFor(() => store.events().length === 1) + first.emit('message', JSON.stringify({ type: 'resync', head: 4, reason: 'trimmed' })) + await waitFor(() => FakeSocket.latest?.url.includes('fromSeq=4') === true) + store.dispose() + }) +}) diff --git a/packages/react/src/store.ts b/packages/react/src/store.ts index 4f539ef..0e9d375 100644 --- a/packages/react/src/store.ts +++ b/packages/react/src/store.ts @@ -1,19 +1,22 @@ import { Data, Effect, Fiber, Option, Predicate, Schema } from 'effect' +import type { SubscriptionSocketConstructor } from '@looms/client' import { decodeLoomsEvent, + readSubscriptionData, stringifyJson, type AnyEventEnvelope, type EventEnvelope, type EventInput, type ProjectionDefinition, type RegisteredEvent, + type SubscriptionTransport, } from '@looms/core' import { EventIndex, ProjectionCache, type LiveFoldDefinition } from './derived' import { materializeEvents } from './materialize' import { createNotifyScheduler, type CoalesceOption } from './notify' -import { consumeSseStreamEffect, eventsFromSseData } from './sse' +import { consumeSseStreamEffect } from './sse' import { emptyTables, type MaterializedTables } from './tables' export type { CoalesceOption } from './notify' @@ -41,10 +44,18 @@ export interface LoomsClientStoreOptions { * Opens `${endpoint}/runs/:id/events?live=true` as an SSE stream. */ endpoint: string - /** Backoff between SSE reconnects. */ + /** Backoff between reconnects. The delay stays fixed, matching the SSE client. */ reconnectDelayMs?: number fetch?: typeof fetch coalesce?: CoalesceOption + /** + * `undefined` keeps the current SSE path. `websocket` and `auto` use a hibernating + * socket when a constructor is available. + */ + transport?: SubscriptionTransport + threadId?: string + includeEphemeral?: boolean + webSocket?: SubscriptionSocketConstructor } export interface LoomsStoreQuery { @@ -74,6 +85,24 @@ export interface LoomsClientStore void } +function socketConstructor( + options: LoomsClientStoreOptions, +): SubscriptionSocketConstructor | undefined { + if (options.transport !== 'websocket' && options.transport !== 'auto') { + return undefined + } + + if (options.webSocket) { + return options.webSocket + } + + if (!('WebSocket' in globalThis) || !Predicate.isFunction(globalThis.WebSocket)) { + return undefined + } + + return globalThis.WebSocket +} + function eventSourceCtor(): typeof EventSource | undefined { if (typeof EventSource === 'undefined') { return undefined @@ -108,6 +137,7 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS let liveStarted = false let liveAbort: AbortController | undefined let eventSource: EventSource | undefined + let socket: { close(code?: number, reason?: string): void } | undefined let reconnectFiber: Fiber.Fiber | undefined let syncing: Promise | null = null @@ -169,12 +199,39 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS notify.schedule() } - const applyEncoded = (data: string) => { - applyBatch(eventsFromSseData(data, options.storeId)) + const applyLiveData = (data: string): 'continue' | 'resync' => { + const read = readSubscriptionData(data, options.storeId) + + switch (read.kind) { + case 'events': + applyBatch(read.events.slice()) + return 'continue' + case 'resync': + fromSeq = read.head + return 'resync' + case 'hello': + case 'ignore': + return 'continue' + + default: { + const unexpected: never = read + return unexpected + } + } } const liveUrl = () => { - return `${endpoint}/runs/${encodeURIComponent(options.storeId)}/events?live=true&fromSeq=${fromSeq}` + const params = new URLSearchParams({ live: 'true', fromSeq: String(fromSeq) }) + + if (options.threadId) { + params.set('threadId', options.threadId) + } + + if (options.includeEphemeral === false) { + params.set('includeEphemeral', 'false') + } + + return `${endpoint}/runs/${encodeURIComponent(options.storeId)}/events?${params.toString()}` } const pull = Effect.gen(function* () { @@ -238,7 +295,7 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS return syncing } - const scheduleReconnect = (Ctor: typeof EventSource) => { + const scheduleReconnect = (start: () => void) => { if (disposed || !liveStarted || reconnectFiber !== undefined) { return } @@ -258,7 +315,7 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS }).pipe(Effect.ignore) if (!disposed && liveStarted) { - startEventSource(Ctor) + start() } }), ) @@ -278,7 +335,19 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS }) es.addEventListener('message', (msg) => { - applyEncoded(msg.data) + if (applyLiveData(msg.data) === 'resync') { + es.close() + + if (eventSource === es) { + eventSource = undefined + } + + setStatus('reconnecting') + + scheduleReconnect(() => { + startEventSource(Ctor) + }) + } }) es.addEventListener('error', () => { @@ -295,7 +364,9 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS eventSource = undefined } - scheduleReconnect(Ctor) + scheduleReconnect(() => { + startEventSource(Ctor) + }) } else if (!live) { setStatus('reconnecting') } @@ -336,9 +407,9 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS live = true setStatus('live') - return yield* consumeSseStreamEffect(body, (frame) => { - applyEncoded(frame.data) - }) + return yield* consumeSseStreamEffect(body, (frame) => + applyLiveData(frame.data) === 'resync' ? false : undefined, + ) }).pipe( Effect.catch(() => Effect.sync(() => { @@ -362,6 +433,57 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS void Effect.runPromise(run, { signal: controller.signal }).catch(() => undefined) } + const startWebSocket = (Ctor: SubscriptionSocketConstructor) => { + if (disposed || !liveStarted) { + return + } + + let url: string + + try { + const http = new URL(liveUrl()) + http.protocol = http.protocol === 'https:' ? 'wss:' : 'ws:' + url = http.toString() + } catch { + startFetchStream() + return + } + + const ws = new Ctor(url) + socket = ws + + ws.addEventListener('open', () => { + live = true + setStatus('live') + }) + + ws.addEventListener('message', (event) => { + if (!Predicate.isString(event.data)) { + return + } + + if (applyLiveData(event.data) === 'resync') { + ws.close() + } + }) + + ws.addEventListener('close', () => { + live = false + + if (socket === ws) { + socket = undefined + } + + if (!disposed && liveStarted) { + setStatus('reconnecting') + + scheduleReconnect(() => { + startWebSocket(Ctor) + }) + } + }) + } + const startLive = () => { if (!options.storeId || (liveStarted && !disposed)) { return @@ -369,6 +491,13 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS liveStarted = true disposed = false + const socketCtor = socketConstructor(options) + + if (socketCtor) { + startWebSocket(socketCtor) + return + } + const ctor = options.fetch === undefined ? eventSourceCtor() : undefined if (ctor) { @@ -389,6 +518,8 @@ export function createLoomsStore(options: LoomsClientStoreOptions): LoomsClientS eventSource?.close() eventSource = undefined + socket?.close() + socket = undefined liveAbort?.abort() liveAbort = undefined } diff --git a/packages/react/src/tables.ts b/packages/react/src/tables.ts index b78ff95..9fef0d4 100644 --- a/packages/react/src/tables.ts +++ b/packages/react/src/tables.ts @@ -1,4 +1,4 @@ -import type { RunLaunchSource, RunStatus, ThreadStatus } from '@looms/core' +import type { RunAttributes, RunLaunchSource, RunStatus, ThreadStatus } from '@looms/core' export interface RunRow { runId: string @@ -8,6 +8,7 @@ export interface RunRow { definitionName: string | null /** The effect in another run that launched this one. */ launchedBy: RunLaunchSource | null + attributes: RunAttributes } export interface ThreadRow { diff --git a/packages/runtime/src/blob-offload.test.ts b/packages/runtime/src/blob-offload.test.ts new file mode 100644 index 0000000..a5a9a9e --- /dev/null +++ b/packages/runtime/src/blob-offload.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { + EventPayloadTooLargeError, + makeMemoryEventStore, + makeMemorySnapshotStore, + memoryBlobStore, +} from '@looms/core' +import { defineWorkflow, workflow } from '@looms/workflow' + +import { createLooms } from './looms' + +const echo = defineWorkflow({ + name: 'blob-echo', + nodes: [{ id: 'done', run: () => null }], +}) + +describe('createLooms blob offload', () => { + test('rejects an event over 1 MiB when no blob store is configured', async () => { + const looms = createLooms({ modules: [workflow({ definitions: [echo] })] }) + const input = { blob: 'x'.repeat(1_500_000) } + + const error = await looms.start(echo, input, { runId: 'too-big' }).then( + () => null, + (cause) => cause, + ) + + expect(error).toBeInstanceOf(EventPayloadTooLargeError) + await looms.stop() + }) + + test('runs a payload over the event limit and reloads it from the offloaded snapshot', async () => { + const inner = await Effect.runPromise(makeMemoryEventStore) + const snapshots = await Effect.runPromise(makeMemorySnapshotStore) + const blobs = memoryBlobStore() + const large = 'x'.repeat(1_500_000) + const input = { blob: large } + + const looms = createLooms({ + modules: [workflow({ definitions: [echo] })], + store: inner, + snapshotStore: snapshots, + blobStore: blobs, + }) + + try { + const started = await looms.start(echo, input, { runId: 'big' }) + expect(started.state.status).toBe('completed') + expect(started.state.startIdentity?.input).toEqual(input) + + const rawEvents = JSON.stringify(await Effect.runPromise(inner.read('big'))) + expect(rawEvents).toContain('"$looms":"blob"') + expect(rawEvents).not.toContain(large) + + const rawSnapshot = await Effect.runPromise(snapshots.loadLatest('big')) + expect(JSON.stringify(rawSnapshot)).toContain('"$looms":"blob"') + expect(JSON.stringify(rawSnapshot)).not.toContain(large) + + const events = await looms.getEvents('big') + expect(JSON.stringify(events)).toContain(large) + } finally { + await looms.stop() + } + + const reloaded = createLooms({ + modules: [workflow({ definitions: [echo] })], + store: inner, + snapshotStore: snapshots, + blobStore: blobs, + rescanTimers: false, + }) + + try { + const state = await reloaded.getRun('big') + expect(state.startIdentity?.input).toEqual(input) + } finally { + await reloaded.stop() + } + }) +}) diff --git a/packages/runtime/src/cancellation.ts b/packages/runtime/src/cancellation.ts index e16d237..60b9669 100644 --- a/packages/runtime/src/cancellation.ts +++ b/packages/runtime/src/cancellation.ts @@ -9,6 +9,7 @@ import { type EffectExecutionRecord, type EventEnvelope, type OutstandingEffect, + type RunLauncher, type RunState, } from '@looms/core' @@ -29,6 +30,7 @@ export interface CancellationOptions { readonly observer?: RuntimeObserver readonly wake: LoomsRuntime['wake'] readonly withRunIngress: RunIngress + readonly launcher: () => RunLauncher } interface WorkerCancellation { @@ -146,6 +148,37 @@ export function createCancellation(options: CancellationOptions): LoomsRuntime[' const now = yield* Clock.currentTimeMillis return yield* repository.commit(store, runId, (current) => { + const rootCancel = threadId === undefined || threadId === current.state.rootThreadId + + const deferClose = + rootCancel && + !current.state.childCloseSent && + current.state.launchedRuns.some((child) => child.close === 'wait') + + if (deferClose) { + const requested = createEvent(runId, { + type: 'runtime.child.close.requested', + payload: { policy: 'wait' }, + threadId: current.state.rootThreadId, + origin: { type: 'system' }, + idempotencyKey: `child-close:${current.state.rootThreadId ?? runId}`, + }) + + return Effect.succeed({ + events: [requested], + value: { + plan: { + events: [], + threadIds: new Set(), + workerCancellations: [], + } satisfies CancellationPlan, + now, + childRunIds: current.state.launchedRuns.map((child) => child.runId), + }, + idempotencyKey: `signal:child-close:${current.state.rootThreadId ?? runId}`, + }) + } + const plan = planCancellation({ runId, seq: current.seq, @@ -156,16 +189,25 @@ export function createCancellation(options: CancellationOptions): LoomsRuntime[' now, }) + const childRunIds: readonly string[] = [] + return Effect.succeed({ events: plan.events, - value: { plan, now }, + value: { plan, now, childRunIds }, idempotencyKey: `signal:cancel:${threadId ?? current.state.rootThreadId ?? ''}`, }) }) }), ) - const { plan, now } = committed.value + const { plan, now, childRunIds } = committed.value + + if (childRunIds.length > 0) { + yield* Effect.forEach(childRunIds, (childRunId) => options.launcher().cancel(childRunId), { + concurrency: 1, + discard: true, + }) + } if (committed.append) { executionContexts.abort(runId, plan.threadIds, 'Effect cancelled') diff --git a/packages/runtime/src/child-run.test.ts b/packages/runtime/src/child-run.test.ts new file mode 100644 index 0000000..85340b7 --- /dev/null +++ b/packages/runtime/src/child-run.test.ts @@ -0,0 +1,235 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect, Predicate } from 'effect' + +import type { RunState } from '@looms/core' +import { defineWorkflow, workflow } from '@looms/workflow' + +import { createLooms } from './looms' + +const child = defineWorkflow({ + name: 'child-value', + nodes: [{ id: 'out', run: () => 7 }], + output: ({ results }) => results.out, +}) + +const parent = defineWorkflow({ + name: 'parent-value', + childExecution: 'run', + nodes: [{ id: 'spawn', run: (ctx) => ctx.spawn(child, {}) }], + output: ({ results }) => results.spawn, +}) + +const mappedChild = defineWorkflow({ + name: 'mapped-child', + nodes: [{ id: 'out', run: (ctx) => ctx.input }], + output: ({ input }) => input, +}) + +const mappedParent = defineWorkflow({ + name: 'mapped-parent', + childExecution: 'run', + nodes: [{ id: 'items', run: (ctx) => ctx.map(mappedChild, [1, 2]) }], + output: ({ results }) => results.items, +}) + +const sleepyChild = defineWorkflow({ + name: 'sleepy-child', + nodes: [{ id: 'nap', run: (ctx) => ctx.sleep(60_000) }], +}) + +function rootOutput(state: RunState) { + return state.rootThreadId ? state.threads[state.rootThreadId]?.output : undefined +} + +async function waitForStatus( + looms: { getRun(runId: string): Promise }, + runId: string, + status: RunState['status'], +) { + const deadline = Date.now() + 2_000 + let state = await looms.getRun(runId) + + while (state.status !== status && Date.now() < deadline) { + await Bun.sleep(10) + state = await looms.getRun(runId) + } + + return state +} + +describe('child runs', () => { + test('returns a child run result to the spawning node', async () => { + const looms = createLooms({ modules: [workflow({ definitions: [child, parent] })] }) + const started = await looms.start(parent, {}) + const state = await waitForStatus(looms, started.runId, 'completed') + const events = await looms.getEvents(started.runId) + const launched = events.find((event) => event.type === 'runtime.run.launched') + + const childRunId = + launched && Predicate.isObject(launched.payload) && Predicate.isString(launched.payload.runId) + ? launched.payload.runId + : undefined + + expect(state.status).toBe('completed') + expect(rootOutput(state)).toBe(7) + expect(childRunId).toEqual(expect.any(String)) + + const childState = await looms.getRun(childRunId ?? '') + expect(childState.status).toBe('completed') + expect(childState.startIdentity?.launchedBy?.runId).toBe(started.runId) + expect(events.some((event) => event.type === 'runtime.child.settled')).toBe(true) + + await looms.stop() + }) + + test('a child run inherits the parent run attributes and secrets', async () => { + const looms = createLooms({ modules: [workflow({ definitions: [child, parent] })] }) + + const started = await looms.start( + parent, + {}, + { + attributes: { tenant: 'acme' }, + secrets: { token: 'parent-secret' }, + }, + ) + + await waitForStatus(looms, started.runId, 'completed') + + const launched = (await looms.getEvents(started.runId)).find( + (event) => event.type === 'runtime.run.launched', + ) + + const childRunId = + launched && Predicate.isObject(launched.payload) && Predicate.isString(launched.payload.runId) + ? launched.payload.runId + : '' + + const store = await looms.store + + expect((await looms.getRun(childRunId)).attributes).toEqual({ tenant: 'acme' }) + + expect( + await Effect.runPromise(store.runSecrets?.get(childRunId) ?? Effect.succeed({})), + ).toEqual({ token: 'parent-secret' }) + + await looms.stop() + }) + + test('collects mapped child runs in item order', async () => { + const looms = createLooms({ + modules: [workflow({ definitions: [mappedChild, mappedParent] })], + }) + + const started = await looms.start(mappedParent, {}) + const state = await waitForStatus(looms, started.runId, 'completed') + const events = await looms.getEvents(started.runId) + const launched = events.filter((event) => event.type === 'runtime.run.launched') + + expect(state.status).toBe('completed') + expect(rootOutput(state)).toEqual([1, 2]) + expect(launched).toHaveLength(2) + + await looms.stop() + }) + + test('keeps the default of running a child in the parent run', async () => { + const threaded = defineWorkflow({ + name: 'threaded-parent', + nodes: [{ id: 'spawn', run: (ctx) => ctx.spawn(child, {}) }], + }) + + const looms = createLooms({ modules: [workflow({ definitions: [child, threaded] })] }) + const started = await looms.start(threaded, {}) + const events = await looms.getEvents(started.runId) + + expect(started.state.status).toBe('completed') + expect(events.some((event) => event.type === 'runtime.run.launched')).toBe(false) + expect(rootOutput(started.state)).toEqual({ spawn: 7 }) + + await looms.stop() + }) + + test('terminates child runs when the parent is cancelled', async () => { + const terminating = defineWorkflow({ + name: 'terminating-parent', + childExecution: 'run', + childClose: 'terminate', + nodes: [{ id: 'spawn', run: (ctx) => ctx.spawn(sleepyChild, {}) }], + }) + + const looms = createLooms({ + modules: [workflow({ definitions: [sleepyChild, terminating] })], + }) + + const started = await looms.start(terminating, {}) + const eventsBefore = await looms.getEvents(started.runId) + const launched = eventsBefore.find((event) => event.type === 'runtime.run.launched') + + const childRunId = + launched && Predicate.isObject(launched.payload) && Predicate.isString(launched.payload.runId) + ? launched.payload.runId + : '' + + expect(started.state.status).toBe('running') + + const cancelled = await looms.cancel(started.runId) + const events = await looms.getEvents(started.runId) + const childState = await looms.getRun(childRunId) + + expect(cancelled.status).toBe('cancelled') + expect(childState.status).toBe('cancelled') + + expect( + events.some( + (event) => + event.type === 'runtime.child.close.requested' && + Predicate.isObject(event.payload) && + event.payload.policy === 'wait', + ), + ).toBe(false) + + await looms.stop() + }) + + test('waits for child runs to finish cancelling', async () => { + const waiting = defineWorkflow({ + name: 'waiting-parent', + childExecution: 'run', + childClose: 'wait', + nodes: [{ id: 'spawn', run: (ctx) => ctx.spawn(sleepyChild, {}) }], + }) + + const looms = createLooms({ modules: [workflow({ definitions: [sleepyChild, waiting] })] }) + const started = await looms.start(waiting, {}) + + const launched = (await looms.getEvents(started.runId)).find( + (event) => event.type === 'runtime.run.launched', + ) + + const childRunId = + launched && Predicate.isObject(launched.payload) && Predicate.isString(launched.payload.runId) + ? launched.payload.runId + : '' + + const cancelled = await looms.cancel(started.runId) + const events = await looms.getEvents(started.runId) + const types = events.map((event) => event.type) + const childState = await looms.getRun(childRunId) + + expect(cancelled.status).toBe('cancelled') + expect(childState.status).toBe('cancelled') + expect(types.indexOf('runtime.child.close.requested')).toBeGreaterThanOrEqual(0) + + expect(types.indexOf('runtime.child.close.requested')).toBeLessThan( + types.indexOf('runtime.run.cancelled'), + ) + + expect(events.find((event) => event.type === 'runtime.child.close.requested')).toMatchObject({ + payload: { policy: 'wait' }, + }) + + await looms.stop() + }) +}) diff --git a/packages/runtime/src/child-run.ts b/packages/runtime/src/child-run.ts new file mode 100644 index 0000000..806b327 --- /dev/null +++ b/packages/runtime/src/child-run.ts @@ -0,0 +1,86 @@ +import { Effect } from 'effect' + +import { + descendantThreadIds, + isTerminalStatus, + type EventInput, + type RunLauncher, + type RunState, +} from '@looms/core' + +/** Tell the launching run that this child has reached a terminal status. */ +export function notifyLaunchingParent( + launcher: RunLauncher, + runId: string, + state: RunState, +): Effect.Effect { + const launchedBy = state.startIdentity?.launchedBy + const status = state.status + + if (!launchedBy || status === 'running') { + return Effect.void + } + + const root = state.rootThreadId ? state.threads[state.rootThreadId] : undefined + const error = status === 'completed' ? null : (root?.error ?? status) + + return launcher.signal( + launchedBy.runId, + [ + { + type: 'runtime.child.settled', + payload: { + runId, + status, + output: root?.output ?? null, + error, + }, + threadId: launchedBy.threadId, + idempotencyKey: `child-settled:${runId}`, + origin: { type: 'run', runId }, + }, + ], + { idempotencyKey: `child-settled:${runId}` }, + ) +} + +/** + * When a run asked to wait for child cancellation and those children are gone, cancel the + * run's own threads. The caller appends these events from the parent wake, which must not + * call `runtime.cancel` on itself. + */ +export function childCloseCancellations(state: RunState): EventInput[] { + if (!state.closeWaiting || state.launchedRuns.some((run) => run.close === 'wait')) { + return [] + } + + const rootId = state.rootThreadId + + if (!rootId) { + return [] + } + + const root = state.threads[rootId] + + if (!root || isTerminalStatus(root.status)) { + return [] + } + + const ids = [...descendantThreadIds(state, rootId).toReversed(), rootId] + + return ids.flatMap((threadId) => { + const record = state.threads[threadId] + + if (!record || isTerminalStatus(record.status)) { + return [] + } + + return [ + { + type: 'runtime.thread.cancelled', + payload: { threadId, reason: 'cancelled' }, + threadId, + }, + ] + }) +} diff --git a/packages/runtime/src/definition-resolver.test.ts b/packages/runtime/src/definition-resolver.test.ts new file mode 100644 index 0000000..2922f0c --- /dev/null +++ b/packages/runtime/src/definition-resolver.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { + makeMemoryEventStore, + type DefinitionResolver, + type RunSummary, + type SnapshotStore, +} from '@looms/core' +import { defineWorkflow, workflow } from '@looms/workflow' + +import { UnknownDefinitionError } from './helpers' +import { createLooms } from './looms' + +function discardingSnapshots(): SnapshotStore { + const headers: RunSummary[] = [] + + return { + loadLatest: () => Effect.succeedNone, + save: () => Effect.void, + saveHeader: (header) => + Effect.sync(() => { + const index = headers.findIndex((item) => item.runId === header.runId) + + if (index >= 0) { + headers[index] = header + return + } + + headers.push(header) + }), + get listHeaders() { + return Effect.succeed(headers) + }, + } +} + +describe('definition resolver', () => { + test('runs a workflow supplied by the host and resolves it again on replay', async () => { + const dynamic = defineWorkflow({ + name: 'dynamic', + version: 'v2', + nodes: [{ id: 'n', run: () => 'ok' }], + }) + + let calls = 0 + + const resolver: DefinitionResolver = { + peek: () => undefined, + resolve: (kind, name, version) => + Effect.sync(() => { + calls += 1 + + if (kind === 'workflow' && name === 'dynamic' && version === 'v2') { + return dynamic + } + + return undefined + }), + } + + const store = await Effect.runPromise(makeMemoryEventStore) + const snapshots = discardingSnapshots() + + const looms = createLooms({ + modules: [workflow()], + store, + snapshotStore: snapshots, + definitionResolver: resolver, + rescanTimers: false, + }) + + try { + const started = await looms.start(dynamic, null, { runId: 'dyn' }) + expect(started.state.status).toBe('completed') + expect(started.state.startIdentity?.definitionVersion).toBe('v2') + const afterStart = calls + + const reloaded = createLooms({ + modules: [workflow()], + store, + snapshotStore: snapshots, + definitionResolver: resolver, + rescanTimers: false, + }) + + try { + const state = await reloaded.getRun('dyn') + expect(state.status).toBe('completed') + expect(state.startIdentity?.definitionVersion).toBe('v2') + expect(calls).toBeGreaterThan(afterStart) + } finally { + await reloaded.stop() + } + } finally { + await looms.stop() + } + }) + + test('rejects an unknown definition before writing events', async () => { + const missing = defineWorkflow({ + name: 'missing', + nodes: [{ id: 'n', run: () => null }], + }) + + const store = await Effect.runPromise(makeMemoryEventStore) + + const looms = createLooms({ + modules: [workflow()], + store, + rescanTimers: false, + }) + + try { + const outcome = await looms.start(missing, null, { runId: 'missing' }).then( + () => undefined, + (cause: unknown) => cause, + ) + + expect(outcome).toBeInstanceOf(UnknownDefinitionError) + + const events = await Effect.runPromise(store.read('missing')) + expect(events).toHaveLength(0) + } finally { + await looms.stop() + } + }) +}) diff --git a/packages/runtime/src/dispatch-effect.ts b/packages/runtime/src/dispatch-effect.ts index 93a533d..e353393 100644 --- a/packages/runtime/src/dispatch-effect.ts +++ b/packages/runtime/src/dispatch-effect.ts @@ -1,25 +1,55 @@ -import type { Context, ManagedRuntime } from 'effect' -import { Effect } from 'effect' +import type { Context, ManagedRuntime, Types } from 'effect' +import { Effect, Predicate } from 'effect' import { asJson, isPrimitiveEffect, isWaitOnTimer, + readEffectFailure, type ComposedRegistry, type EffectContext, + type EffectFailedPayload, type EventInput, RunLauncherTag, - type RegisteredDefinition, + type DefinitionResolver, type RunLauncher, type RuntimeEffect, } from '@looms/core' import { threadStartedEvents } from './helpers' +function causeMessage(cause: unknown): string { + if (Predicate.isString(cause)) { + return cause + } + + if (cause instanceof Error) { + return cause.message + } + + return String(cause) +} + +function failedAttempt(effectId: string, threadId: string, cause: unknown): EventInput { + const failure = readEffectFailure(cause) + const error = failure?.error ?? causeMessage(cause) + const payload: Types.Mutable = { effectId, error } + + if (failure?.nonRetryable) { + payload.nonRetryable = true + } + + if (failure?.retryAfterMs !== undefined) { + payload.retryAfterMs = failure.retryAfterMs + } + + return { type: 'runtime.effect.failed', payload, threadId } +} + export function dispatchEffect( registry: ComposedRegistry, services: ManagedRuntime.ManagedRuntime, - definitions: ReadonlyMap, + definitions: DefinitionResolver, launcher: RunLauncher, effect: RuntimeEffect, ctx: EffectContext, @@ -99,6 +129,22 @@ export function dispatchEffect( ]) } + case 'runtime.launch': + return launcher + .launch({ + runId: eff.runId, + kind: eff.kind, + definitionName: eff.definitionName, + definitionVersion: eff.definitionVersion, + input: eff.input, + idempotencyKey: eff.runId, + close: eff.close, + }) + .pipe(Effect.map(() => [])) + + case 'runtime.cancelRun': + return launcher.cancel(eff.runId).pipe(Effect.map(() => [])) + default: { const exhaustiveCheck: never = eff return exhaustiveCheck @@ -110,11 +156,7 @@ export function dispatchEffect( if (!handler) { return Effect.succeed([ - { - type: 'runtime.effect.failed', - payload: { effectId: effCtx.effectId, error: `No handler for ${eff.type}` }, - threadId: currentThreadId, - }, + failedAttempt(effCtx.effectId, currentThreadId, `No handler for ${eff.type}`), ]) } @@ -129,18 +171,7 @@ export function dispatchEffect( }) return execution.pipe( - Effect.catch((err) => - Effect.succeed([ - { - type: 'runtime.effect.failed', - payload: { - effectId: effCtx.effectId, - error: err instanceof Error ? err.message : String(err), - }, - threadId: currentThreadId, - }, - ]), - ), + Effect.catch((err) => Effect.succeed([failedAttempt(effCtx.effectId, currentThreadId, err)])), ) } diff --git a/packages/runtime/src/errors.ts b/packages/runtime/src/errors.ts index 8d3611a..2e5b1a9 100644 --- a/packages/runtime/src/errors.ts +++ b/packages/runtime/src/errors.ts @@ -2,6 +2,7 @@ import { Data } from 'effect' import type { JsonValue, + RunAttributes, RunLaunchSource, RunStartIdentity as DurableRunStartIdentity, } from '@looms/core' @@ -15,6 +16,22 @@ export interface RunStartIdentity { readonly idempotencyKey: string | null readonly launchedBy: RunLaunchSource | null readonly correlationId: string | null + readonly attributes: RunAttributes +} + +/** The stored secret context differs. The message never includes the secret values. */ +export class SecretContextConflictError extends Data.TaggedError('SecretContextConflictError')<{ + readonly runId: string + readonly message: string +}> { + constructor(runId: string) { + super({ + runId, + message: `Run "${runId}" already has a different secret context`, + }) + + this.name = 'SecretContextConflictError' + } } export class StartRunConflictError extends Data.TaggedError('StartRunConflictError')<{ diff --git a/packages/runtime/src/events-api.test.ts b/packages/runtime/src/events-api.test.ts index ea89d41..d142656 100644 --- a/packages/runtime/src/events-api.test.ts +++ b/packages/runtime/src/events-api.test.ts @@ -61,10 +61,11 @@ describe('GET /runs/:id/events', () => { expect(res).not.toBeNull() expect(res!.headers.get('content-type')).toBe('text/event-stream') expect(res!.headers.get('cache-control')).toBe('no-cache, no-transform') - const frames = await readSseFrames(res!, 3) - expect(frames.length).toBeGreaterThanOrEqual(1) - expect(frames[0]).toContain('id: 1') - expect(frames[0]).toContain('data: ') + const frames = await readSseFrames(res!, 4) + const firstEvent = frames.find((frame) => frame.includes('id: 1')) + expect(firstEvent).toBeDefined() + expect(firstEvent).toContain('data: ') + expect(frames[0]).toContain('"type":"hello"') }) test('Last-Event-ID skips already-seen events', async () => { @@ -90,9 +91,11 @@ describe('GET /runs/:id/events', () => { ) expect(res).not.toBeNull() - const frames = await readSseFrames(res!, 1) - expect(frames[0]).toContain('id: 2') - expect(frames[0]).not.toContain('id: 1\n') + const frames = await readSseFrames(res!, 2) + const resumed = frames.find((frame) => frame.includes('id: 2')) + expect(resumed).toBeDefined() + expect(resumed).not.toContain('id: 1\n') + expect(frames.some((frame) => frame.includes('id: 1\n'))).toBe(false) }) test('non-live pull returns the event resource', async () => { diff --git a/packages/runtime/src/filesystem-blob-store.test.ts b/packages/runtime/src/filesystem-blob-store.test.ts new file mode 100644 index 0000000..fbb8cf4 --- /dev/null +++ b/packages/runtime/src/filesystem-blob-store.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, test } from 'bun:test' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { Effect } from 'effect' + +import { offloadJson, resolveJson } from '@looms/core' + +import { filesystemBlobStore } from './filesystem-blob-store' + +describe('filesystemBlobStore', () => { + test('round-trips an offloaded value', async () => { + const directory = await mkdtemp(join(tmpdir(), 'looms-blobs-')) + + try { + const blobs = filesystemBlobStore(directory) + const value = { text: 'z'.repeat(100) } + const stored = await Effect.runPromise(offloadJson(value, blobs, { thresholdBytes: 16 })) + const resolved = await Effect.runPromise(resolveJson(stored, blobs)) + + expect(resolved).toEqual(value) + expect(JSON.stringify(stored)).toContain('"$looms":"blob"') + } finally { + await rm(directory, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/runtime/src/filesystem-blob-store.ts b/packages/runtime/src/filesystem-blob-store.ts new file mode 100644 index 0000000..6fd7119 --- /dev/null +++ b/packages/runtime/src/filesystem-blob-store.ts @@ -0,0 +1,96 @@ +import { mkdir, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises' + +import { Data, Effect, Predicate } from 'effect' + +import { blobStoreFromBytes, BlobStoreError, type BlobBytes, type BlobStore } from '@looms/core' + +class MissingBlobFile extends Data.TaggedError('MissingBlobFile') {} + +/** + * Content-addressed blob store rooted at `directory`. For Bun and Node hosts. + * A Durable Object should use `r2BlobStore` instead. + */ +export function filesystemBlobStore(directory: string): BlobStore { + return blobStoreFromBytes(filesystemBlobBytes(directory)) +} + +function filesystemBlobBytes(directory: string): BlobBytes { + return { + read: (key) => readOptional(`${directory}/${key}`, key), + + write: (key, body) => + Effect.tryPromise({ + try: () => mkdir(`${directory}/${folderOf(key)}`, { recursive: true }), + catch: (cause) => + new BlobStoreError(`Unable to create a directory for blob key ${key}`, cause), + }).pipe( + Effect.andThen( + Effect.tryPromise({ + try: () => writeFile(`${directory}/${key}`, body), + catch: (cause) => new BlobStoreError(`Unable to write blob key ${key}`, cause), + }), + ), + ), + + remove: (key) => + Effect.tryPromise({ + try: () => rm(`${directory}/${key}`, { force: true }), + catch: (cause) => new BlobStoreError(`Unable to delete blob key ${key}`, cause), + }), + + exists: (key) => + Effect.tryPromise({ + try: () => stat(`${directory}/${key}`), + catch: (cause): BlobStoreError | MissingBlobFile => + isMissingFile(cause) + ? new MissingBlobFile() + : new BlobStoreError(`Unable to check blob key ${key}`, cause), + }).pipe( + Effect.as(true), + Effect.catchTag('MissingBlobFile', () => Effect.succeed(false)), + ), + + list: (prefix) => listFolder(directory, prefix), + } +} + +function readOptional(path: string, key: string): Effect.Effect { + return Effect.tryPromise({ + try: () => readFile(path), + catch: (cause): BlobStoreError | MissingBlobFile => + isMissingFile(cause) + ? new MissingBlobFile() + : new BlobStoreError(`Unable to read blob key ${key}`, cause), + }).pipe( + Effect.catchTag('MissingBlobFile', () => Effect.succeed(null)), + Effect.map((body) => (body === null ? null : new Uint8Array(body))), + ) +} + +function listFolder( + directory: string, + prefix: string, +): Effect.Effect, BlobStoreError> { + const folder = prefix.endsWith('/') ? prefix.slice(0, -1) : prefix + + return Effect.tryPromise({ + try: () => readdir(`${directory}/${folder}`), + catch: (cause): BlobStoreError | MissingBlobFile => + isMissingFile(cause) + ? new MissingBlobFile() + : new BlobStoreError(`Unable to list blob keys ${prefix}`, cause), + }).pipe( + Effect.catchTag('MissingBlobFile', () => Effect.succeed([])), + Effect.map((names) => names.map((name) => `${folder}/${name}`)), + ) +} + +function folderOf(key: string): string { + const slash = key.lastIndexOf('/') + + return slash < 0 ? '' : key.slice(0, slash) +} + +function isMissingFile(cause: unknown): boolean { + return Predicate.isObject(cause) && 'code' in cause && cause.code === 'ENOENT' +} diff --git a/packages/runtime/src/helpers.test.ts b/packages/runtime/src/helpers.test.ts index 81b5913..9d0b432 100644 --- a/packages/runtime/src/helpers.test.ts +++ b/packages/runtime/src/helpers.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from 'bun:test' import { Effect, Schema } from 'effect' -import { DEFAULT_DEFINITION_VERSION, definitionKey } from '@looms/core' +import { DEFAULT_DEFINITION_VERSION, staticDefinitionResolver } from '@looms/core' import { threadStartedEvents } from './helpers' @@ -15,12 +15,8 @@ describe('threadStartedEvents', () => { value: { kind: 'workflow', name: 'checkout', version: DEFAULT_DEFINITION_VERSION }, } - const definitions = new Map([ - [definitionKey('workflow', 'checkout', DEFAULT_DEFINITION_VERSION), def], - ]) - const events = await Effect.runPromise( - threadStartedEvents(definitions, { + threadStartedEvents(staticDefinitionResolver([def]), { kind: 'workflow', definitionName: 'checkout', definitionVersion: DEFAULT_DEFINITION_VERSION, @@ -43,12 +39,10 @@ describe('threadStartedEvents', () => { }) }) - test('rejects unknown definitions with explicit error', async () => { - const definitions = new Map() - + test('rejects unknown definitions with explicit error', () => { expect( Effect.runPromise( - threadStartedEvents(definitions, { + threadStartedEvents(staticDefinitionResolver([]), { kind: 'workflow', definitionName: 'checkout', definitionVersion: DEFAULT_DEFINITION_VERSION, @@ -69,12 +63,8 @@ describe('threadStartedEvents', () => { value: { kind: 'workflow', name: 'checkout', version: DEFAULT_DEFINITION_VERSION }, } - const definitions = new Map([ - [definitionKey('workflow', 'checkout', DEFAULT_DEFINITION_VERSION), def], - ]) - const events = await Effect.runPromise( - threadStartedEvents(definitions, { + threadStartedEvents(staticDefinitionResolver([def]), { kind: 'workflow', definitionName: 'checkout', definitionVersion: DEFAULT_DEFINITION_VERSION, diff --git a/packages/runtime/src/helpers.ts b/packages/runtime/src/helpers.ts index c3d4961..06262b0 100644 --- a/packages/runtime/src/helpers.ts +++ b/packages/runtime/src/helpers.ts @@ -1,4 +1,4 @@ -import { Data, Effect, Layer, Predicate, type Context } from 'effect' +import { Data, Effect, Layer, Predicate, type Context, type Types } from 'effect' import { asJson, @@ -13,6 +13,7 @@ import { validateEventsEffect, validateInputEffect, withSnapshotStore, + hasEntries, type AnyRuntimeModule, type AppendableEvent, type EventCatalog, @@ -21,28 +22,83 @@ import { type EventOrigin, type EventStore, InvalidEventError, + type DefinitionResolveError, + type DefinitionResolver, type JsonValue, type OutstandingEffect, - type RegisteredDefinition, + type RetryPolicy, + type SecretContext, + type TraceContext, type RunCursor, + type SchemaInput, + type VersionedDefinition, type RunState, type SnapshotStore, } from '@looms/core' -import type { EffectWorkerTask } from './types' +import type { EffectWorkerDeadlines, EffectWorkerTask } from './types' + +/** What a dispatch carries beyond the effect itself. */ +export interface WorkerTaskExtras { + /** Progress from the previous attempt's last heartbeat. */ + readonly details?: JsonValue | null + readonly secrets?: SecretContext + readonly trace?: TraceContext | null + readonly retry?: RetryPolicy + readonly scheduleToStartAt?: number | null +} export function workerTask( runId: string, item: OutstandingEffect, attempt: number, + extras: WorkerTaskExtras = {}, ): EffectWorkerTask { - return { + const task: Types.Mutable = { runId, effectId: item.effectId, attempt, type: item.effect.type, input: 'input' in item.effect ? item.effect.input : null, } + + if (extras.details != null) { + task.details = extras.details + } + + if (extras.secrets && hasEntries(extras.secrets)) { + task.secrets = extras.secrets + } + + if (extras.trace) { + task.trace = extras.trace + } + + const deadlines = workerDeadlines(extras) + + if (deadlines) { + task.deadlines = deadlines + } + + return task +} + +function workerDeadlines(extras: WorkerTaskExtras): EffectWorkerDeadlines | undefined { + const deadlines: Types.Mutable = {} + + if (extras.scheduleToStartAt != null) { + deadlines.scheduleToStartAt = extras.scheduleToStartAt + } + + if (extras.retry?.startToCloseTimeoutMs) { + deadlines.startToCloseTimeoutMs = extras.retry.startToCloseTimeoutMs + } + + if (extras.retry?.heartbeatTimeoutMs) { + deadlines.heartbeatTimeoutMs = extras.retry.heartbeatTimeoutMs + } + + return Object.keys(deadlines).length > 0 ? deadlines : undefined } export class UnknownDefinitionError extends Data.TaggedError('UnknownDefinitionError')<{ @@ -178,7 +234,7 @@ export function groupOutstanding( * event with raw input. */ export function threadStartedEvents( - definitions: ReadonlyMap, + definitions: DefinitionResolver, args: { kind: string definitionName: string @@ -187,11 +243,9 @@ export function threadStartedEvents( threadId: string parentThreadId: string | null }, -): Effect.Effect, UnknownDefinitionError> { +): Effect.Effect, UnknownDefinitionError | DefinitionResolveError> { return Effect.gen(function* () { - const def = definitions.get( - definitionKey(args.kind, args.definitionName, args.definitionVersion), - ) + const def = yield* definitions.resolve(args.kind, args.definitionName, args.definitionVersion) if (!def) { return yield* new UnknownDefinitionError( @@ -204,9 +258,10 @@ export function threadStartedEvents( const raw = args.input ?? null let startedInput = raw let validationError: string | undefined + const input = schemaInputOf(def) - if (def?.input) { - const validated = yield* validateInputEffect(def.input, raw).pipe( + if (input) { + const validated = yield* validateInputEffect(input, raw).pipe( Effect.match({ onFailure: (error) => ({ ok: false as const, error: error.message }), onSuccess: (value) => ({ ok: true as const, value }), @@ -249,6 +304,22 @@ export function threadStartedEvents( }) } +function schemaInputOf(definition: VersionedDefinition): SchemaInput | undefined { + if (!('input' in definition)) { + return undefined + } + + return isSchemaInput(definition.input) ? definition.input : undefined +} + +function isSchemaInput(value: unknown): value is SchemaInput { + if (value === null || (!Predicate.isObject(value) && !Predicate.isFunction(value))) { + return false + } + + return '~standard' in value || 'ast' in value +} + export function synthesizedThreadFailed( before: RunState, after: RunState, diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index 1f041f8..ac6ab09 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -11,6 +11,7 @@ export { StartRunConflictError, type CreateRuntimeOptions, type EffectWorker, + type EffectWorkerDeadlines, type EffectWorkerTask, type LoomsRuntime, type RunOperationalStatus, @@ -20,8 +21,10 @@ export { type WakeScheduler, type WorkerCallback, type WorkerCallbackInput, + type WorkerCallbackResult, } from './runtime' export { createLooms, type Looms, type CreateLoomsOptions, type StartResult } from './looms' +export { loadSubscriptionOpening, readSubscriptionBounds } from './subscription' export { serveHttp, createFetchHandler, @@ -29,8 +32,19 @@ export { type FetchHandlerOptions, authAllowed, authDenied, + authResponse, bearerAuth, defaultAuthorize, + grantAuth, + issueRunGrant, + grantSubprotocol, + runGrantToken, + verifyRunGrant, + GrantError, + type GrantAuthOptions, + type GrantFailure, + type RunGrant, + runRouteInfo, type AuthDecision, type AuthorizedAccess, type Authorize, @@ -40,5 +54,6 @@ export { type RunningServer, type LoomsFetchResult, } from './server' +export { filesystemBlobStore } from './filesystem-blob-store' export { createEventStreamResponse, type EventStreamOptions } from './sse' export { notifyObserver, type RuntimeObservation, type RuntimeObserver } from './observer' diff --git a/packages/runtime/src/local-effect-executor.ts b/packages/runtime/src/local-effect-executor.ts index e611713..d9eb578 100644 --- a/packages/runtime/src/local-effect-executor.ts +++ b/packages/runtime/src/local-effect-executor.ts @@ -9,10 +9,14 @@ import { type ComposedRegistry, type EventEnvelope, type EventInput, + type DefinitionResolver, type OutstandingEffect, - type RegisteredDefinition, + type LaunchRunArgs, + type RunAttributes, type RunLauncher, type RunLaunchSource, + type SecretContext, + type TraceContext, } from '@looms/core' import { dispatchEffect } from './dispatch-effect' @@ -31,7 +35,7 @@ import type { RunExecutionContext } from './run-execution-context' export interface LocalEffectExecutorOptions { readonly registry: ComposedRegistry readonly services: ManagedRuntime.ManagedRuntime - readonly definitions: ReadonlyMap + readonly definitionResolver: DefinitionResolver readonly executionContext: RunExecutionContext readonly liveAppends: RuntimeRepository['liveAppends'] readonly runInContext: (effect: Effect.Effect) => Promise @@ -39,15 +43,40 @@ export interface LocalEffectExecutorOptions { readonly launcher: RunLauncher } +/** Run-level values every effect sees. Runs the effects launch inherit them unless the launch overrides them. */ +export interface RunContext { + readonly correlationId: string + readonly attributes: RunAttributes + readonly secrets: SecretContext + readonly trace: TraceContext | null +} + export interface ExecuteLocalEffectsInput { readonly runId: string - /** Inherited by runs the effects launch. */ - readonly correlationId: string + readonly context: RunContext readonly outstanding: ReadonlyArray readonly attempts: ReadonlyMap readonly now: number } +function runLaunchedPayload(args: LaunchRunArgs, targetRunId: string, effectId: string) { + const definitionVersion = args.definitionVersion ?? DEFAULT_DEFINITION_VERSION + + const base = { + runId: targetRunId, + kind: args.kind, + definitionName: args.definitionName, + definitionVersion, + effectId, + } + + if (args.close === undefined) { + return base + } + + return { ...base, close: args.close } +} + /** * Stamps every launch with the effect that made it, replacing anything the handler passed, and * records a `runtime.run.launched` event for each launch the target accepts. @@ -55,7 +84,7 @@ export interface ExecuteLocalEffectsInput { function launcherFor( launcher: RunLauncher, runId: string, - correlationId: string, + context: RunContext, item: OutstandingEffect, launched: EventEnvelope[], ): RunLauncher { @@ -71,30 +100,36 @@ function launcherFor( launch: (args) => { const targetRunId = args.runId ?? createRunId() - return launcher.launch({ ...args, runId: targetRunId, launchedBy, correlationId }).pipe( - Effect.tap(() => - Effect.sync(() => { - launched.push( - createEvent(runId, { - type: 'runtime.run.launched', - payload: { - runId: targetRunId, - kind: args.kind, - definitionName: args.definitionName, - definitionVersion: args.definitionVersion ?? DEFAULT_DEFINITION_VERSION, - effectId: item.effectId, - }, - threadId: item.threadId, - causationId: item.causingEventId, - correlationId, - origin: { type: 'thread', threadId: item.threadId }, - }), - ) - }), - ), - ) + return launcher + .launch({ + ...args, + runId: targetRunId, + launchedBy, + correlationId: context.correlationId, + attributes: args.attributes ?? context.attributes, + secrets: args.secrets ?? context.secrets, + trace: args.trace ?? context.trace ?? undefined, + }) + .pipe( + Effect.tap(() => + Effect.sync(() => { + launched.push( + createEvent(runId, { + type: 'runtime.run.launched', + payload: runLaunchedPayload(args, targetRunId, item.effectId), + threadId: item.threadId, + causationId: item.causingEventId, + correlationId: context.correlationId, + origin: { type: 'thread', threadId: item.threadId }, + }), + ) + }), + ), + ) }, status: launcher.status, + signal: (targetRunId, events, options) => launcher.signal(targetRunId, events, options), + cancel: (targetRunId) => launcher.cancel(targetRunId), } } @@ -129,7 +164,7 @@ export function executeLocalEffects( const { registry, services, - definitions, + definitionResolver, executionContext, liveAppends, runInContext, @@ -182,14 +217,17 @@ export function executeLocalEffects( const dispatched = dispatchEffect( registry, services, - definitions, - launcherFor(launcher, input.runId, input.correlationId, item, launched), + definitionResolver, + launcherFor(launcher, input.runId, input.context, item, launched), item.effect, { effectId: item.effectId, runId: input.runId, threadId: item.threadId, causingEventId: item.causingEventId, + attributes: input.context.attributes, + secrets: input.context.secrets, + trace: input.context.trace, signal: controller.signal, emit: appendLive, }, @@ -264,7 +302,23 @@ export function executeLocalEffects( const { outcomes } = settled - if (outcomes.length === 0) { + const sideEffect = + item.effect.type === 'runtime.launch' || item.effect.type === 'runtime.cancelRun' + + if (outcomes.length === 0 && sideEffect) { + const attempt = input.attempts.get(item.effectId) ?? 1 + + produced.push( + createEvent(input.runId, { + type: 'runtime.effect.completed', + payload: { effectId: item.effectId, attempt }, + threadId: item.threadId, + effectId: item.effectId, + causationId: item.causingEventId, + origin: { type: 'system' }, + }), + ) + } else if (outcomes.length === 0) { produced.push(createEffectFailedEvent(input.runId, item, 'empty-outcome')) } @@ -273,6 +327,7 @@ export function executeLocalEffects( const attempt = input.attempts.get(item.effectId) ?? 1 const payload = Predicate.isObject(outcome.payload) ? outcome.payload : {} const error = Predicate.isString(payload.error) ? payload.error : 'effect failed' + const retryAfterMs = payload.retryAfterMs const failure = planAttemptFailure({ runId: input.runId, @@ -282,6 +337,8 @@ export function executeLocalEffects( now: input.now, retry, origin: { type: 'system' }, + nonRetryable: payload.nonRetryable === true, + retryAfterMs: Predicate.isNumber(retryAfterMs) ? retryAfterMs : undefined, }) if (failure.retrying) { diff --git a/packages/runtime/src/looms.ts b/packages/runtime/src/looms.ts index a1fc06d..ef6420e 100644 --- a/packages/runtime/src/looms.ts +++ b/packages/runtime/src/looms.ts @@ -3,14 +3,18 @@ import { Data, Effect, Predicate } from 'effect' import { EventStoreTag, makeMemoryEventStore, + makeMemorySnapshotStore, snapshotStoreOf, + withBlobOffload, SnapshotStoreTag, type AnyRuntimeModule, type DebuggerHost, type DefinitionInput, type DefinitionRef, + type DefinitionResolver, type EventEnvelope, type EventInput, + type BlobStore, type EventStore, type EventStoreTrimCoverage, type JsonValue, @@ -27,8 +31,10 @@ import { type EffectWorker, type LoomsRuntime, type RunOperationalStatus, + type StartRunArgs, type WakeScheduler, type WorkerCallbackInput, + type WorkerCallbackResult, } from './runtime' import { createFetchHandler, @@ -42,6 +48,11 @@ import { export interface CreateLoomsOptions { /** Runtime modules to enable. Pass `agent()`, `workflow()`, `approval()`, and/or your own. */ readonly modules?: readonly AnyRuntimeModule[] + /** + * Loads a definition that was not registered on a module. Registered definitions are checked + * first. A run pins the resolved version on its start event. + */ + readonly definitionResolver?: DefinitionResolver readonly store?: EventStore | Promise | (() => Promise) readonly serve?: boolean | { port?: number; hostname?: string } readonly maxWakeIterations?: number @@ -51,6 +62,11 @@ export interface CreateLoomsOptions { readonly snapshotEvery?: number /** Defaults to the EventStore's attached store (S2) or an in-memory SnapshotStore. */ readonly snapshotStore?: SnapshotStore + /** + * Content-addressed store for JSON values that would exceed the 1 MiB event limit or the + * 8 MiB snapshot limit. Reads resolve references before folds, projections, and callers see them. + */ + readonly blobStore?: BlobStore readonly trimAfterSnapshot?: { keepSnapshots: number coverage?: (runId: string) => EventStoreTrimCoverage | Promise @@ -64,6 +80,8 @@ export interface CreateLoomsOptions { /** Starts runs from effect handlers. Defaults to launching into this runtime's own store. */ readonly launcher?: RunLauncher readonly worker?: EffectWorker + /** Maximum concurrent in-flight worker effects dispatched per run (fairness guard). */ + readonly maxInFlightWorkerEffects?: number /** Failure-isolated runtime observation sink. */ readonly observer?: RuntimeObserver /** @@ -71,7 +89,8 @@ export interface CreateLoomsOptions { * and operations routes return 503; `bearerAuth({ worker, operations })` enables them. * * A mounted debugger is authorized as `{ access: 'read', name: 'debugger' }`. Bearer tokens on - * reads block browser navigation, so hosts that need auth should use a cookie or session policy. + * reads block browser navigation. `grantAuth` covers a browser EventSource or WebSocket that can + * only send a query parameter or a subprotocol. */ readonly authorize?: Authorize /** @@ -95,7 +114,14 @@ export interface Looms { start( definition: TDef, input?: DefinitionInput, - options?: { runId?: string; threadId?: string; idempotencyKey?: string }, + options?: { + runId?: string + threadId?: string + idempotencyKey?: string + attributes?: StartRunArgs['attributes'] + secrets?: StartRunArgs['secrets'] + trace?: StartRunArgs['trace'] + }, ): Promise /** Start a run by `{ kind, definitionName }` when you only have names (HTTP bodies, CLIs). */ startRun(args: { @@ -106,6 +132,9 @@ export interface Looms { runId?: string threadId?: string idempotencyKey?: string + attributes?: StartRunArgs['attributes'] + secrets?: StartRunArgs['secrets'] + trace?: StartRunArgs['trace'] }): Promise /** Commit a run's start and schedule a wake without waiting for it. */ launchRun(args: { @@ -116,6 +145,9 @@ export interface Looms { runId?: string threadId?: string idempotencyKey?: string + attributes?: StartRunArgs['attributes'] + secrets?: StartRunArgs['secrets'] + trace?: StartRunArgs['trace'] }): Promise getRun(runId: string): Promise getEvents(runId: string, options?: { fromSeq?: number; limit?: number }): Promise @@ -135,7 +167,7 @@ export interface Looms { cancel(runId: string, threadId?: string): Promise /** Start a new run that continues where a finished run ended. */ fork(sourceRunId: string, options?: { runId?: string }): Promise - workerCallback(runId: string, input: WorkerCallbackInput): Promise + workerCallback(runId: string, input: WorkerCallbackInput): Promise fetch(req: Request): Promise serve(options?: { port?: number; hostname?: string }): RunningServer stop(): Promise @@ -175,7 +207,7 @@ export function createLooms(options: CreateLoomsOptions = {}): Looms { Effect.gen(function* () { const configuredStore = options.store - const store = !configuredStore + let store = !configuredStore ? yield* makeMemoryEventStore : Predicate.isFunction(configuredStore) ? yield* Effect.tryPromise({ @@ -189,20 +221,31 @@ export function createLooms(options: CreateLoomsOptions = {}): Looms { }) : configuredStore + if (options.blobStore) { + const snapshots = + options.snapshotStore ?? snapshotStoreOf(store) ?? (yield* makeMemorySnapshotStore) + + store = withBlobOffload(store, options.blobStore, { snapshots }) + } + const runtime = createRuntime({ modules, store, maxWakeIterations: options.maxWakeIterations, maxPendingRunOperations: options.maxPendingRunOperations, snapshotEvery: options.snapshotEvery, - snapshotStore: options.snapshotStore ?? snapshotStoreOf(store), + snapshotStore: options.blobStore + ? snapshotStoreOf(store) + : (options.snapshotStore ?? snapshotStoreOf(store)), trimAfterSnapshot: options.trimAfterSnapshot, runCacheSize: options.runCacheSize, scheduler: options.scheduler, launcher: options.launcher, localLauncher: options.launcher === undefined, worker: options.worker, + maxInFlightWorkerEffects: options.maxInFlightWorkerEffects, observer: options.observer, + definitionResolver: options.definitionResolver, }) if (options.rescanTimers !== false) { @@ -262,6 +305,9 @@ export function createLooms(options: CreateLoomsOptions = {}): Looms { runId: startOpts?.runId, threadId: startOpts?.threadId, idempotencyKey: startOpts?.idempotencyKey, + attributes: startOpts?.attributes, + secrets: startOpts?.secrets, + trace: startOpts?.trace, }), ), getRun: (runId) => runEffect((i) => i.runtime.getRun(runId)), diff --git a/packages/runtime/src/planners.test.ts b/packages/runtime/src/planners.test.ts index 53eefd5..0d34708 100644 --- a/packages/runtime/src/planners.test.ts +++ b/packages/runtime/src/planners.test.ts @@ -24,12 +24,16 @@ function runningState(overrides: Partial = {}): RunState { rootThreadId: null, startIdentity: null, forkedFrom: null, + attributes: {}, threads: {}, waits: {}, outstandingEffects: [], effectExecutions: {}, completedEffectIds: [], processedIdempotencyKeys: [], + launchedRuns: [], + closeWaiting: false, + childCloseSent: false, ...overrides, } } @@ -63,6 +67,7 @@ describe('deadline transition planner', () => { nextAttemptAt: null, deadlineAt: 1_001, lastHeartbeatAt: null, + heartbeatDetails: null, lastError: null, }, }, @@ -95,6 +100,7 @@ describe('deadline transition planner', () => { nextAttemptAt: null, deadlineAt: 900, lastHeartbeatAt: null, + heartbeatDetails: null, lastError: null, }, future: { @@ -104,6 +110,7 @@ describe('deadline transition planner', () => { nextAttemptAt: null, deadlineAt: 1_200, lastHeartbeatAt: null, + heartbeatDetails: null, lastError: null, }, }, @@ -143,6 +150,7 @@ describe('deadline transition planner', () => { deadlineAt: 9_000, startedAt: null, lastHeartbeatAt: null, + heartbeatDetails: null, lastError: 'failed', }, }, @@ -186,6 +194,7 @@ describe('worker effect planner', () => { nextAttemptAt: 1_000, deadlineAt: null, lastHeartbeatAt: null, + heartbeatDetails: null, lastError: 'temporary', }, }, @@ -201,6 +210,7 @@ describe('worker effect planner', () => { attempt: 2, type: 'worker.send', input: { message: 'hello' }, + deadlines: { scheduleToStartAt: 1_500 }, }) expect(plan?.queuedEvent?.type).toBe('runtime.effect.queued') @@ -249,6 +259,7 @@ describe('worker effect routing', () => { deadlineAt: null, startedAt: null, lastHeartbeatAt: null, + heartbeatDetails: null, lastError: null, } : undefined, diff --git a/packages/runtime/src/repository.ts b/packages/runtime/src/repository.ts index d606502..84ac519 100644 --- a/packages/runtime/src/repository.ts +++ b/packages/runtime/src/repository.ts @@ -7,11 +7,14 @@ import { emptyCursor, EventStoreTrimmedError, hashRunState, + primePinnedDefinitions, summarizeRun, trimEventStoreSafely, type AppendableEvent, type AppendResult, type ComposedRegistry, + type DefinitionResolveError, + type DefinitionResolver, type EventEnvelope, type EventStore, type EventStoreAppendError, @@ -100,11 +103,19 @@ export interface RuntimeRepositoryOptions { readonly runCache: RunCursorCache readonly executionContexts: RunExecutionContexts readonly observer?: RuntimeObserver + readonly definitionResolver: DefinitionResolver } export function createRuntimeRepository(options: RuntimeRepositoryOptions) { - const { registry, snapshotStore, trimAfterSnapshot, keepSnapshots, runCache, executionContexts } = - options + const { + registry, + snapshotStore, + trimAfterSnapshot, + keepSnapshots, + runCache, + executionContexts, + definitionResolver, + } = options const headerFingerprints = new Map() @@ -196,7 +207,7 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { expectedTail?: number, ): Effect.Effect< RunCursor, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError > => Effect.gen(function* () { if (expectedTail !== undefined && cursor.seq === expectedTail) { @@ -226,6 +237,7 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { } delta = yield* readRemaining(store, runId, delta, expectedTail) + yield* primePinnedDefinitions(definitionResolver, delta) return advanceCursor(cursor, delta, registry) }) @@ -235,7 +247,7 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { runId: string, ): Effect.Effect< RunCursor, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError > => Effect.gen(function* () { let cursor = emptyCursor(runId) @@ -310,6 +322,8 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { delta = yield* readRemaining(store, runId, delta) } + yield* primePinnedDefinitions(definitionResolver, delta) + return advanceCursor(cursor, delta, registry) }) @@ -318,7 +332,7 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { runId: string, ): Effect.Effect< RunCursor, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError > => Effect.gen(function* () { const cached = runCache.get(runId) @@ -439,7 +453,7 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { commandRecords = 0, ): Effect.Effect< RunCursor, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError > => Effect.gen(function* () { const echo = assignSequences(batch, appendResult) @@ -457,6 +471,7 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { } } + yield* primePinnedDefinitions(definitionResolver, echo) next = advanceCursor(next, echo, registry) executionContexts.get(runId).liveCount = 0 @@ -484,7 +499,8 @@ export function createRuntimeRepository(options: RuntimeRepositoryOptions) { | EventStoreAppendError | EventStoreTruncationError | EventStoreTrimmedError - | EventStoreError, + | EventStoreError + | DefinitionResolveError, R > => Effect.gen(function* () { diff --git a/packages/runtime/src/retry-policy.ts b/packages/runtime/src/retry-policy.ts index eadfe3e..2178dc3 100644 --- a/packages/runtime/src/retry-policy.ts +++ b/packages/runtime/src/retry-policy.ts @@ -18,6 +18,17 @@ export function retryBackoff(retry: RetryPolicy, attempt: number): number { ) } +/** Policy backoff, extended when the failed attempt asked to wait longer. */ +export function retryDelay(retry: RetryPolicy, attempt: number, retryAfterMs?: number): number { + const backoff = retryBackoff(retry, attempt) + + if (retryAfterMs === undefined || !Number.isFinite(retryAfterMs) || retryAfterMs < 0) { + return backoff + } + + return Math.max(backoff, retryAfterMs) +} + export interface AttemptFailurePlan { readonly event: EventEnvelope readonly retrying: boolean @@ -32,10 +43,13 @@ export function planAttemptFailure(input: { readonly now: number readonly retry: RetryPolicy | undefined readonly origin: EventOrigin + /** Customer/domain failures must not be retried even when attempts remain. */ + readonly nonRetryable?: boolean + readonly retryAfterMs?: number }): AttemptFailurePlan { const { runId, item, attempt, error, retry } = input - if (!retry || attempt >= retry.maxAttempts) { + if (input.nonRetryable || !retry || attempt >= retry.maxAttempts) { return { event: createEffectFailedEvent(runId, item, error), retrying: false } } @@ -46,7 +60,7 @@ export function planAttemptFailure(input: { payload: { effectId: item.effectId, attempt, - nextAttemptAt: input.now + retryBackoff(retry, attempt), + nextAttemptAt: input.now + retryDelay(retry, attempt, input.retryAfterMs), error, }, threadId: item.threadId, diff --git a/packages/runtime/src/run-context.test.ts b/packages/runtime/src/run-context.test.ts new file mode 100644 index 0000000..7fcb831 --- /dev/null +++ b/packages/runtime/src/run-context.test.ts @@ -0,0 +1,370 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { + RunLauncherTag, + complete, + defineEffect, + defineRuntimeModule, + defineThread, + hasEntries, + invoke, + makeMemoryEventStore, + type EventStore, +} from '@looms/core' + +import { SecretContextConflictError, StartRunConflictError } from './errors' +import { createLooms } from './looms' +import { authDenied, authAllowed, bearerAuth } from './server' + +const SECRET = 'super-secret-token' + +const seen = defineEffect({ + type: 'context.seen', + execute: (_input, ctx) => [ + { + type: 'context.observed', + payload: { + tenant: ctx.attributes.tenant ?? '', + hasSecret: ctx.secrets.token === SECRET, + }, + threadId: ctx.threadId, + }, + ], +}) + +const contextThread = defineThread({ + kind: 'context', + initialState: () => ({ done: false }), + step: (state, event) => (event.type === 'context.observed' ? { ...state, done: true } : state), + effects: (state) => (state.done ? [complete({ ok: true })] : [invoke(seen, {}, 'see')]), +}) + +const context = { kind: 'context', name: 'context', version: 'v1' } + +const launchChild = defineEffect({ + type: 'context.launch', + execute: (_input, ctx) => + Effect.gen(function* () { + const launcher = yield* RunLauncherTag + + yield* launcher.launch({ + runId: `${ctx.runId}-child`, + kind: 'context', + definitionName: 'context', + input: null, + }) + + return [ + { + type: 'context.observed', + payload: { tenant: '', hasSecret: false }, + threadId: ctx.threadId, + }, + ] + }), +}) + +const parentThread = defineThread({ + kind: 'parent', + initialState: () => ({ launched: false }), + step: (state, event) => + event.type === 'context.observed' ? { ...state, launched: true } : state, + effects: (state) => + state.launched ? [complete({ ok: true })] : [invoke(launchChild, {}, 'launch')], +}) + +const contextModule = defineRuntimeModule({ + namespace: 'context', + protocolVersion: '1.0.0', + threads: { context: contextThread, parent: parentThread }, + effects: { seen, launch: launchChild }, + definitions: [ + { kind: 'context', name: 'context', version: 'v1' }, + { kind: 'parent', name: 'parent', version: 'v1' }, + ], +}) + +function containsSecret(encoded: string): boolean { + return encoded.includes(SECRET) +} + +describe('run attributes and secret context', () => { + test('records attributes and keeps secrets out of the log, state, and HTTP responses', async () => { + const dispatched: Array<{ secrets?: { token?: string } }> = [] + + const looms = createLooms({ + modules: [contextModule], + worker: { handles: ['context.seen'], dispatch: (task) => dispatched.push(task) }, + }) + + const started = await looms.start(context, null, { + runId: 'run_context', + attributes: { tenant: 'acme', project: 'billing' }, + secrets: { token: SECRET }, + }) + + expect(started.state.attributes).toEqual({ tenant: 'acme', project: 'billing' }) + expect(containsSecret(JSON.stringify(started))).toBe(false) + + const events = await looms.getEvents('run_context') + const state = await looms.getRun('run_context') + const summaries = await looms.fetch(new Request('http://looms.test/runs/summaries')) + const listed = await summaries?.json() + + expect(containsSecret(JSON.stringify(events))).toBe(false) + expect(containsSecret(JSON.stringify(state))).toBe(false) + expect(containsSecret(JSON.stringify(listed))).toBe(false) + expect(state.attributes).toEqual({ tenant: 'acme', project: 'billing' }) + + expect(events.find((event) => event.type === 'runtime.run.started')?.payload).toMatchObject({ + attributes: { tenant: 'acme', project: 'billing' }, + }) + + const store = await looms.store + expect(store.runSecrets).toBeDefined() + + if (store.runSecrets) { + expect(await Effect.runPromise(store.runSecrets.get('run_context'))).toEqual({ + token: SECRET, + }) + } + + expect(dispatched[0]?.secrets).toEqual({ token: SECRET }) + await looms.stop() + }) + + test('a worker reads the secret context only while its attempt is running', async () => { + const looms = createLooms({ + modules: [contextModule], + worker: { handles: ['context.seen'], dispatch: () => undefined }, + authorize: bearerAuth({ worker: 'worker-token' }), + }) + + await looms.start(context, null, { runId: 'run_worker', secrets: { token: SECRET } }) + const effectId = (await looms.getRun('run_worker')).outstandingEffects[0]?.effectId ?? '' + + const call = async (path: string, init: { token?: string; body?: string } = {}) => { + const response = await looms.fetch( + new Request( + `http://looms.test/runs/run_worker/effects/${encodeURIComponent(effectId)}/${path}`, + { + method: 'POST', + headers: { authorization: `Bearer ${init.token ?? 'worker-token'}` }, + body: init.body ?? '{}', + }, + ), + ) + + return { status: response?.status, headers: response?.headers, body: await response?.json() } + } + + expect((await call('1/secrets')).body).toEqual({ status: 'stale' }) + expect((await call('1/started')).body).toEqual({ status: 'accepted', cancelRequested: false }) + + const accepted = await call('1/secrets') + expect(accepted.body).toEqual({ status: 'accepted', secrets: { token: SECRET } }) + expect(accepted.headers?.get('cache-control')).toBe('no-store') + + expect((await call('1/secrets', { token: 'wrong' })).status).toBe(401) + expect((await call('2/secrets')).body).toEqual({ status: 'stale' }) + + const observed = { + type: 'context.observed', + payload: { tenant: '', hasSecret: true }, + threadId: (await looms.getRun('run_worker')).outstandingEffects[0]?.threadId, + } + + await call('1/complete', { body: JSON.stringify({ events: [observed] }) }) + + expect((await looms.getRun('run_worker')).status).toBe('completed') + expect((await call('1/secrets')).body).toEqual({ status: 'stale' }) + expect(containsSecret(JSON.stringify(await looms.getEvents('run_worker')))).toBe(false) + await looms.stop() + }) + + test('a different attribute or secret context conflicts without revealing the secret', async () => { + const looms = createLooms({ modules: [contextModule] }) + + await looms.start(context, null, { + runId: 'run_once', + idempotencyKey: 'same', + attributes: { tenant: 'acme' }, + secrets: { token: SECRET }, + }) + + const attributeConflict = await looms + .start(context, null, { + runId: 'run_once', + idempotencyKey: 'same', + attributes: { tenant: 'other' }, + secrets: { token: SECRET }, + }) + .then( + () => null, + (error) => error, + ) + + expect(attributeConflict).toBeInstanceOf(StartRunConflictError) + + const secretConflict = await looms + .start(context, null, { + runId: 'run_once', + idempotencyKey: 'same', + attributes: { tenant: 'acme' }, + secrets: { token: 'rotated' }, + }) + .then( + () => null, + (error) => error, + ) + + expect(secretConflict).toBeInstanceOf(SecretContextConflictError) + expect(containsSecret(JSON.stringify(secretConflict))).toBe(false) + expect(JSON.stringify(secretConflict)).not.toContain('rotated') + + const retried = await looms.start(context, null, { + runId: 'run_once', + idempotencyKey: 'same', + attributes: { tenant: 'acme' }, + }) + + expect(retried.runId).toBe('run_once') + + const store = await looms.store + + if (store.runSecrets) { + expect(await Effect.runPromise(store.runSecrets.get('run_once'))).toEqual({ token: SECRET }) + } + + await looms.stop() + }) + + test('authorization sees attributes from the stored run and the start body', async () => { + const decisions: Array<{ attributes?: { tenant?: string } }> = [] + + const looms = createLooms({ + modules: [contextModule], + authorize: (_request, route) => + Effect.sync(() => { + decisions.push(route) + return route.attributes?.tenant === 'acme' ? authAllowed : authDenied(403, 'Forbidden') + }), + }) + + const started = await looms.fetch( + new Request('http://looms.test/runs', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + kind: 'context', + definitionName: 'context', + runId: 'run_auth', + attributes: { tenant: 'acme' }, + }), + }), + ) + + expect(started?.status).toBe(200) + + const read = await looms.fetch(new Request('http://looms.test/runs/run_auth')) + expect(read?.status).toBe(200) + expect(decisions.some((route) => route.attributes?.tenant === 'acme')).toBe(true) + await looms.stop() + }) + + test('authorization reads only the start event to find attributes', async () => { + const memory = await Effect.runPromise(makeMemoryEventStore) + let fullReads = 0 + + const store: EventStore = { + ...memory, + read: (runId, options) => { + if (options?.limit !== 1) { + fullReads += 1 + } + + return memory.read(runId, options) + }, + } + + const readsAtDecision: number[] = [] + + const looms = createLooms({ + modules: [contextModule], + store, + authorize: (_request, route) => + Effect.sync(() => { + if (route.name === 'runs.get') { + readsAtDecision.push(fullReads) + } + + return route.attributes?.tenant === 'acme' ? authAllowed : authDenied(403, 'Forbidden') + }), + }) + + await looms.start(context, null, { runId: 'run_cheap_auth', attributes: { tenant: 'acme' } }) + fullReads = 0 + + const read = await looms.fetch(new Request('http://looms.test/runs/run_cheap_auth')) + + expect(read?.status).toBe(200) + expect(readsAtDecision).toEqual([0]) + await looms.stop() + }) + + test('a launched run and a fork inherit attributes and secrets without logging them', async () => { + const looms = createLooms({ modules: [contextModule] }) + + await looms.start({ kind: 'parent', name: 'parent', version: 'v1' }, null, { + runId: 'run_parent', + attributes: { tenant: 'acme' }, + secrets: { token: SECRET }, + }) + + const deadline = Date.now() + 5_000 + + while ((await looms.getRun('run_parent-child')).rootThreadId === null) { + if (Date.now() > deadline) { + throw new Error('the child run did not start') + } + + await Bun.sleep(5) + } + + const child = await looms.getRun('run_parent-child') + const childEvents = await looms.getEvents('run_parent-child') + expect(child.attributes).toEqual({ tenant: 'acme' }) + expect(containsSecret(JSON.stringify(childEvents))).toBe(false) + + const store = await looms.store + + if (store.runSecrets) { + expect(await Effect.runPromise(store.runSecrets.get('run_parent-child'))).toEqual({ + token: SECRET, + }) + } + + const finished = Date.now() + 5_000 + + while ((await looms.getRun('run_parent')).status !== 'completed') { + if (Date.now() > finished) { + throw new Error('the parent run did not complete') + } + + await Bun.sleep(5) + } + + const forked = await looms.fork('run_parent', { runId: 'run_fork' }) + expect(forked.state.attributes).toEqual({ tenant: 'acme' }) + expect(containsSecret(JSON.stringify(await looms.getEvents('run_fork')))).toBe(false) + + if (store.runSecrets) { + expect(hasEntries(await Effect.runPromise(store.runSecrets.get('run_fork')))).toBe(true) + expect(await Effect.runPromise(store.runSecrets.get('run_fork'))).toEqual({ token: SECRET }) + } + + await looms.stop() + }) +}) diff --git a/packages/runtime/src/run-start.ts b/packages/runtime/src/run-start.ts index 817ac25..5eb16c7 100644 --- a/packages/runtime/src/run-start.ts +++ b/packages/runtime/src/run-start.ts @@ -1,6 +1,6 @@ import { Effect, Predicate } from 'effect' -import type { JsonValue, RunState } from '@looms/core' +import { sameStringRecord, type JsonValue, type RunState } from '@looms/core' import { RuntimeExecutionError, StartRunConflictError, type RunStartIdentity } from './errors' @@ -69,6 +69,10 @@ export function assertSameDurableStart( mismatches.push('correlationId') } + if (!sameStringRecord(existing.attributes ?? {}, requested.attributes)) { + mismatches.push('attributes') + } + return mismatches.length > 0 ? Effect.fail(new StartRunConflictError(runId, requested, existing, mismatches)) : Effect.succeed(existing.rootThreadId) diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index ac0b33b..39e6b8b 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -1,28 +1,37 @@ -import { Clock, Effect, ManagedRuntime, Predicate } from 'effect' +import { Clock, Effect, ManagedRuntime, Predicate, type Types } from 'effect' import { + cachedDefinitionResolver, + chainDefinitionResolvers, composeModules, createEvent, createThreadId, createRunId, DEFAULT_DEFINITION_VERSION, DEFAULT_SNAPSHOT_EVERY, - definitionKey, + staticDefinitionResolver, descendantThreadIds, EventStoreTag, foldRun, + hasEntries, isRunTerminal, unavailableRunLauncher, planFork, project, + readTraceContext, replayTo, + sameStringRecord, summarizeRun, + type EventStore, + type SecretContext, type AnyRuntimeModule, type EventInput, type EventOrigin, type RunLauncher, + type RunStartedPayload, type RunState, type SignalInterruptRule, + type TraceContext, type InvalidEventError, } from '@looms/core' @@ -34,6 +43,7 @@ import { RunNotFoundError, RunTerminalError, RuntimeExecutionError, + SecretContextConflictError, StartRunConflictError, } from './errors' import { @@ -54,6 +64,7 @@ import { assertSameDurableStart } from './run-start' import { createWakeEngine } from './wake-engine' import { createTimeoutScheduler, type WakeScheduler } from './wake-scheduler' import { createWorkerCallbacks } from './worker-callbacks' +import { assertWorkerDeadlines } from './worker-effect-dispatch' export type { RegisteredDefinition } from '@looms/core' export type { WakeScheduler } from './wake-scheduler' @@ -72,6 +83,7 @@ export { RunOverloadedError } from './ingress' export type { CreateRuntimeOptions, EffectWorker, + EffectWorkerDeadlines, EffectWorkerTask, LoomsRuntime, RunOperationalStatus, @@ -79,7 +91,7 @@ export type { StartRunArgs, WakeError, } from './types' -export type { WorkerCallback, WorkerCallbackInput } from '@looms/core' +export type { WorkerCallback, WorkerCallbackInput, WorkerCallbackResult } from '@looms/core' import type { CreateRuntimeOptions, LoomsRuntime, StartRunArgs } from './types' function currentTimeMillis(): number { @@ -111,6 +123,75 @@ const RESCAN_CONCURRENCY = 16 const DEFAULT_KEEP_SNAPSHOTS = 1 const DEFAULT_MAX_PENDING_RUN_OPERATIONS = 256 +interface StartedPayloadInput { + readonly threadId: string + readonly kind: string + readonly definitionName: string + readonly definitionVersion: string + readonly input: StartRunArgs['input'] + readonly requestedThreadId: string | null + readonly launchedBy: StartRunArgs['launchedBy'] + readonly attributes: StartRunArgs['attributes'] + readonly trace: TraceContext | undefined +} + +function startedPayload(input: StartedPayloadInput): RunStartedPayload { + const payload: Types.Mutable = { + rootThreadId: input.threadId, + kind: input.kind, + definitionName: input.definitionName, + definitionVersion: input.definitionVersion, + input: input.input ?? null, + requestedThreadId: input.requestedThreadId, + } + + if (input.launchedBy) { + payload.launchedBy = input.launchedBy + } + + if (input.attributes && hasEntries(input.attributes)) { + payload.attributes = input.attributes + } + + if (input.trace) { + payload.trace = input.trace + } + + return payload +} + +function reconcileRunSecrets( + store: EventStore, + runId: string, + requested: SecretContext | undefined, +) { + const next = requested ?? {} + + return Effect.gen(function* () { + const stored = store.runSecrets + + if (!stored) { + if (hasEntries(next)) { + return yield* new RuntimeExecutionError('This event store cannot persist a secret context') + } + + return yield* Effect.void + } + + const current = yield* stored.get(runId) + + if (sameStringRecord(current, next) || requested === undefined) { + return yield* Effect.void + } + + if (!hasEntries(current)) { + return yield* stored.put(runId, next) + } + + return yield* new SecretContextConflictError(runId) + }) +} + export function createRuntime( options: CreateRuntimeOptions, ): LoomsRuntime { @@ -119,14 +200,28 @@ export function createRuntime [definitionKey(def.kind, def.name, def.version), def] as const, - ), + assertWorkerDeadlines(registry, options.worker) + + const staticResolver = staticDefinitionResolver( + registry.definitions.map((definition) => ({ + ...definition.value, + kind: definition.kind, + name: definition.name, + version: definition.version, + })), + ) + + const definitionResolver = cachedDefinitionResolver( + options.definitionResolver + ? chainDefinitionResolvers([staticResolver, options.definitionResolver]) + : staticResolver, ) + for (const module of options.modules) { + module.bindHost?.({ definitionResolver }) + } + // Module composition erases the heterogeneous service identifier union at this host boundary. // oxlint-disable-next-line effecttsgo/any-unknown-in-error-context const services = ManagedRuntime.make(moduleServices(options.modules)) @@ -155,6 +250,7 @@ export function createRuntime event.type === 'runtime.child.settled', + ) + + if (terminal && !settlementOnly) { return yield* new RunTerminalError(runId, current.state.status) } const folded = foldRun(batch, registry, { runId, initial: current.state }) - const satisfiedEvents = waitSatisfiedEvents(folded, batch).map((input) => + const forWaits = + terminal || current.state.closeWaiting + ? batch.filter((event) => event.type !== 'runtime.child.settled') + : batch + + const satisfiedEvents = waitSatisfiedEvents(folded, forWaits).map((input) => createEvent(runId, input), ) @@ -223,7 +330,13 @@ export function createRuntime scheduler, launcher: () => launcher, }) @@ -360,8 +480,21 @@ export function createRuntime launcher, }) + const appendAndWake: LoomsRuntime['accept'] = (runId, events, signalOpts) => + withRunIngress(runId, signalAdmitted(runId, events, signalOpts)).pipe( + Effect.flatMap((state) => { + if (signalOpts?.coalesce && executionContexts.get(runId).waking) { + executionContexts.get(runId).wakeAgain = true + return Effect.succeed(state) + } + + return runtime.wake(runId) + }), + ) + runtime = { modules: options.modules, registry, @@ -555,12 +688,11 @@ export function createRuntime admitExternalEvents(events).pipe( - Effect.flatMap((admitted) => - withRunIngress(runId, signalAdmitted(runId, admitted, signalOpts)), - ), - Effect.andThen(runtime.wake(runId)), + Effect.flatMap((admitted) => appendAndWake(runId, admitted, signalOpts)), ), + accept: (runId, events, signalOpts) => appendAndWake(runId, events, signalOpts), + cancel: cancellation, fork: (sourceRunId, forkOptions) => { @@ -598,6 +730,14 @@ export function createRuntime + runtime + .accept(runId, events, { ...signalOptions, coalesce: true }) + .pipe(Effect.provideService(EventStoreTag, store), Effect.asVoid), + cancel: (runId) => + runtime.cancel(runId).pipe(Effect.provideService(EventStoreTag, store), Effect.asVoid), } : unavailableRunLauncher) diff --git a/packages/runtime/src/server.test.ts b/packages/runtime/src/server.test.ts index 52bf30a..5fa5d08 100644 --- a/packages/runtime/src/server.test.ts +++ b/packages/runtime/src/server.test.ts @@ -493,6 +493,7 @@ describe('server routing', () => { status: 'completed', kind: 'workflow', definitionName: 'summarized', + attributes: {}, }, ]) diff --git a/packages/runtime/src/server.ts b/packages/runtime/src/server.ts index b2e30b9..a5a0a17 100644 --- a/packages/runtime/src/server.ts +++ b/packages/runtime/src/server.ts @@ -1,9 +1,11 @@ export { createFetchHandler, handlesPath, type FetchHandlerOptions } from './server/fetch-handler' export { isLoomsApiPath } from './server/http' export { mountDebugger, type DebuggerMount } from './server/debugger-host' +export { runRouteInfo } from './server/run-route' export { authAllowed, authDenied, + authResponse, bearerAuth, defaultAuthorize, type AuthDecision, @@ -13,6 +15,17 @@ export { type RouteAccess, type RouteInfo, } from './server/auth' +export { + grantAuth, + issueRunGrant, + grantSubprotocol, + runGrantToken, + verifyRunGrant, + GrantError, + type GrantAuthOptions, + type GrantFailure, + type RunGrant, +} from './server/grant' export interface RunningServer { port: number diff --git a/packages/runtime/src/server/auth.ts b/packages/runtime/src/server/auth.ts index 83008f3..4dba778 100644 --- a/packages/runtime/src/server/auth.ts +++ b/packages/runtime/src/server/auth.ts @@ -1,5 +1,7 @@ import { Effect } from 'effect' +import type { RunAttributes } from '@looms/core' + /** * What a route can do. `public` routes (health) are never passed to `authorize`; `read` and * `write` cover run state, `worker` covers external worker callbacks, and `operations` covers @@ -14,6 +16,8 @@ export interface RouteInfo { /** Stable route name, such as `runs.signal` or `effects.retry`. */ readonly name: string readonly runId?: string + /** Present when the run has labels. Safe for the authorizer to branch on. */ + readonly attributes?: RunAttributes } export type AuthDecision = diff --git a/packages/runtime/src/server/fetch-handler.ts b/packages/runtime/src/server/fetch-handler.ts index 831604d..bcb7a82 100644 --- a/packages/runtime/src/server/fetch-handler.ts +++ b/packages/runtime/src/server/fetch-handler.ts @@ -1,6 +1,6 @@ -import { Effect, Predicate } from 'effect' +import { Data, Effect, Option, Predicate, Schema } from 'effect' -import type { EventStore } from '@looms/core' +import { hasEntries, RunAttributesSchema, type EventStore, type RunAttributes } from '@looms/core' import type { LoomsRuntime } from '../runtime' import { authResponse, defaultAuthorize, type Authorize } from './auth' @@ -11,13 +11,14 @@ import { definitionRoutes } from './routes/definitions' import { effectRoutes } from './routes/effects' import { operationRoutes } from './routes/operations' import { runRoutes } from './routes/runs' +import { readStoredAttributes } from './run-route' export interface FetchHandlerOptions { readonly runtime: LoomsRuntime readonly store: EventStore /** * Decides whether a request may use a route. Defaults to open reads and writes with worker and - * operations routes unavailable; use `bearerAuth` or your own policy to change that. + * operations routes unavailable; use `bearerAuth`, `grantAuth`, or your own policy to change that. */ readonly authorize?: Authorize /** Debugger UI served beside the API and authorized as `{ access: 'read', name: 'debugger' }`. */ @@ -33,11 +34,38 @@ function createRoutes(options: RouteOptions): readonly Route[] { ] } +const StartAttributesSchema = Schema.Struct({ + attributes: Schema.optional(RunAttributesSchema), +}) + +class UnreadableStartBody extends Data.TaggedError('UnreadableStartBody')<{ + readonly cause: unknown +}> {} + +const noAttributes: RunAttributes = {} + +function readRequestAttributes(req: Request): Effect.Effect { + return Effect.tryPromise({ + try: () => req.clone().json(), + catch: (cause) => new UnreadableStartBody({ cause }), + }).pipe( + Effect.map((body) => + Option.match(Schema.decodeUnknownOption(StartAttributesSchema)(body), { + onNone: () => noAttributes, + onSome: (decoded) => decoded.attributes ?? noAttributes, + }), + ), + Effect.catchTag('UnreadableStartBody', () => Effect.succeed(noAttributes)), + ) +} + function authorizeRoute( req: Request, matched: MatchedRoute, authorize: Authorize, -): Effect.Effect { + runtime: LoomsRuntime, + store: EventStore, +): Effect.Effect { const { access, name } = matched.route if (access === 'public') { @@ -46,10 +74,26 @@ function authorizeRoute( const runId = matched.params.runId - return authorize( - req, - Predicate.isString(runId) ? { access, name, runId } : { access, name }, - ).pipe(Effect.map(authResponse)) + return Effect.gen(function* () { + const stored = Predicate.isString(runId) + ? yield* readStoredAttributes(runtime, store, runId) + : noAttributes + + const requested = + matched.route.name === 'runs.start' ? yield* readRequestAttributes(req) : noAttributes + + const attributes = hasEntries(stored) ? stored : requested + + const route = Predicate.isString(runId) + ? hasEntries(attributes) + ? { access, name, runId, attributes } + : { access, name, runId } + : hasEntries(attributes) + ? { access, name, attributes } + : { access, name } + + return yield* authorize(req, route).pipe(Effect.map(authResponse)) + }) } /** Whether the fetch handler answers `pathname`: the HTTP API or the debugger mount. */ @@ -90,7 +134,13 @@ export function createFetchHandler( return new Response('Not Found', { status: 404 }) } - const denied = yield* authorizeRoute(req, matched, authorize) + const denied = yield* authorizeRoute( + req, + matched, + authorize, + options.runtime, + options.store, + ) if (denied) { return denied diff --git a/packages/runtime/src/server/grant.test.ts b/packages/runtime/src/server/grant.test.ts new file mode 100644 index 0000000..c016a4a --- /dev/null +++ b/packages/runtime/src/server/grant.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { createLooms, grantAuth, issueRunGrant, verifyRunGrant } from '../index' + +const secret = 'grant-secret' +const runId = 'run_1' + +const claims = { + runId, + principal: 'user_1', + access: 'read' as const, + exp: 5_000, +} + +describe('run grants', () => { + test('round-trips a signed grant and rejects a different secret', async () => { + const token = await Effect.runPromise(issueRunGrant(secret, claims)) + const grant = await Effect.runPromise(verifyRunGrant(secret, token, 4_000)) + + expect(grant).toEqual(claims) + + const forged = await Effect.runPromise( + verifyRunGrant('other-secret', token, 4_000).pipe(Effect.flip), + ) + + expect(forged.reason).toBe('signature') + }) + + test('rejects a tampered payload, a bad shape, and an empty principal', async () => { + const token = await Effect.runPromise(issueRunGrant(secret, claims)) + const [version, payload, signature] = token.split('.') + const flipped = payload?.[0] === 'A' ? 'B' : 'A' + const tampered = `${version}.${flipped}${payload?.slice(1)}.${signature}` + + const forged = await Effect.runPromise( + verifyRunGrant(secret, tampered, 4_000).pipe(Effect.flip), + ) + + expect(forged.reason).toBe('signature') + + const malformed = await Effect.runPromise( + verifyRunGrant(secret, 'not-a-grant', 4_000).pipe(Effect.flip), + ) + + expect(malformed.reason).toBe('malformed') + + const empty = await Effect.runPromise( + issueRunGrant(secret, { ...claims, principal: '' }).pipe(Effect.flip), + ) + + expect(empty.reason).toBe('claims') + }) + + test('expires at exp and honors clock skew', async () => { + const token = await Effect.runPromise(issueRunGrant(secret, claims)) + const expired = await Effect.runPromise(verifyRunGrant(secret, token, 5_000).pipe(Effect.flip)) + + expect(expired.reason).toBe('expired') + + const skewed = await Effect.runPromise(verifyRunGrant(secret, token, 5_000, 1)) + + expect(skewed.principal).toBe('user_1') + }) + + test('authorizes only a read of the granted run', async () => { + const token = await Effect.runPromise(issueRunGrant(secret, claims)) + + const looms = createLooms({ + modules: [], + authorize: grantAuth({ secret, now: () => 4_000 }), + }) + + const status = (input: RequestInfo, init?: RequestInit) => looms.fetch(new Request(input, init)) + + try { + const missing = await status(`http://looms.test/runs/${runId}/threads`) + const query = await status(`http://looms.test/runs/${runId}/threads?grant=${token}`) + + const protocol = await status(`http://looms.test/runs/${runId}/threads`, { + headers: { 'sec-websocket-protocol': `chat, ${token}` }, + }) + + const otherRun = await status(`http://looms.test/runs/run_2/threads?grant=${token}`) + const unscoped = await status(`http://looms.test/runs?grant=${token}`) + + const write = await status(`http://looms.test/runs/${runId}/wake?grant=${token}`, { + method: 'POST', + }) + + const badQuery = await status(`http://looms.test/runs/${runId}/threads?grant=nope`, { + headers: { 'sec-websocket-protocol': token }, + }) + + expect(missing?.status).toBe(401) + expect(query?.status).toBe(200) + expect(protocol?.status).toBe(200) + expect(otherRun?.status).toBe(403) + expect(unscoped?.status).toBe(403) + expect(write?.status).toBe(403) + expect(badQuery?.status).toBe(401) + } finally { + await looms.stop() + } + }) +}) diff --git a/packages/runtime/src/server/grant.ts b/packages/runtime/src/server/grant.ts new file mode 100644 index 0000000..cece27b --- /dev/null +++ b/packages/runtime/src/server/grant.ts @@ -0,0 +1,277 @@ +import { Data, Effect, Option, Schema } from 'effect' + +import { authAllowed, authDenied, type AuthDecision, type Authorize, type RouteInfo } from './auth' + +const textEncoder = new TextEncoder() +const textDecoder = new TextDecoder() + +const unauthorized = authDenied(401, 'Unauthorized') +const forbidden = authDenied(403, 'Forbidden') + +const GrantPayloadSchema = Schema.Struct({ + v: Schema.Literal(1), + runId: Schema.String, + principal: Schema.String, + access: Schema.Literal('read'), + exp: Schema.Finite, +}) + +const GrantPayloadJson = Schema.fromJsonString(GrantPayloadSchema) +const encodeGrantPayload = Schema.encodeSync(GrantPayloadJson) +const decodeGrantPayload = Schema.decodeUnknownOption(GrantPayloadJson) + +export type GrantFailure = 'malformed' | 'signature' | 'expired' | 'claims' + +export class GrantError extends Data.TaggedError('GrantError')<{ + readonly reason: GrantFailure +}> {} + +/** Claims carried by a run grant. `exp` is an exclusive expiry in Unix milliseconds. */ +export interface RunGrant { + readonly runId: string + readonly principal: string + readonly access: 'read' + readonly exp: number +} + +export interface GrantAuthOptions { + readonly secret: string + /** Clock used for expiry. Defaults to `Date.now`. */ + readonly now?: () => number + /** + * Extra time after `exp` that still verifies. Covers clock skew between the issuer and the host. + * Defaults to 0. + */ + readonly clockSkewMs?: number +} + +interface GrantPayload { + readonly v: 1 + readonly runId: string + readonly principal: string + readonly access: 'read' + readonly exp: number +} + +/** + * Signs a read grant for one run. The token is URL-safe and is also a legal WebSocket subprotocol + * value, so a browser can send it as a query parameter or as `Sec-WebSocket-Protocol`. + */ +export function issueRunGrant(secret: string, claims: RunGrant): Effect.Effect { + return Effect.gen(function* () { + const payload = grantPayload(claims) + + if (!payload) { + return yield* new GrantError({ reason: 'claims' }) + } + + const json = yield* Effect.try({ + try: () => encodeGrantPayload(payload), + catch: () => new GrantError({ reason: 'claims' }), + }) + + const encoded = encodeBase64Url(textEncoder.encode(json)) + const key = yield* importHmacKey(secret) + + const signature = yield* Effect.tryPromise({ + try: () => crypto.subtle.sign('HMAC', key, textEncoder.encode(encoded)), + catch: () => new GrantError({ reason: 'signature' }), + }) + + return `v1.${encoded}.${encodeBase64Url(new Uint8Array(signature))}` + }) +} + +/** Verifies the HMAC and expiry. A grant for another run fails later, when a route is checked. */ +export function verifyRunGrant( + secret: string, + token: string, + now: number, + clockSkewMs = 0, +): Effect.Effect { + return Effect.gen(function* () { + const parts = token.split('.') + + if (parts.length !== 3 || parts[0] !== 'v1') { + return yield* new GrantError({ reason: 'malformed' }) + } + + const encoded = parts[1] ?? '' + const signature = decodeBase64Url(parts[2] ?? '') + const payloadBytes = decodeBase64Url(encoded) + + if (!signature || !payloadBytes) { + return yield* new GrantError({ reason: 'malformed' }) + } + + const key = yield* importHmacKey(secret) + + const authentic = yield* Effect.tryPromise({ + try: () => crypto.subtle.verify('HMAC', key, signature, textEncoder.encode(encoded)), + catch: () => new GrantError({ reason: 'signature' }), + }) + + if (!authentic) { + return yield* new GrantError({ reason: 'signature' }) + } + + const decoded = decodeGrantPayload(textDecoder.decode(payloadBytes)) + + if ( + Option.isNone(decoded) || + decoded.value.runId.length === 0 || + decoded.value.principal.length === 0 + ) { + return yield* new GrantError({ reason: 'claims' }) + } + + const skew = finiteSkew(clockSkewMs) + + if (decoded.value.exp <= now - skew) { + return yield* new GrantError({ reason: 'expired' }) + } + + return { + runId: decoded.value.runId, + principal: decoded.value.principal, + access: decoded.value.access, + exp: decoded.value.exp, + } + }) +} + +/** + * Authorizes a read of the granted run. The token comes from the `grant` query parameter, or, when + * that is absent, from a `v1.` WebSocket subprotocol. Writes, worker callbacks, operations, and + * reads of any other run are refused. A host that accepts the subprotocol has to echo it on the + * WebSocket response; see `grantSubprotocol`. + */ +export function grantAuth(options: GrantAuthOptions): Authorize { + const now = options.now ?? Date.now + const skew = finiteSkew(options.clockSkewMs) + + return (request, route) => + Effect.gen(function* () { + const token = runGrantToken(request) + + if (token === undefined) { + return unauthorized + } + + return yield* verifyRunGrant(options.secret, token, now(), skew).pipe( + Effect.map((grant) => grantDecision(grant, route)), + Effect.catchTag('GrantError', () => Effect.succeed(unauthorized)), + ) + }) +} + +/** The grant token on this request, if the client sent one. */ +export function runGrantToken(request: Request): string | undefined { + const query = new URL(request.url).searchParams.get('grant') + + if (query) { + return query + } + + return grantSubprotocol(request) +} + +/** + * The `v1.` grant the client offered as a WebSocket subprotocol. The upgrade response must echo it, + * or the browser fails the handshake. + */ +export function grantSubprotocol(request: Request): string | undefined { + return websocketGrant(request.headers.get('sec-websocket-protocol')) +} + +function grantDecision(grant: RunGrant, route: RouteInfo): AuthDecision { + if (route.access !== 'read' || route.runId !== grant.runId) { + return forbidden + } + + return authAllowed +} + +function grantPayload(claims: RunGrant): GrantPayload | undefined { + if (claims.runId.length === 0 || claims.principal.length === 0 || !Number.isFinite(claims.exp)) { + return undefined + } + + return { + v: 1, + runId: claims.runId, + principal: claims.principal, + access: 'read', + exp: claims.exp, + } +} + +function finiteSkew(clockSkewMs: number | undefined): number { + if (clockSkewMs === undefined || !Number.isFinite(clockSkewMs) || clockSkewMs < 0) { + return 0 + } + + return clockSkewMs +} + +function websocketGrant(header: string | null): string | undefined { + if (header === null) { + return undefined + } + + for (const part of header.split(',')) { + const protocol = part.trim() + + if (protocol.startsWith('v1.')) { + return protocol + } + } + + return undefined +} + +function importHmacKey(secret: string): Effect.Effect { + return Effect.tryPromise({ + try: () => + crypto.subtle.importKey( + 'raw', + textEncoder.encode(secret), + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign', 'verify'], + ), + catch: () => new GrantError({ reason: 'signature' }), + }) +} + +function encodeBase64Url(bytes: Uint8Array): string { + let binary = '' + + for (const byte of bytes) { + binary += String.fromCharCode(byte) + } + + let encoded = btoa(binary).replaceAll('+', '-').replaceAll('/', '_') + + while (encoded.endsWith('=')) { + encoded = encoded.slice(0, -1) + } + + return encoded +} + +function decodeBase64Url(value: string): Uint8Array | undefined { + if (value.length === 0 || value.length % 4 === 1 || !/^[A-Za-z0-9_-]+$/.test(value)) { + return undefined + } + + const padded = value + '='.repeat((4 - (value.length % 4)) % 4) + const binary = atob(padded.replaceAll('-', '+').replaceAll('_', '/')) + const bytes = new Uint8Array(new ArrayBuffer(binary.length)) + + for (let index = 0; index < binary.length; index += 1) { + bytes[index] = binary.charCodeAt(index) + } + + return bytes +} diff --git a/packages/runtime/src/server/routes/effects.ts b/packages/runtime/src/server/routes/effects.ts index a63c6e5..5d7f2fe 100644 --- a/packages/runtime/src/server/routes/effects.ts +++ b/packages/runtime/src/server/routes/effects.ts @@ -1,6 +1,11 @@ import { Effect, Schema } from 'effect' -import type { WorkerCallbackInput, WorkerCallbackKind } from '@looms/core' +import { + JsonValueSchema, + type JsonValue, + type WorkerCallbackInput, + type WorkerCallbackKind, +} from '@looms/core' import { EventInputBodySchema, provideStore, readJson, type RouteOptions } from '../http' import { integer, oneOf, route, type Route } from '../route-table' @@ -13,9 +18,28 @@ const WORKER_CALLBACK_KINDS = [ 'cancelled', ] as const satisfies readonly WorkerCallbackKind[] +interface FailWorkerCallback { + effectId: string + attempt: number + kind: 'fail' + error: string + nonRetryable?: boolean + retryAfterMs?: number +} + +interface HeartbeatWorkerCallback { + effectId: string + attempt: number + kind: 'heartbeat' + details?: JsonValue +} + const WorkerCallbackBodySchema = Schema.Struct({ events: Schema.optional(Schema.Array(EventInputBodySchema)), error: Schema.optional(Schema.String), + nonRetryable: Schema.optional(Schema.Boolean), + retryAfterMs: Schema.optional(Schema.Finite), + details: Schema.optional(JsonValueSchema), }) function workerCallbackInput( @@ -31,10 +55,47 @@ function workerCallbackInput( switch (params.action) { case 'complete': return { ...callback, kind: 'complete', events: body.events ?? [] } - case 'fail': - return body.error ? { ...callback, kind: 'fail', error: body.error } : null - default: + + case 'fail': { + if (body.error === undefined) { + return null + } + + const failure: FailWorkerCallback = { + ...callback, + kind: 'fail', + error: body.error, + } + + if (body.nonRetryable !== undefined) { + failure.nonRetryable = body.nonRetryable + } + + if (body.retryAfterMs !== undefined) { + failure.retryAfterMs = body.retryAfterMs + } + + return failure + } + + case 'heartbeat': { + const heartbeat: HeartbeatWorkerCallback = { ...callback, kind: 'heartbeat' } + + if (body.details !== undefined) { + heartbeat.details = body.details + } + + return heartbeat + } + + case 'started': + case 'cancelled': return { ...callback, kind: params.action } + + default: { + const exhaustive: never = params.action + return exhaustive + } } } @@ -62,6 +123,25 @@ export function effectRoutes({ runtime, store }: RouteOptions): readonly Route[] ), ), }), + route({ + method: 'POST', + path: '/runs/:runId/effects/:effectId/:attempt/secrets', + name: 'effects.workerSecrets', + access: 'worker', + params: { attempt: integer }, + handle: ({ params }) => + provideStore( + runtime.workerSecrets(params.runId, { + effectId: params.effectId, + attempt: params.attempt, + }), + store, + ).pipe( + Effect.map((result) => + Response.json(result, { headers: { 'cache-control': 'no-store' } }), + ), + ), + }), route({ method: 'POST', path: '/runs/:runId/effects/:effectId/:attempt/:action', @@ -80,8 +160,8 @@ export function effectRoutes({ runtime, store }: RouteOptions): readonly Route[] return Response.json({ error: 'error required' }, { status: 400 }) } - const state = yield* provideStore(runtime.workerCallback(params.runId, input), store) - return Response.json({ runId: params.runId, state }) + const result = yield* provideStore(runtime.workerCallback(params.runId, input), store) + return Response.json(result) }), }), ] diff --git a/packages/runtime/src/server/routes/runs.ts b/packages/runtime/src/server/routes/runs.ts index 1dab2e5..2e122b6 100644 --- a/packages/runtime/src/server/routes/runs.ts +++ b/packages/runtime/src/server/routes/runs.ts @@ -4,6 +4,9 @@ import { createRunId, EventStoreTag, JsonValueSchema, + RunAttributesSchema, + readTraceHeaders, + SecretContextSchema, stringifyJson, type EventStore, type JsonValue, @@ -32,6 +35,8 @@ const StartRunBodySchema = Schema.Struct({ input: Schema.optional(JsonValueSchema), runId: Schema.optional(Schema.String), idempotencyKey: Schema.optional(Schema.String), + attributes: Schema.optional(RunAttributesSchema), + secrets: Schema.optional(SecretContextSchema), }) const SignalBodySchema = Schema.Struct({ @@ -185,6 +190,9 @@ export function runRoutes({ runtime, store }: RouteOptions): readonly Route[] { input: body.input ?? null, runId, idempotencyKey, + attributes: body.attributes, + secrets: body.secrets, + trace: readTraceHeaders(req.headers), } if (wantsRunStream(req, url)) { diff --git a/packages/runtime/src/server/run-route.ts b/packages/runtime/src/server/run-route.ts new file mode 100644 index 0000000..ae04618 --- /dev/null +++ b/packages/runtime/src/server/run-route.ts @@ -0,0 +1,59 @@ +import { Effect } from 'effect' + +import { + EventStoreTag, + hasEntries, + isJsonObject, + readAttributes, + type EventStore, + type RunAttributes, +} from '@looms/core' + +import type { LoomsRuntime } from '../runtime' +import type { AuthorizedAccess, RouteInfo } from './auth' + +const noAttributes: RunAttributes = {} + +/** + * Attributes are immutable and recorded on `runtime.run.started`, which is seq 1 of every + * untrimmed log, forks included. Only a trimmed log needs the snapshot-backed fold. + */ +export function readStoredAttributes( + runtime: LoomsRuntime, + store: EventStore, + runId: string, +): Effect.Effect { + const folded = runtime.getRun(runId).pipe( + Effect.provideService(EventStoreTag, store), + Effect.map((state) => state.attributes), + ) + + return store.read(runId, { fromSeq: 1, limit: 1 }).pipe( + Effect.flatMap(([first]) => { + if (first === undefined) { + return Effect.succeed(noAttributes) + } + + if (first.type === 'runtime.run.started' && isJsonObject(first.payload)) { + return Effect.succeed(readAttributes(first.payload.attributes)) + } + + return folded + }), + Effect.catch(() => folded), + ) +} + +/** + * What `authorize` sees for a request about one run: the run's attributes are included when it has + * any. Every path that serves a run, including a WebSocket upgrade, authorizes through this. + */ +export function runRouteInfo( + runtime: LoomsRuntime, + store: EventStore, + route: { readonly access: AuthorizedAccess; readonly name: string; readonly runId: string }, +): Effect.Effect { + return readStoredAttributes(runtime, store, route.runId).pipe( + Effect.map((attributes) => (hasEntries(attributes) ? { ...route, attributes } : route)), + ) +} diff --git a/packages/runtime/src/sse.test.ts b/packages/runtime/src/sse.test.ts index d9f4940..18ecce4 100644 --- a/packages/runtime/src/sse.test.ts +++ b/packages/runtime/src/sse.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from 'bun:test' import { Effect } from 'effect' -import { makeMemoryEventStore } from '@looms/core' +import { createEvent, makeMemoryEventStore } from '@looms/core' import { createEventStreamResponse } from './sse' @@ -55,4 +55,80 @@ describe('createEventStreamResponse', () => { await reader.cancel() abort.abort() }) + + test('tells the client to resync when the cursor was trimmed', async () => { + const store = await Effect.runPromise(makeMemoryEventStore) + + const event = createEvent('run_trim', { + type: 'runtime.thread.started', + payload: { + threadId: 'thr_a', + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v1', + input: null, + parentThreadId: null, + }, + threadId: 'thr_a', + origin: { type: 'system' }, + }) + + const second = createEvent('run_trim', { + type: 'runtime.thread.started', + payload: { + threadId: 'thr_a', + kind: 'workflow', + definitionName: 'checkout', + definitionVersion: 'v1', + input: null, + parentThreadId: null, + }, + threadId: 'thr_a', + origin: { type: 'system' }, + }) + + await Effect.runPromise(store.append('run_trim', [event, second], { expectedTail: 0 })) + + if (!store.trim) { + throw new Error('memory store does not trim') + } + + await Effect.runPromise(store.trim('run_trim', 2)) + + const abort = new AbortController() + + const res = createEventStreamResponse({ + signal: abort.signal, + store, + runId: 'run_trim', + fromSeq: 1, + heartbeatMs: 60_000, + }) + + const reader = res.body?.getReader() + + if (!reader) { + throw new Error('missing stream body') + } + + const decoder = new TextDecoder() + let text = '' + + while (!text.includes('resync')) { + const chunk = await reader.read() + + if (chunk.done) { + break + } + + text += decoder.decode(chunk.value) + } + + expect(text.startsWith(': keepalive\n\n')).toBe(true) + expect(text).toContain('"type":"resync"') + expect(text).toContain('"head":2') + expect(text).toContain('"reason":"trimmed"') + await reader.cancel() + abort.abort() + }) }) diff --git a/packages/runtime/src/sse.ts b/packages/runtime/src/sse.ts index 8e5ef99..a48b292 100644 --- a/packages/runtime/src/sse.ts +++ b/packages/runtime/src/sse.ts @@ -1,6 +1,13 @@ import { Effect, Fiber, Schedule, Stream } from 'effect' -import { encodeLoomsEvent, stringifyJson, type EventEnvelope, type EventStore } from '@looms/core' +import { + encodeLoomsEvent, + encodeSubscriptionFrame, + type EventEnvelope, + type EventStore, +} from '@looms/core' + +import { readSubscriptionBounds } from './subscription' export interface EventStreamOptions { signal: AbortSignal @@ -102,23 +109,52 @@ export function createEventStreamResponse(options: EventStreamOptions): Response } } - subscriptionFiber = Effect.runFork( - store.subscribe(runId, { fromSeq }).pipe( - Stream.tap((event: EventEnvelope) => - Effect.sync(() => { - write( - `id: ${event.seq}\ndata: ${stringifyJson({ batch: [encodeLoomsEvent(event)] })}\n\n`, - ) - }), - ), - Stream.runDrain, - ), - ) - // Send an immediate keepalive so bytes and headers flush over the wire immediately, // preventing dev server proxies and Bun.serve from aborting quiet connections. + // It has to be enqueued before the subscription fiber, which writes hello synchronously. write(': keepalive\n\n') + subscriptionFiber = Effect.runFork( + Effect.gen(function* () { + const bounds = yield* readSubscriptionBounds(store, runId) + + if (fromSeq < bounds.head) { + write( + `data: ${encodeSubscriptionFrame({ + type: 'resync', + head: bounds.head, + reason: 'trimmed', + })}\n\n`, + ) + + close() + return + } + + write( + `data: ${encodeSubscriptionFrame({ + type: 'hello', + head: bounds.head, + tail: bounds.tail, + })}\n\n`, + ) + + yield* store.subscribe(runId, { fromSeq }).pipe( + Stream.tap((event: EventEnvelope) => + Effect.sync(() => { + write( + `id: ${event.seq}\ndata: ${encodeSubscriptionFrame({ + type: 'batch', + batch: [encodeLoomsEvent(event)], + })}\n\n`, + ) + }), + ), + Stream.runDrain, + ) + }), + ) + heartbeatFiber = Effect.runFork( Effect.repeat( Effect.sleep(heartbeatMs).pipe( diff --git a/packages/runtime/src/subscription.ts b/packages/runtime/src/subscription.ts new file mode 100644 index 0000000..903e0b1 --- /dev/null +++ b/packages/runtime/src/subscription.ts @@ -0,0 +1,45 @@ +import { Effect } from 'effect' + +import { + subscriptionReplay, + type EventEnvelope, + type EventStore, + type EventStoreError, + type SubscriptionHello, + type SubscriptionOpening, +} from '@looms/core' + +export function readSubscriptionBounds( + store: EventStore, + runId: string, +): Effect.Effect<{ readonly head: number; readonly tail: number }, EventStoreError> { + if (store.bounds) { + return store.bounds(runId) + } + + return store.tail(runId).pipe(Effect.map((tail) => ({ head: 1, tail }))) +} + +/** Hello plus the retained events, or a resync when `fromSeq` is below the log head. */ +export function loadSubscriptionOpening( + store: EventStore, + runId: string, + fromSeq: number, +): Effect.Effect { + return Effect.gen(function* () { + const bounds = yield* readSubscriptionBounds(store, runId) + const decision = subscriptionReplay(bounds.head, fromSeq) + + if (decision.kind === 'resync') { + return { kind: 'resync', head: decision.head } + } + + const events: readonly EventEnvelope[] = yield* store.read(runId, { + fromSeq: decision.fromSeq, + }) + + const hello: SubscriptionHello = { type: 'hello', head: bounds.head, tail: bounds.tail } + + return { kind: 'open', hello, events } + }) +} diff --git a/packages/runtime/src/trace.test.ts b/packages/runtime/src/trace.test.ts new file mode 100644 index 0000000..6de68bd --- /dev/null +++ b/packages/runtime/src/trace.test.ts @@ -0,0 +1,243 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect, Predicate, Schema } from 'effect' + +import { + RunLauncherTag, + complete, + defineEffect, + defineEventCatalog, + defineRuntimeModule, + defineThread, + invoke, + type TraceContext, +} from '@looms/core' +import { defineWorkflow, workflow } from '@looms/workflow' + +import { createLooms } from './looms' +import type { EffectWorkerTask } from './types' + +const CALLER: TraceContext = { + traceparent: '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01', + tracestate: 'congo=t61rcWkgMzE', +} + +const OVERRIDE: TraceContext = { + traceparent: '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', +} + +const TracedInput = Schema.Struct({ launch: Schema.Boolean, override: Schema.Boolean }) + +const catalog = defineEventCatalog('traced', { + observed: Schema.Struct({}), + launched: Schema.Struct({}), +}) + +const seen = new Map() + +const observe = defineEffect({ + type: 'traced.observe', + execute: (_input, ctx) => { + seen.set(ctx.runId, ctx.trace) + return [{ type: 'traced.observed', payload: {}, threadId: ctx.threadId }] + }, +}) + +const remote = defineEffect({ type: 'traced.remote', execute: () => [] }) + +const launch = defineEffect({ + type: 'traced.launch', + input: TracedInput, + execute: (input, ctx) => + Effect.gen(function* () { + const launcher = yield* RunLauncherTag + + yield* launcher.launch({ + runId: `${ctx.runId}-child`, + kind: 'traced', + definitionName: 'traced', + input: { launch: false, override: false }, + trace: input.override ? OVERRIDE : undefined, + }) + + return [{ type: 'traced.launched', payload: {}, threadId: ctx.threadId }] + }), +}) + +const tracedThread = defineThread({ + kind: 'traced', + input: TracedInput, + initialState: (ctx) => ({ + launch: ctx.input?.launch ?? false, + override: ctx.input?.override ?? false, + observed: false, + launched: false, + }), + step: (state, event) => { + switch (event.type) { + case 'traced.observed': + return { ...state, observed: true } + case 'traced.launched': + return { ...state, launched: true } + default: + return state + } + }, + effects: (state) => { + if (!state.observed) { + return [invoke(observe, {}, 'observe'), invoke(remote, {}, 'remote')] + } + + if (state.launch && !state.launched) { + return [invoke(launch, { launch: false, override: state.override }, 'launch')] + } + + return [complete({ ok: true })] + }, +}) + +const tracedModule = defineRuntimeModule({ + namespace: 'traced', + protocolVersion: '1.0.0', + events: catalog, + threads: { traced: tracedThread }, + effects: { observe, remote, launch }, + definitions: [{ kind: 'traced', name: 'traced', version: 'v1', input: TracedInput }], +}) + +const traced = { kind: 'traced', name: 'traced', version: 'v1' } + +function tracedLooms(dispatched: EffectWorkerTask[] = []) { + return createLooms({ + modules: [tracedModule], + worker: { handles: ['traced.remote'], dispatch: (task) => void dispatched.push(task) }, + }) +} + +describe('trace propagation', () => { + test('carries the caller trace from the start request to effects, workers, and launched runs', async () => { + const dispatched: EffectWorkerTask[] = [] + const looms = tracedLooms(dispatched) + + const response = await looms.fetch( + new Request('http://looms.test/runs', { + method: 'POST', + headers: { 'content-type': 'application/json', ...CALLER }, + body: JSON.stringify({ + kind: 'traced', + definitionName: 'traced', + runId: 'run_trace', + input: { launch: true, override: false }, + }), + }), + ) + + expect(response?.status).toBe(200) + + const started = (await looms.getEvents('run_trace')).find( + (event) => event.type === 'runtime.run.started', + ) + + expect(started?.payload).toMatchObject({ trace: CALLER }) + expect(seen.get('run_trace')).toEqual(CALLER) + expect(dispatched.find((task) => task.runId === 'run_trace')?.trace).toEqual(CALLER) + expect((await looms.getRun('run_trace-child')).startIdentity?.trace).toEqual(CALLER) + + const deadline = Date.now() + 2_000 + + while (!seen.has('run_trace-child') && Date.now() < deadline) { + await Bun.sleep(5) + } + + expect(seen.get('run_trace-child')).toEqual(CALLER) + + await looms.stop() + }) + + test('a launch can name its own trace context', async () => { + const looms = tracedLooms() + + await looms.start( + traced, + { launch: true, override: true }, + { + runId: 'run_override', + trace: CALLER, + }, + ) + + expect((await looms.getRun('run_override-child')).startIdentity?.trace).toEqual(OVERRIDE) + await looms.stop() + }) + + test('drops an invalid traceparent instead of storing it', async () => { + const looms = tracedLooms() + + await looms.start( + traced, + { launch: false, override: false }, + { + runId: 'run_invalid', + trace: { traceparent: 'not-a-trace' }, + }, + ) + + const started = (await looms.getEvents('run_invalid')).find( + (event) => event.type === 'runtime.run.started', + ) + + expect(Predicate.isObject(started?.payload) && 'trace' in started.payload).toBe(false) + expect(seen.get('run_invalid')).toBeNull() + await looms.stop() + }) + + test('a retried start from another trace is the same run', async () => { + const looms = tracedLooms() + const input = { launch: false, override: false } + + await looms.start(traced, input, { runId: 'run_retry', idempotencyKey: 'k', trace: CALLER }) + + const retried = await looms.start(traced, input, { + runId: 'run_retry', + idempotencyKey: 'k', + trace: OVERRIDE, + }) + + expect(retried.runId).toBe('run_retry') + expect((await looms.getRun('run_retry')).startIdentity?.trace).toEqual(CALLER) + await looms.stop() + }) + + test('a workflow child run inherits the parent trace', async () => { + const child = defineWorkflow({ name: 'trace-child', nodes: [{ id: 'out', run: () => 1 }] }) + + const parent = defineWorkflow({ + name: 'trace-parent', + childExecution: 'run', + nodes: [{ id: 'spawn', run: (ctx) => ctx.spawn(child, {}) }], + }) + + const looms = createLooms({ modules: [workflow({ definitions: [child, parent] })] }) + const started = await looms.start(parent, {}, { trace: CALLER }) + const deadline = Date.now() + 2_000 + let state = await looms.getRun(started.runId) + + while (state.status !== 'completed' && Date.now() < deadline) { + await Bun.sleep(5) + state = await looms.getRun(started.runId) + } + + const launched = (await looms.getEvents(started.runId)).find( + (event) => event.type === 'runtime.run.launched', + ) + + const childRunId = + launched && Predicate.isObject(launched.payload) && Predicate.isString(launched.payload.runId) + ? launched.payload.runId + : '' + + expect(state.status).toBe('completed') + expect((await looms.getRun(childRunId)).startIdentity?.trace).toEqual(CALLER) + await looms.stop() + }) +}) diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 850756b..8a8b371 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -3,9 +3,14 @@ import type { Effect } from 'effect' import type { AnyRuntimeModule, ComposedRegistry, + DefinitionResolver, EventEnvelope, EventInput, + WorkerCallback, WorkerCallbackInput, + DefinitionResolveError, + WorkerCallbackResult, + WorkerSecretsResult, EventStore, EventStoreAppendError, EventStoreError, @@ -14,6 +19,9 @@ import type { EventStoreTrimCoverage, RunLauncher, RunLaunchSource, + RunAttributes, + SecretContext, + TraceContext, EventStoreTruncationError, JsonValue, ProjectionDefinition, @@ -42,6 +50,12 @@ export interface CreateRuntimeOptions< TModules extends readonly AnyRuntimeModule[] = readonly AnyRuntimeModule[], > { readonly modules: TModules + /** + * Loads a definition that was not registered on a module. Registered definitions are checked + * first. Lookups are cached, including misses. Start fails with `UnknownDefinitionError` + * and writes no events when the definition is missing. + */ + readonly definitionResolver?: DefinitionResolver readonly store?: EventStore /** * Durable events between mid-wake snapshots. Default 200; `0` disables. @@ -50,6 +64,8 @@ export interface CreateRuntimeOptions< */ readonly snapshotEvery?: number readonly maxWakeIterations?: number + /** Maximum concurrent in-flight worker effects dispatched per run (fairness guard). */ + readonly maxInFlightWorkerEffects?: number /** Maximum active and queued mutation operations per run. Default 256. */ readonly maxPendingRunOperations?: number /** Defaults to the EventStore's attached store or an in-memory SnapshotStore. */ @@ -92,6 +108,25 @@ export interface EffectWorkerTask { readonly attempt: number readonly type: string readonly input: JsonValue + /** Progress from the previous attempt's last heartbeat, when it reported any. */ + readonly details?: JsonValue + /** The run's W3C trace context, for parenting the worker's spans. */ + readonly trace?: TraceContext + /** The attempt's deadlines from the effect's retry policy, when it sets any. */ + readonly deadlines?: EffectWorkerDeadlines + /** + * Present only on the in-memory dispatch. A transport that persists the task must leave this out + * and let the worker read the secret context through an authenticated callback. + */ + readonly secrets?: SecretContext +} + +/** Looms enforces these; a worker can use them to pace heartbeats or give up early. */ +export interface EffectWorkerDeadlines { + /** Epoch ms. A `started` callback after this is stale. */ + readonly scheduleToStartAt?: number + readonly startToCloseTimeoutMs?: number + readonly heartbeatTimeoutMs?: number } export interface EffectWorker { @@ -101,6 +136,14 @@ export interface EffectWorker { readonly canDispatch?: (task: EffectWorkerTask) => boolean dispatch(task: EffectWorkerTask): void | Promise cancel?(task: EffectWorkerTask): void | Promise + /** + * What a rejected `dispatch` means. With `'unacknowledged'` (the default) the task may still have + * arrived, so the attempt waits for its schedule-to-start timeout or an operator. With + * `'acknowledged'` a rejection means the transport refused it, so the attempt fails under the + * retry policy. A transport can still lose an accepted message, so every effect such a worker + * claims must set `retry.scheduleToStartTimeoutMs`. + */ + readonly dispatchSemantics?: 'unacknowledged' | 'acknowledged' } export interface StartRunArgs { @@ -115,6 +158,18 @@ export interface StartRunArgs { launchedBy?: RunLaunchSource /** Recorded on `runtime.run.started` to group related runs. */ correlationId?: string + /** Immutable labels stored on the start event and the run summary. */ + attributes?: RunAttributes + /** + * Bearer material stored beside the log. A retry that sends a different context conflicts. + * Omitting it on a retry keeps the context already stored. + */ + secrets?: SecretContext + /** + * The caller's W3C trace context, recorded on `runtime.run.started` when it is a valid version-00 + * `traceparent`. It is not part of the start identity, so a retry from another trace is not a conflict. + */ + trace?: TraceContext } export interface StartResult { @@ -157,14 +212,23 @@ export interface LoomsRuntime< signal( runId: string, events: ReadonlyArray, - options?: { idempotencyKey?: string }, + options?: { idempotencyKey?: string; coalesce?: boolean }, + ): Effect.Effect + /** + * Append events the runtime itself produced for another run. Unlike `signal`, this does not + * reject reserved runtime event types or rewrite their origin. + */ + accept( + runId: string, + events: ReadonlyArray, + options?: { idempotencyKey?: string; coalesce?: boolean }, ): Effect.Effect wake(runId: string): Effect.Effect getRun( runId: string, ): Effect.Effect< RunState, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError, + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError, EventStoreTag > getEvents( @@ -183,7 +247,7 @@ export interface LoomsRuntime< runId: string, ): Effect.Effect< RunOperationalStatus, - EventStoreError | EventStoreTrimmedError | EventStoreTruncationError, + EventStoreError | EventStoreTrimmedError | EventStoreTruncationError | DefinitionResolveError, EventStoreTag > retryEffect( @@ -203,27 +267,50 @@ export interface LoomsRuntime< sourceRunId: string, options?: { runId?: string }, ): Effect.Effect - /** Applies a worker's report about one effect attempt; stale or foreign reports are ignored. */ + /** + * Applies a worker's report about one effect attempt. A report for another attempt, or one that + * arrives after the attempt settled, is `stale` and changes nothing. + */ workerCallback( runId: string, input: WorkerCallbackInput, - ): Effect.Effect + ): Effect.Effect< + WorkerCallbackResult, + Error | EventStoreAppendError | EventStoreError, + EventStoreTag + > + /** + * The run's secret context for a worker attempt that has started and not settled. Any other + * attempt is `stale`, so a redelivered or superseded message never sees the secrets. + */ + workerSecrets( + runId: string, + callback: WorkerCallback, + ): Effect.Effect< + WorkerSecretsResult, + Error | EventStoreTruncationError | EventStoreTrimmedError | EventStoreError, + EventStoreTag + > readonly listRuns: Effect.Effect /** Indexed run rows. Folds a run only when its header has not been written yet. */ readonly listRunSummaries: Effect.Effect< readonly RunSummary[], - EventStoreError | EventStoreTrimmedError | EventStoreTruncationError | SnapshotStoreError, + | EventStoreError + | EventStoreTrimmedError + | EventStoreTruncationError + | SnapshotStoreError + | DefinitionResolveError, EventStoreTag > readonly rescanTimers: Effect.Effect< number, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError, + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError, EventStoreTag > /** Reschedule every durable timer and effect deadline after restart or repair. */ readonly recoverDeadlines: Effect.Effect< number, - EventStoreTruncationError | EventStoreTrimmedError | EventStoreError, + EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError, EventStoreTag > readonly dispose: Effect.Effect diff --git a/packages/runtime/src/wake-engine.ts b/packages/runtime/src/wake-engine.ts index 341c9d6..26e481c 100644 --- a/packages/runtime/src/wake-engine.ts +++ b/packages/runtime/src/wake-engine.ts @@ -7,12 +7,15 @@ import { foldRun, isRunTerminal, type ComposedRegistry, + type DefinitionResolver, type EventEnvelope, - type RegisteredDefinition, + type EventStore, + type RunCursor, type RunLauncher, type RunState, } from '@looms/core' +import { childCloseCancellations, notifyLaunchingParent } from './child-run' import { nextRuntimeDeadline, planDeadlineTransitions, @@ -44,11 +47,12 @@ function currentTimeMillis(): number { export interface WakeEngineOptions { readonly registry: ComposedRegistry readonly services: ManagedRuntime.ManagedRuntime - readonly definitions: ReadonlyMap + readonly definitionResolver: DefinitionResolver readonly executionContexts: RunExecutionContexts readonly repository: RuntimeRepository readonly snapshotEvery: number readonly maxWakeIterations: number + readonly maxInFlightWorkerEffects?: number readonly worker?: EffectWorker readonly observer?: RuntimeObserver readonly scheduler: () => WakeScheduler @@ -59,7 +63,7 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake const { registry, services, - definitions, + definitionResolver, executionContexts, repository, snapshotEvery, @@ -76,6 +80,40 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake const observe = (event: Parameters[0]) => notifyObserver(options.observer, event) + const closeLaunchedRuns = (store: EventStore, runId: string, cursor: RunCursor) => + Effect.gen(function* () { + let next = cursor + const state = next.state + + if (isRunTerminal(state) && !state.childCloseSent && state.launchedRuns.length > 0) { + const requested = createEvent(runId, { + type: 'runtime.child.close.requested', + payload: { policy: 'terminate' }, + threadId: state.rootThreadId, + origin: { type: 'system' }, + idempotencyKey: `child-close:${runId}`, + }) + + const appended = yield* appendObserved(store, runId, stripSeq([requested])) + next = yield* applyAppend(store, runId, next, stripSeq([requested]), appended) + + yield* Effect.forEach( + state.launchedRuns, + (child) => options.launcher().cancel(child.runId), + { + concurrency: 1, + discard: true, + }, + ) + } + + if (isRunTerminal(next.state)) { + yield* notifyLaunchingParent(options.launcher(), runId, next.state) + } + + return next + }) + return (runId) => Effect.gen(function* () { observe({ type: 'run.wake', runId, at: currentTimeMillis() }) @@ -139,6 +177,7 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake return yield* Effect.gen(function* () { const store = yield* EventStoreTag + const secrets = store.runSecrets ? yield* store.runSecrets.get(runId) : {} const runInContext = Effect.runPromiseWith(yield* Effect.context()) executionContext.liveStore = store @@ -146,6 +185,13 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake do { let cursor = yield* loadCursor(store, runId) + const closeEvents = childCloseCancellations(cursor.state) + + if (closeEvents.length > 0) { + const created = closeEvents.map((input) => createEvent(runId, input)) + const appended = yield* appendObserved(store, runId, stripSeq(created)) + cursor = yield* applyAppend(store, runId, cursor, stripSeq(created), appended) + } if (isRunTerminal(cursor.state)) { const now = yield* Clock.currentTimeMillis @@ -161,6 +207,8 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake cursor = yield* applyAppend(store, runId, cursor, stripSeq(events), appended) } + cursor = yield* closeLaunchedRuns(store, runId, cursor) + const nextDeadline = nextRuntimeDeadline(cursor.state, now) if (nextDeadline !== undefined) { @@ -247,6 +295,8 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake registry, worker: options.worker, now, + maxInFlightWorkerEffects: options.maxInFlightWorkerEffects, + secrets, }) if (workerPlans.length > 0) { @@ -267,7 +317,12 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake ) } - const dispatchEvent = yield* dispatchWorkerEffect(runId, workerPlan, options.worker) + const dispatchEvent = yield* dispatchWorkerEffect( + runId, + workerPlan, + options.worker, + now, + ) const dispatchAppend = yield* appendObserved( store, @@ -378,7 +433,7 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake { registry, services, - definitions, + definitionResolver, executionContext, liveAppends, runInContext, @@ -387,7 +442,12 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake }, { runId, - correlationId: cursor.state.startIdentity?.correlationId ?? runId, + context: { + correlationId: cursor.state.startIdentity?.correlationId ?? runId, + attributes: cursor.state.attributes ?? {}, + secrets, + trace: cursor.state.startIdentity?.trace ?? null, + }, outstanding, attempts, now, @@ -461,6 +521,10 @@ export function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake cursor = yield* applyAppend(store, runId, cursor, stripSeq([completed]), appendResult) } + if (isRunTerminal(cursor.state)) { + cursor = yield* closeLaunchedRuns(store, runId, cursor) + } + // Snapshot on park: the next wake (on any instance) starts from here. cursor = yield* persistSnapshot(store, runId, cursor) const nextDeadline = nextRuntimeDeadline(cursor.state, plannedAt) diff --git a/packages/runtime/src/worker-callbacks.ts b/packages/runtime/src/worker-callbacks.ts index b63c9a0..e49e247 100644 --- a/packages/runtime/src/worker-callbacks.ts +++ b/packages/runtime/src/worker-callbacks.ts @@ -9,7 +9,10 @@ import { type OutstandingEffect, type RetryPolicy, type RunState, + type WorkerCallback, type WorkerCallbackInput, + type WorkerSecretsResult, + type WorkerCallbackResult, } from '@looms/core' import { materializeEffectOutcome } from './helpers' @@ -61,9 +64,28 @@ export interface WorkerCallbacksOptions { interface WorkerCallbackPlan { readonly events: ReadonlyArray readonly notices: ReadonlyArray + readonly result: WorkerCallbackResult } -const IGNORED: WorkerCallbackPlan = { events: [], notices: [] } +const STALE: WorkerCallbackPlan = { + events: [], + notices: [], + result: { status: 'stale', cancelRequested: false }, +} + +const CANCELLED: WorkerCallbackPlan = { + events: [], + notices: [], + result: { status: 'cancelled', cancelRequested: true }, +} + +function accepted( + events: ReadonlyArray, + notices: ReadonlyArray = [], + cancelRequested = false, +): WorkerCallbackPlan { + return { events, notices, result: { status: 'accepted', cancelRequested } } +} /** * Turn a worker callback into events against the current durable state. Callbacks for another @@ -88,6 +110,15 @@ function planWorkerCallback(input: { execution?.attempt === callback.attempt && execution.status === 'cancel_requested' + // A message already handed to a worker cannot be pulled back. Tell it to stop instead. + if ( + execution?.attempt === callback.attempt && + execution.status === 'cancel_requested' && + callback.kind !== 'cancelled' + ) { + return Effect.succeed(CANCELLED) + } + if ( !execution || execution.attempt !== callback.attempt || @@ -97,10 +128,9 @@ function planWorkerCallback(input: { execution.status === 'retry_wait' || execution.status === 'timed_out' || execution.status === 'ambiguous' || - (callback.kind === 'cancelled' && !isCancellationAcknowledgement) || - (execution.status === 'cancel_requested' && callback.kind !== 'cancelled') + (callback.kind === 'cancelled' && !isCancellationAcknowledgement) ) { - return Effect.succeed(IGNORED) + return Effect.succeed(STALE) } const retry = item ? registry.effects.get(item.effect.type)?.retry : undefined @@ -117,8 +147,8 @@ function planWorkerCallback(input: { case 'started': { return Effect.succeed( item && execution.status === 'dispatched' - ? { events: [workerStartedEvent(runId, item, callback.attempt, retry, now)], notices: [] } - : IGNORED, + ? accepted([workerStartedEvent(runId, item, callback.attempt, retry, now)]) + : STALE, ) } @@ -129,38 +159,47 @@ function planWorkerCallback(input: { !item || (!implicitStart && execution.status !== 'started' && execution.status !== 'heartbeat') ) { - return Effect.succeed(IGNORED) + return Effect.succeed(STALE) } - return Effect.succeed({ - events: [ - ...(implicitStart ? [workerStartedEvent(runId, item, callback.attempt, retry, now)] : []), - createEvent(runId, { - type: 'runtime.effect.heartbeat', - payload: { - effectId: item.effectId, - attempt: callback.attempt, - heartbeatAt: now, - deadlineAt: workerDeadline(retry, execution.startedAt ?? now, now), - }, - threadId: item.threadId, - origin: WORKER_ORIGIN, - }), - ], - notices: [notice('effect.heartbeat')], - }) + const recorded = { + effectId: item.effectId, + attempt: callback.attempt, + heartbeatAt: now, + deadlineAt: workerDeadline(retry, execution.startedAt ?? now, now), + } + + const payload = + callback.details === undefined ? recorded : { ...recorded, details: callback.details } + + return Effect.succeed( + accepted( + [ + ...(implicitStart + ? [workerStartedEvent(runId, item, callback.attempt, retry, now)] + : []), + createEvent(runId, { + type: 'runtime.effect.heartbeat', + payload, + threadId: item.threadId, + origin: WORKER_ORIGIN, + }), + ], + [notice('effect.heartbeat')], + ), + ) } case 'complete': { if (!item) { - return Effect.succeed(IGNORED) + return Effect.succeed(STALE) } return Effect.forEach(callback.events, (event) => materializeEffectOutcome(registry.catalogs, runId, item, event), ).pipe( - Effect.map((outcomes) => ({ - events: [ + Effect.map((outcomes) => + accepted([ ...outcomes, createEvent(runId, { type: 'runtime.effect.completed', @@ -170,15 +209,14 @@ function planWorkerCallback(input: { causationId: item.causingEventId, origin: WORKER_ORIGIN, }), - ], - notices: [], - })), + ]), + ), ) } case 'fail': { if (!item) { - return Effect.succeed(IGNORED) + return Effect.succeed(STALE) } const failure = planAttemptFailure({ @@ -189,38 +227,45 @@ function planWorkerCallback(input: { now, retry, origin: WORKER_ORIGIN, + nonRetryable: callback.nonRetryable, + retryAfterMs: callback.retryAfterMs, }) - return Effect.succeed({ - events: [failure.event], - notices: failure.retrying - ? [ - { - type: 'effect.retry', - runId, - at: now, - effectId: item.effectId, - attempt: callback.attempt, - detail: callback.error, - }, - ] - : [], - }) + return Effect.succeed( + accepted( + [failure.event], + failure.retrying + ? [ + { + type: 'effect.retry', + runId, + at: now, + effectId: item.effectId, + attempt: callback.attempt, + detail: callback.error, + }, + ] + : [], + ), + ) } case 'cancelled': { - return Effect.succeed({ - events: [ - createEvent(runId, { - type: 'runtime.effect.cancelled', - payload: { effectId: callback.effectId, attempt: callback.attempt }, - threadId: item?.threadId ?? null, - effectId: callback.effectId, - origin: WORKER_ORIGIN, - }), - ], - notices: [notice('effect.cancellation')], - }) + return Effect.succeed( + accepted( + [ + createEvent(runId, { + type: 'runtime.effect.cancelled', + payload: { effectId: callback.effectId, attempt: callback.attempt }, + threadId: item?.threadId ?? null, + effectId: callback.effectId, + origin: WORKER_ORIGIN, + }), + ], + [notice('effect.cancellation')], + true, + ), + ) } default: { @@ -242,19 +287,44 @@ export function createWorkerCallbacks(options: WorkerCallbacksOptions) { const committed = yield* repository.commit(store, runId, (cursor) => planWorkerCallback({ runId, state: cursor.state, callback, registry, now }).pipe( - Effect.map((plan) => ({ events: plan.events, value: plan.notices })), + Effect.map((plan) => ({ + events: plan.events, + value: { notices: plan.notices, result: plan.result }, + })), ), ) if (committed.append) { - for (const notice of committed.value) { + for (const notice of committed.value.notices) { notifyObserver(options.observer, notice) } } - return committed.cursor.state + return committed.value.result }), - ).pipe(Effect.andThen(options.wake(runId))) + ).pipe(Effect.tap((result) => (result.status === 'stale' ? Effect.void : options.wake(runId)))) + + const workerSecrets = (runId: string, callback: WorkerCallback) => + Effect.gen(function* () { + const store = yield* EventStoreTag + const { state } = yield* repository.loadCursor(store, runId) + const execution = state.effectExecutions[callback.effectId] + + if (execution?.attempt !== callback.attempt || isRunTerminal(state)) { + return { status: 'stale' } satisfies WorkerSecretsResult + } + + if (execution.status === 'cancel_requested') { + return { status: 'cancelled' } satisfies WorkerSecretsResult + } + + if (execution.status !== 'started' && execution.status !== 'heartbeat') { + return { status: 'stale' } satisfies WorkerSecretsResult + } + + const secrets = store.runSecrets ? yield* store.runSecrets.get(runId) : {} + return { status: 'accepted', secrets } satisfies WorkerSecretsResult + }) - return { workerCallback: applyWorkerCallback } + return { workerCallback: applyWorkerCallback, workerSecrets } } diff --git a/packages/runtime/src/worker-effect-dispatch.ts b/packages/runtime/src/worker-effect-dispatch.ts index 478a88d..b1bc907 100644 --- a/packages/runtime/src/worker-effect-dispatch.ts +++ b/packages/runtime/src/worker-effect-dispatch.ts @@ -5,14 +5,18 @@ import { deferredUntil, isPrimitiveEffect, type ComposedRegistry, + type EffectDefinition, type EffectExecutionRecord, type EventEnvelope, type OutstandingEffect, + type RetryPolicy, type RunState, + type SecretContext, } from '@looms/core' import { RuntimeExecutionError } from './errors' import { workerTask } from './helpers' +import { planAttemptFailure } from './retry-policy' import type { EffectWorker, EffectWorkerTask } from './types' export interface WorkerEffectDispatchPlan { @@ -20,6 +24,7 @@ export interface WorkerEffectDispatchPlan { readonly attempt: number readonly queuedEvent?: EventEnvelope readonly task: EffectWorkerTask + readonly retry: RetryPolicy | undefined readonly scheduleToStartDeadlineAt: number | null } @@ -103,67 +108,106 @@ export function planWorkerEffects(input: { readonly registry: ComposedRegistry readonly worker?: EffectWorker readonly now: number + readonly maxInFlightWorkerEffects?: number + readonly secrets?: SecretContext }): ReadonlyArray { - return input.state.outstandingEffects - .filter((item) => { - const dueAt = deferredUntil(item.effect) - - if (dueAt !== undefined && dueAt > input.now + 5) { - return false - } - - const execution = input.state.effectExecutions[item.effectId] - - if (!isWorkerEffect({ ...input, item, execution })) { - return false - } - - return ( - !execution || - execution.status === 'queued' || - (execution.status === 'retry_wait' && - execution.nextAttemptAt !== null && - execution.nextAttemptAt <= input.now + 5) - ) - }) - .map((item) => { - const existing = input.state.effectExecutions[item.effectId] - - const attempt = - existing?.status === 'queued' ? existing.attempt : (existing?.attempt ?? 0) + 1 - - const retry = input.registry.effects.get(item.effect.type)?.retry - - return { - item, - attempt, - queuedEvent: - existing?.status === 'queued' - ? undefined - : createEvent(input.runId, { - type: 'runtime.effect.queued', - payload: { effectId: item.effectId, attempt, deadlineAt: null }, - threadId: item.threadId, - causationId: item.causingEventId, - origin: { type: 'system' }, - }), - task: workerTask(input.runId, item, attempt), - scheduleToStartDeadlineAt: retry?.scheduleToStartTimeoutMs - ? input.now + retry.scheduleToStartTimeoutMs - : null, - } - }) + const workerItems = input.state.outstandingEffects.filter((item) => + isWorkerEffect({ ...input, item, execution: input.state.effectExecutions[item.effectId] }), + ) + + const due = workerItems.filter((item) => { + const dueAt = deferredUntil(item.effect) + + if (dueAt !== undefined && dueAt > input.now + 5) { + return false + } + + const execution = input.state.effectExecutions[item.effectId] + + return ( + !execution || + execution.status === 'queued' || + (execution.status === 'retry_wait' && + execution.nextAttemptAt !== null && + execution.nextAttemptAt <= input.now + 5) + ) + }) + + return withinInFlightLimit(input, workerItems, due).map((item) => { + const existing = input.state.effectExecutions[item.effectId] + + const attempt = existing?.status === 'queued' ? existing.attempt : (existing?.attempt ?? 0) + 1 + + const retry = input.registry.effects.get(item.effect.type)?.retry + + const scheduleToStartDeadlineAt = retry?.scheduleToStartTimeoutMs + ? input.now + retry.scheduleToStartTimeoutMs + : null + + return { + item, + attempt, + retry, + queuedEvent: + existing?.status === 'queued' + ? undefined + : createEvent(input.runId, { + type: 'runtime.effect.queued', + payload: { effectId: item.effectId, attempt, deadlineAt: null }, + threadId: item.threadId, + causationId: item.causingEventId, + origin: { type: 'system' }, + }), + task: workerTask(input.runId, item, attempt, { + details: existing?.heartbeatDetails, + secrets: input.secrets, + trace: input.state.startIdentity?.trace, + retry, + scheduleToStartAt: scheduleToStartDeadlineAt, + }), + scheduleToStartDeadlineAt, + } + }) +} + +const IN_FLIGHT_STATUSES: ReadonlySet = new Set([ + 'dispatched', + 'started', + 'heartbeat', + 'cancel_requested', +]) + +function withinInFlightLimit( + input: { readonly state: RunState; readonly maxInFlightWorkerEffects?: number }, + workerItems: ReadonlyArray, + due: ReadonlyArray, +): ReadonlyArray { + const max = input.maxInFlightWorkerEffects + + if (max === undefined || max <= 0) { + return due + } + + const inFlight = workerItems.filter((item) => { + const status = input.state.effectExecutions[item.effectId]?.status + + return status !== undefined && IN_FLIGHT_STATUSES.has(status) + }).length + + return due.slice(0, Math.max(0, max - inFlight)) } /** - * Hand a planned attempt to the worker. A rejected dispatch may still have reached the worker, so - * with a schedule-to-start deadline it is recorded as dispatched and recovered by that timeout; - * without one it is parked as ambiguous for an operator. + * Hand a planned attempt to the worker. When an unacknowledged dispatch rejects, the task may still + * have reached the worker, so with a schedule-to-start deadline it is recorded as dispatched and + * recovered by that timeout; without one it is parked as ambiguous for an operator. An acknowledged + * rejection means nothing was sent, so the attempt fails under the retry policy. */ export function dispatchWorkerEffect( runId: string, plan: WorkerEffectDispatchPlan, worker: EffectWorker | undefined, + now: number, ): Effect.Effect { const dispatched = createEvent(runId, { type: 'runtime.effect.dispatched', @@ -198,11 +242,63 @@ export function dispatchWorkerEffect( new RuntimeExecutionError(cause instanceof Error ? cause.message : String(cause)), }).pipe( Effect.match({ - onFailure: (error) => - plan.scheduleToStartDeadlineAt === null + onFailure: (error) => { + if (worker.dispatchSemantics === 'acknowledged') { + return planAttemptFailure({ + runId, + item: plan.item, + attempt: plan.attempt, + error: `worker dispatch failed: ${error.message}`, + now, + retry: plan.retry, + origin: { type: 'system' }, + }).event + } + + return plan.scheduleToStartDeadlineAt === null ? ambiguous(`worker dispatch outcome is ambiguous: ${error.message}`) - : dispatched, + : dispatched + }, onSuccess: () => dispatched, }), ) } + +/** + * An acknowledged transport can lose a message it accepted, for example to a dead-letter queue. + * Only a schedule-to-start timeout notices, so every effect such a worker claims needs one. + */ +export function assertWorkerDeadlines( + registry: ComposedRegistry, + worker: EffectWorker | undefined, +): void { + if (worker?.dispatchSemantics !== 'acknowledged') { + return + } + + const missing = [...registry.effects.values()] + .filter( + (definition) => claimsType(worker, definition) && !definition.retry?.scheduleToStartTimeoutMs, + ) + .map((definition) => definition.type) + + if (missing.length > 0) { + throw new Error( + `An acknowledged worker claims effects without retry.scheduleToStartTimeoutMs: ${missing.join(', ')}`, + ) + } +} + +function claimsType(worker: EffectWorker, definition: EffectDefinition): boolean { + if (definition.execution) { + return definition.execution === 'worker' + } + + const { handles } = worker + + const handled = Predicate.isFunction(handles) + ? handles(definition.type) + : (handles?.includes(definition.type) ?? false) + + return handled || !definition.hasHandler +} diff --git a/packages/runtime/src/worker-protocol.test.ts b/packages/runtime/src/worker-protocol.test.ts index fe4f539..6993f75 100644 --- a/packages/runtime/src/worker-protocol.test.ts +++ b/packages/runtime/src/worker-protocol.test.ts @@ -7,6 +7,7 @@ import { defineEffect, defineRuntimeModule, defineThread, + failEffect, invoke, makeMemoryEventStore, type EventEnvelope, @@ -14,7 +15,7 @@ import { type RetryPolicy, } from '@looms/core' -import { bearerAuth, createLooms, type EffectWorkerTask } from './index' +import { bearerAuth, createLooms, createRuntime, type EffectWorkerTask } from './index' const workerDefinition = { kind: 'worker-test', @@ -22,7 +23,10 @@ const workerDefinition = { value: { kind: 'worker-test', name: 'worker-test' }, } -function workerModule(retry: RetryPolicy, execute?: () => ReadonlyArray) { +function workerModule( + retry: RetryPolicy, + execute?: () => ReadonlyArray | Effect.Effect, unknown>, +) { const effect = defineEffect({ type: 'worker.do', retry, @@ -61,6 +65,18 @@ function effectIdentity(events: readonly EventEnvelope[]) { return { effectId: payload.effectId, attempt: payload.attempt } } +async function waitFor(condition: () => boolean, timeoutMs = 1_000): Promise { + const deadline = Date.now() + timeoutMs + + while (!condition()) { + if (Date.now() > deadline) { + throw new Error('condition not met in time') + } + + await Bun.sleep(5) + } +} + function local(): ReadonlyArray { return [{ type: 'worker.done', payload: { local: true } }] } @@ -418,7 +434,9 @@ describe('durable external effect workers', () => { expect(unauthorized?.status).toBe(401) expect(unknownAction?.status).toBe(404) expect(startedResponse?.status).toBe(200) + expect(await startedResponse?.json()).toEqual({ status: 'accepted', cancelRequested: false }) expect(completedResponse?.status).toBe(200) + expect(await completedResponse?.json()).toEqual({ status: 'accepted', cancelRequested: false }) expect( (await looms.getEvents(started.runId)).some((event) => event.type === 'worker.done'), @@ -703,4 +721,228 @@ describe('durable external effect workers', () => { expect((await looms.getRun(started.runId)).status).toBe('cancelled') await looms.stop() }) + + test('an acknowledged dispatch rejection retries under the policy', async () => { + const dispatched: EffectWorkerTask[] = [] + + const looms = createLooms({ + modules: [workerModule({ maxAttempts: 3, backoffMs: 5, scheduleToStartTimeoutMs: 60_000 })], + worker: { + dispatchSemantics: 'acknowledged', + dispatch: (task) => { + dispatched.push(task) + + if (dispatched.length === 1) { + throw new Error('429 Too Many Requests') + } + }, + }, + }) + + const started = await looms.start(workerDefinition, {}) + + await waitFor(() => dispatched.length === 2) + + const types = (await looms.getEvents(started.runId)).map((event) => event.type) + + expect(types).toContain('runtime.effect.retry.scheduled') + expect(types).not.toContain('runtime.effect.ambiguous') + expect(dispatched.map((task) => task.attempt)).toEqual([1, 2]) + await looms.stop() + }) + + test('an acknowledged worker needs a schedule-to-start timeout on every effect it claims', () => { + expect(() => + createRuntime({ + modules: [workerModule({ maxAttempts: 3 })], + worker: { dispatchSemantics: 'acknowledged', dispatch: () => undefined }, + }), + ).toThrow('worker.do') + }) + + test('a worker task carries the deadlines from the retry policy', async () => { + const dispatched: EffectWorkerTask[] = [] + + const looms = createLooms({ + modules: [ + workerModule({ + maxAttempts: 1, + scheduleToStartTimeoutMs: 1_000, + startToCloseTimeoutMs: 2_000, + heartbeatTimeoutMs: 500, + }), + ], + worker: { dispatch: (task) => dispatched.push(task) }, + }) + + const before = Date.now() + + await looms.start(workerDefinition, {}) + + const deadlines = dispatched[0]?.deadlines + + expect(deadlines?.startToCloseTimeoutMs).toBe(2_000) + expect(deadlines?.heartbeatTimeoutMs).toBe(500) + expect(deadlines?.scheduleToStartAt).toBeGreaterThanOrEqual(before + 1_000) + await looms.stop() + }) + + test('a non-retryable failure spends no further attempts', async () => { + const looms = createLooms({ + modules: [ + workerModule({ maxAttempts: 3, backoffMs: 1 }, () => + failEffect({ error: 'bad input', nonRetryable: true }), + ), + ], + }) + + const started = await looms.start(workerDefinition, {}) + const types = (await looms.getEvents(started.runId)).map((event) => event.type) + + expect(types).toContain('runtime.effect.failed') + expect(types).not.toContain('runtime.effect.retry.scheduled') + expect((await looms.getRun(started.runId)).status).toBe('failed') + await looms.stop() + }) + + test('retryAfterMs extends the policy backoff', async () => { + const looms = createLooms({ + modules: [ + workerModule({ maxAttempts: 3, backoffMs: 10 }, () => + failEffect({ error: 'later', retryAfterMs: 60_000 }), + ), + ], + }) + + const before = Date.now() + const started = await looms.start(workerDefinition, {}) + + const payload = (await looms.getEvents(started.runId)).find( + (event) => event.type === 'runtime.effect.retry.scheduled', + )?.payload + + const nextAttemptAt = + Predicate.isReadonlyObject(payload) && !Array.isArray(payload) + ? payload.nextAttemptAt + : undefined + + expect(nextAttemptAt).toBeGreaterThanOrEqual(before + 60_000) + + await looms.stop() + }) + + test('maxInFlightWorkerEffects caps planned dispatches per run', async () => { + const dispatched: EffectWorkerTask[] = [] + + const multiEffect = defineEffect({ type: 'worker.multi' }) + + const fanoutModule = defineRuntimeModule({ + namespace: 'fanout', + protocolVersion: '1.0.0', + effects: { multiEffect }, + threads: { + fanout: defineThread({ + kind: 'fanout', + initialState: () => ({}), + step: (state) => state, + effects: () => [ + invoke(multiEffect, {}, 'm1'), + invoke(multiEffect, {}, 'm2'), + invoke(multiEffect, {}, 'm3'), + invoke(multiEffect, {}, 'm4'), + ], + }), + }, + definitions: [{ kind: 'fanout', name: 'fanout', version: 'v1' }], + }) + + const looms = createLooms({ + modules: [fanoutModule], + maxInFlightWorkerEffects: 2, + worker: { + dispatch: (task) => { + dispatched.push(task) + }, + }, + }) + + const started = await looms.start({ kind: 'fanout', name: 'fanout', version: 'v1' }, {}) + + expect(dispatched).toHaveLength(2) + + const first = { effectId: dispatched[0]?.effectId ?? '', attempt: 1 } + + await looms.workerCallback(started.runId, { ...first, kind: 'started' }) + await looms.workerCallback(started.runId, { ...first, kind: 'complete', events: [] }) + + expect(dispatched).toHaveLength(3) + + await looms.stop() + }) + + test('heartbeat details resume on the next worker attempt', async () => { + const dispatched: EffectWorkerTask[] = [] + + const looms = createLooms({ + modules: [workerModule({ maxAttempts: 3, backoffMs: 5 })], + worker: { dispatch: (task) => dispatched.push(task) }, + }) + + const started = await looms.start(workerDefinition, {}) + const identity = effectIdentity(await looms.getEvents(started.runId)) + + const heartbeat = await looms.workerCallback(started.runId, { + ...identity, + kind: 'heartbeat', + details: { page: 2 }, + }) + + expect(heartbeat).toEqual({ status: 'accepted', cancelRequested: false }) + + const failed = await looms.workerCallback(started.runId, { + ...identity, + kind: 'fail', + error: 'transient', + }) + + expect(failed).toEqual({ status: 'accepted', cancelRequested: false }) + await Bun.sleep(40) + + expect(dispatched.at(-1)?.attempt).toBe(2) + expect(dispatched.at(-1)?.details).toEqual({ page: 2 }) + + const stale = await looms.workerCallback(started.runId, { + ...identity, + kind: 'heartbeat', + details: { page: 9 }, + }) + + expect(stale).toEqual({ status: 'stale', cancelRequested: false }) + await looms.stop() + }) + + test('a heartbeat during cancellation tells the worker to stop', async () => { + const looms = createLooms({ + modules: [workerModule({ maxAttempts: 1, cancellationTimeoutMs: 60_000 })], + worker: { dispatch: () => undefined, cancel: () => undefined }, + }) + + const started = await looms.start(workerDefinition, {}) + const identity = effectIdentity(await looms.getEvents(started.runId)) + await looms.workerCallback(started.runId, { ...identity, kind: 'started' }) + await looms.cancel(started.runId) + + const heartbeat = await looms.workerCallback(started.runId, { + ...identity, + kind: 'heartbeat', + details: { page: 1 }, + }) + + expect(heartbeat).toEqual({ status: 'cancelled', cancelRequested: true }) + + const events = await looms.getEvents(started.runId) + const heartbeats = events.filter((event) => event.type === 'runtime.effect.heartbeat') + expect(heartbeats).toHaveLength(0) + await looms.stop() + }) }) diff --git a/packages/schedule/src/definition-resolver.test.ts b/packages/schedule/src/definition-resolver.test.ts new file mode 100644 index 0000000..f21fb52 --- /dev/null +++ b/packages/schedule/src/definition-resolver.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, test } from 'bun:test' + +import { Effect } from 'effect' + +import { + makeMemoryEventStore, + type DefinitionResolver, + type RunSummary, + type SnapshotStore, +} from '@looms/core' +import { createLooms } from '@looms/runtime' + +import { defineSchedule } from './definitions' +import { schedule } from './module' +import { every } from './specs' + +function discardingSnapshots(): SnapshotStore { + const headers: RunSummary[] = [] + + return { + loadLatest: () => Effect.succeedNone, + save: () => Effect.void, + saveHeader: (header) => + Effect.sync(() => { + const index = headers.findIndex((item) => item.runId === header.runId) + + if (index >= 0) { + headers[index] = header + return + } + + headers.push(header) + }), + get listHeaders() { + return Effect.succeed(headers) + }, + } +} + +describe('schedule definition resolver', () => { + test('replay reads misfire from the resolved definition', async () => { + const nightly = defineSchedule({ + name: 'nightly', + version: 'v3', + target: { kind: 'workflow', name: 'job', version: 'v1' }, + spec: every(1), + misfire: 'skip', + }) + + const resolver: DefinitionResolver = { + peek: () => undefined, + resolve: (kind, name, version) => + Effect.succeed( + kind === 'schedule' && name === 'nightly' && version === 'v3' ? nightly : undefined, + ), + } + + const store = await Effect.runPromise(makeMemoryEventStore) + const snapshots = discardingSnapshots() + + const looms = createLooms({ + modules: [schedule({ minIntervalMs: 60_000 })], + store, + snapshotStore: snapshots, + definitionResolver: resolver, + rescanTimers: false, + }) + + try { + const started = await looms.start(nightly, null, { runId: 'nightly' }) + const thread = Object.values(started.state.threads).find((item) => item.kind === 'schedule') + expect(thread?.state).toMatchObject({ misfire: 'skip' }) + + const reloaded = createLooms({ + modules: [schedule({ minIntervalMs: 60_000 })], + store, + snapshotStore: snapshots, + definitionResolver: resolver, + rescanTimers: false, + }) + + try { + const state = await reloaded.getRun('nightly') + const replayed = Object.values(state.threads).find((item) => item.kind === 'schedule') + expect(replayed?.state).toMatchObject({ misfire: 'skip' }) + } finally { + await reloaded.stop() + } + } finally { + await looms.stop() + } + }) +}) diff --git a/packages/schedule/src/definitions-store.ts b/packages/schedule/src/definitions-store.ts index 0a53a1b..79fa719 100644 --- a/packages/schedule/src/definitions-store.ts +++ b/packages/schedule/src/definitions-store.ts @@ -17,5 +17,5 @@ export function makeScheduleDefinitionStore( return makeDefinitionStore('schedule', definitions) } -export const ScheduleDefinitionsLive = (definitions: ReadonlyArray) => - Layer.succeed(ScheduleDefinitionsTag, makeScheduleDefinitionStore(definitions)) +export const ScheduleDefinitionsLive = (definitions: ScheduleDefinitionStore) => + Layer.succeed(ScheduleDefinitionsTag, definitions) diff --git a/packages/schedule/src/effects.ts b/packages/schedule/src/effects.ts index 6667037..5ced07d 100644 --- a/packages/schedule/src/effects.ts +++ b/packages/schedule/src/effects.ts @@ -86,7 +86,7 @@ export const fireEffect = scheduleModule.effect({ const launcher = yield* RunLauncherTag const now = yield* Clock.currentTimeMillis - const definition = definitions.get(input.definitionName, input.definitionVersion) + const definition = yield* definitions.resolve(input.definitionName, input.definitionVersion) if (!definition) { return yield* new ScheduleEffectError({ diff --git a/packages/schedule/src/module.ts b/packages/schedule/src/module.ts index dcb2187..cdacd6e 100644 --- a/packages/schedule/src/module.ts +++ b/packages/schedule/src/module.ts @@ -1,4 +1,4 @@ -import { defineModule } from '@looms/core' +import { defineModule, makeDefinitionStore } from '@looms/core' import type { ScheduleDefinition } from './definitions' import { ScheduleDefinitionsLive } from './definitions-store' @@ -15,14 +15,18 @@ export interface ScheduleModuleOptions { } export function schedule(options: ScheduleModuleOptions = {}) { - const definitions = options.definitions ?? [] + const listed = options.definitions ?? [] + const definitions = makeDefinitionStore('schedule', listed) const minIntervalMs = options.minIntervalMs ?? DEFAULT_MIN_INTERVAL_MS return defineModule(scheduleModule, () => ({ - definitions, + definitions: listed, threads: { schedule: createScheduleThread(definitions, minIntervalMs) }, effects: { plan: planEffect, fire: fireEffect }, projections: { scheduleStatus }, services: () => ScheduleDefinitionsLive(definitions), + bindHost(host) { + definitions.useResolver(host.definitionResolver) + }, })) } diff --git a/packages/schedule/src/threads.ts b/packages/schedule/src/threads.ts index 4964dff..85a8080 100644 --- a/packages/schedule/src/threads.ts +++ b/packages/schedule/src/threads.ts @@ -2,11 +2,11 @@ import { Option, Schema } from 'effect' import { complete, - definitionKey, emit, fail, invoke, stringifyJson, + type DefinitionStore, type JsonValue, type RuntimeEffect, type ThreadDefinition, @@ -47,24 +47,15 @@ const decodeStart = Schema.decodeUnknownOption(ScheduleStartSchema) const decodeUpdate = Schema.decodeUnknownOption(ScheduleUpdatedPayloadSchema) export function createScheduleThread( - definitions: readonly ScheduleDefinition[], + definitions: DefinitionStore, minIntervalMs: number, ): ThreadDefinition { - const byKey = new Map( - definitions.map( - (definition) => - [definitionKey(definition.kind, definition.name, definition.version), definition] as const, - ), - ) - return scheduleModule.thread({ kind: 'schedule', input: ScheduleStartSchema, forgetsCompletedEffects: true, initialState: (ctx) => { - const definition = byKey.get( - definitionKey('schedule', ctx.definitionName, ctx.definitionVersion), - ) + const definition = definitions.get(ctx.definitionName, ctx.definitionVersion) const base = { input: null, @@ -178,7 +169,7 @@ export function createScheduleThread( } case 'schedule.updated': - return applyUpdate(state, event.payload, byKey, minIntervalMs) + return applyUpdate(state, event.payload, definitions, minIntervalMs) case 'schedule.paused': return { ...state, paused: true, nextAt: null } @@ -283,7 +274,7 @@ export function createScheduleThread( function applyUpdate( state: ScheduleState, payload: JsonValue, - byKey: ReadonlyMap, + definitions: DefinitionStore, minIntervalMs: number, ): ScheduleState { const decoded = decodeUpdate(payload) @@ -295,7 +286,7 @@ function applyUpdate( const patch = decoded.value const definitionName = patch.definitionName ?? state.definitionName const definitionVersion = patch.definitionVersion ?? state.definitionVersion - const definition = byKey.get(definitionKey('schedule', definitionName, definitionVersion)) + const definition = definitions.get(definitionName, definitionVersion) if ( (patch.definitionName !== undefined || patch.definitionVersion !== undefined) && diff --git a/packages/workflow/src/definitions-store.ts b/packages/workflow/src/definitions-store.ts index f66d318..fbc17bb 100644 --- a/packages/workflow/src/definitions-store.ts +++ b/packages/workflow/src/definitions-store.ts @@ -17,5 +17,5 @@ export function makeWorkflowDefinitionStore( return makeDefinitionStore('workflow', definitions) } -export const WorkflowDefinitionsLive = (definitions: ReadonlyArray) => - Layer.succeed(WorkflowDefinitionsTag, makeWorkflowDefinitionStore(definitions)) +export const WorkflowDefinitionsLive = (definitions: WorkflowDefinitionStore) => + Layer.succeed(WorkflowDefinitionsTag, definitions) diff --git a/packages/workflow/src/definitions.ts b/packages/workflow/src/definitions.ts index b0dc151..ef37418 100644 --- a/packages/workflow/src/definitions.ts +++ b/packages/workflow/src/definitions.ts @@ -127,9 +127,24 @@ export interface WorkflowDefinition< readonly inputLimitBytes?: number readonly outputLimitBytes?: number readonly checkpointLimitBytes?: number + /** + * `thread` keeps spawned, mapped, and looped children in this run. + * `run` launches each child as its own run. + */ + readonly childExecution?: 'thread' | 'run' + /** + * When this run is cancelled, `terminate` cancels child runs immediately and `wait` cancels + * this run only after those children have finished cancelling. Ignored when children are threads. + */ + readonly childClose?: 'terminate' | 'wait' output?(ctx: { input: TInput; results: { [nodeId: string]: JsonValue | null } }): TOutput } +export interface WorkflowChildMode { + readonly execution: 'thread' | 'run' + readonly close: 'terminate' | 'wait' +} + export type AnyWorkflowDefinition = WorkflowDefinition function compatibleNodeStatus(status: string | undefined): NodeStatus { diff --git a/packages/workflow/src/effects.ts b/packages/workflow/src/effects.ts index 3b5074f..dab34bf 100644 --- a/packages/workflow/src/effects.ts +++ b/packages/workflow/src/effects.ts @@ -13,6 +13,7 @@ import { nodeResultBuilders, type NodeResult, type NodeState, + type WorkflowChildMode, type WorkflowDefinition, type WorkflowIteration, } from './definitions' @@ -97,11 +98,16 @@ export const scheduleEffect = workflowModule.effect({ const workflows = yield* WorkflowDefinitionsTag const definition = input.definitionName - ? workflows.get(input.definitionName, input.definitionVersion) + ? yield* workflows.resolve(input.definitionName, input.definitionVersion) : undefined if (!definition) { - return [] + return input.definitionName + ? threadFailed( + ctx.threadId, + `Unknown workflow definition ${input.definitionName}@${input.definitionVersion}`, + ) + : [] } const nodes = { ...input.nodes } satisfies Record @@ -236,7 +242,7 @@ export const runNodeEffect = workflowModule.effect({ const workflows = yield* WorkflowDefinitionsTag const definition = input.definitionName - ? workflows.get(input.definitionName, input.definitionVersion) + ? yield* workflows.resolve(input.definitionName, input.definitionVersion) : undefined const run: NodeRun = { @@ -352,7 +358,7 @@ function nodeResultEvents( ? nodeChildWorkflowId(threadId, nodeId, attempt) : createThreadId()) - return spawnRequested(run, childThreadId, result, result.input) + return spawnRequested(definition, run, childThreadId, result, result.input) } case 'sleep': { @@ -401,6 +407,7 @@ function nodeResultEvents( items: result.items, concurrency: result.concurrency, limitBytes, + ...childPolicy(definition), }, threadId, }, @@ -415,7 +422,7 @@ function nodeResultEvents( } const childThreadId = iterationChildWorkflowId(threadId, nodeId, 'loop', attempt, index) - return spawnRequested(run, childThreadId, result, result.input, index) + return spawnRequested(definition, run, childThreadId, result, result.input, index) } default: { @@ -426,7 +433,15 @@ function nodeResultEvents( }) } +function childPolicy(definition: WorkflowDefinition): WorkflowChildMode { + return { + execution: definition.childExecution ?? 'thread', + close: definition.childClose ?? 'terminate', + } +} + function spawnRequested( + definition: WorkflowDefinition, run: NodeRun, childThreadId: string, child: { readonly kind: string; readonly name: string; readonly version: string }, @@ -445,6 +460,7 @@ function spawnRequested( definitionVersion: child.version, input, loopIndex, + ...childPolicy(definition), }, threadId: run.threadId, }, diff --git a/packages/workflow/src/events.ts b/packages/workflow/src/events.ts index 0695e46..f06d2ed 100644 --- a/packages/workflow/src/events.ts +++ b/packages/workflow/src/events.ts @@ -42,6 +42,12 @@ export const WorkflowSpawnRequestedPayloadSchema = Schema.Struct({ input: Schema.Json, /** Set for `loop` iterations: the node re-runs after this child instead of finishing. */ loopIndex: Schema.NullOr(Schema.Finite).pipe(Schema.withDecodingDefaultKey(Effect.succeed(null))), + execution: Schema.Union([Schema.Literal('thread'), Schema.Literal('run')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('thread' as const)), + ), + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('terminate' as const)), + ), }) export const WorkflowMapRequestedPayloadSchema = Schema.Struct({ @@ -53,6 +59,12 @@ export const WorkflowMapRequestedPayloadSchema = Schema.Struct({ items: Schema.Array(Schema.Json), concurrency: Schema.Finite, limitBytes: Schema.Finite, + execution: Schema.Union([Schema.Literal('thread'), Schema.Literal('run')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('thread' as const)), + ), + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('terminate' as const)), + ), }) export const WorkflowSleepRequestedPayloadSchema = Schema.Struct({ diff --git a/packages/workflow/src/module.ts b/packages/workflow/src/module.ts index 13ed970..6cd5cbf 100644 --- a/packages/workflow/src/module.ts +++ b/packages/workflow/src/module.ts @@ -1,4 +1,4 @@ -import { defineModule } from '@looms/core' +import { defineModule, makeDefinitionStore } from '@looms/core' import type { WorkflowDefinition } from './definitions' import { WorkflowDefinitionsLive } from './definitions-store' @@ -12,9 +12,11 @@ export interface WorkflowModuleOptions { } export function workflow(options: WorkflowModuleOptions = {}) { - const definitions = options.definitions ?? [] + const listed = options.definitions ?? [] + const definitions = makeDefinitionStore('workflow', listed) + return defineModule(workflowModule, () => ({ - definitions, + definitions: listed, threads: { workflow: workflowThread }, effects: { schedule: scheduleEffect, @@ -22,5 +24,8 @@ export function workflow(options: WorkflowModuleOptions = {}) { }, projections: { nodes }, services: () => WorkflowDefinitionsLive(definitions), + bindHost(host) { + definitions.useResolver(host.definitionResolver) + }, })) } diff --git a/packages/workflow/src/threads.ts b/packages/workflow/src/threads.ts index c31e75f..52040e2 100644 --- a/packages/workflow/src/threads.ts +++ b/packages/workflow/src/threads.ts @@ -2,14 +2,17 @@ import { Effect, Predicate, Schema } from 'effect' import { cancel, + cancelRun, createWaitId, DEFAULT_DEFINITION_VERSION, + deterministicRunId, EventInputSchema, emit, fail, invoke, isJsonObject, isWithdrawnError, + launch, RuntimeEffectSchema, spawn, stableEffectId, @@ -21,7 +24,7 @@ import { waitOutcome, } from '@looms/core' -import { NodeStateSchema, type NodeState } from './definitions' +import { NodeStateSchema, type NodeState, type WorkflowChildMode } from './definitions' import { runNodeEffect, scheduleEffect } from './effects' import { iterationChildWorkflowId, nodeRunKey, WorkflowValueTooLargeError } from './graph' import { workflowModule } from './scope' @@ -39,6 +42,12 @@ export const WorkflowPendingSpawnSchema = Schema.Struct({ attempt: attemptNumber, input: Schema.Json, loopIndex: Schema.NullOr(Schema.Finite).pipe(Schema.withDecodingDefaultKey(Effect.succeed(null))), + execution: Schema.Union([Schema.Literal('thread'), Schema.Literal('run')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('thread' as const)), + ), + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('terminate' as const)), + ), }) export type WorkflowPendingSpawn = Schema.Schema.Type @@ -54,6 +63,12 @@ export const WorkflowPendingMapSchema = Schema.Struct({ /** Items before this index have been started; later ones wait for a free slot. */ nextIndex: Schema.Finite, outputs: Schema.Record(Schema.String, Schema.Json), + execution: Schema.Union([Schema.Literal('thread'), Schema.Literal('run')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('thread' as const)), + ), + close: Schema.Union([Schema.Literal('terminate'), Schema.Literal('wait')]).pipe( + Schema.withDecodingDefaultKey(Effect.succeed('terminate' as const)), + ), }) export type WorkflowPendingMap = Schema.Schema.Type @@ -136,12 +151,37 @@ function runNodeInvocation(state: WorkflowState, nodeId: string) { } function spawnAndWait( + runId: string, threadId: string, - child: Pick, + child: Pick< + WorkflowPendingSpawn, + 'kind' | 'definitionName' | 'definitionVersion' | 'execution' | 'close' + >, childThreadId: string, input: JsonValue, tag: { readonly [key: string]: JsonValue }, ): RuntimeEffect[] { + if (child.execution === 'run') { + const childRunId = deterministicRunId(runId, childThreadId) + + return [ + launch({ + runId: childRunId, + kind: child.kind, + definitionName: child.definitionName, + definitionVersion: child.definitionVersion, + input, + close: child.close, + tag: `launch_${childRunId}`, + }), + wait({ + waitId: createWaitId(threadId, `spawn_${childRunId}`), + on: { type: 'runtime.child.settled', match: { runId: childRunId } }, + tag, + }), + ] + } + return [ spawn({ childThreadId, @@ -161,6 +201,16 @@ function spawnAndWait( ] } +function childMode( + execution: JsonValue | undefined, + close: JsonValue | undefined, +): WorkflowChildMode { + return { + execution: execution === 'run' ? 'run' : 'thread', + close: close === 'wait' ? 'wait' : 'terminate', + } +} + function mapChildThreadId(threadId: string, map: WorkflowPendingMap, index: number): string { return iterationChildWorkflowId(threadId, map.nodeId, 'map', map.attempt, index) } @@ -171,7 +221,7 @@ interface NodeEffect { } /** Every effect a node currently needs, attributed to that node. */ -function nodeEffects(state: WorkflowState, threadId: string): NodeEffect[] { +function nodeEffects(state: WorkflowState, threadId: string, runId: string): NodeEffect[] { const effects: NodeEffect[] = [] const add = (nodeId: string, items: readonly RuntimeEffect[]) => { @@ -189,7 +239,7 @@ function nodeEffects(state: WorkflowState, threadId: string): NodeEffect[] { add( item.nodeId, - spawnAndWait(threadId, item, item.childThreadId, item.input, { + spawnAndWait(runId, threadId, item, item.childThreadId, item.input, { ...tag, loopIndex: item.loopIndex, }), @@ -202,6 +252,7 @@ function nodeEffects(state: WorkflowState, threadId: string): NodeEffect[] { add( map.nodeId, spawnAndWait( + runId, threadId, map, mapChildThreadId(threadId, map, index), @@ -226,29 +277,75 @@ function nodeEffects(state: WorkflowState, threadId: string): NodeEffect[] { return effects } -function nodeIdForEffect(state: WorkflowState, threadId: string, effectId: string): string | null { - const match = nodeEffects(state, threadId).find( +function nodeIdForEffect( + state: WorkflowState, + threadId: string, + runId: string, + effectId: string, +): string | null { + const match = nodeEffects(state, threadId, runId).find( (item) => stableEffectId(threadId, item.effect) === effectId, ) return match?.nodeId ?? null } +interface UnsettledChild { + readonly key: string + readonly target: string + readonly execution: 'thread' | 'run' + readonly close: 'terminate' | 'wait' +} + +function childTarget(runId: string, childThreadId: string, execution: 'thread' | 'run'): string { + return execution === 'run' ? deterministicRunId(runId, childThreadId) : childThreadId +} + /** Children a node started that have not reported back yet. */ -function unsettledChildren(state: WorkflowState, nodeId: string, threadId: string): string[] { - const children = state.pendingSpawns - .filter((item) => item.nodeId === nodeId) - .map((item) => item.childThreadId) +function unsettledChildren( + state: WorkflowState, + nodeId: string, + threadId: string, + runId: string, +): UnsettledChild[] { + const children: UnsettledChild[] = [] + + for (const item of state.pendingSpawns) { + if (item.nodeId !== nodeId) { + continue + } + + const execution = item.execution === 'run' ? 'run' : 'thread' + + children.push({ + key: item.childThreadId, + target: childTarget(runId, item.childThreadId, execution), + execution, + close: item.close === 'wait' ? 'wait' : 'terminate', + }) + } for (const map of state.pendingMaps) { if (map.nodeId !== nodeId) { continue } + const execution = map.execution === 'run' ? 'run' : 'thread' + const close = map.close === 'wait' ? 'wait' : 'terminate' + for (let index = 0; index < map.nextIndex; index++) { - if (!(String(index) in map.outputs)) { - children.push(mapChildThreadId(threadId, map, index)) + if (String(index) in map.outputs) { + continue } + + const key = mapChildThreadId(threadId, map, index) + + children.push({ + key, + target: childTarget(runId, key, execution), + execution, + close, + }) } } @@ -257,32 +354,55 @@ function unsettledChildren(state: WorkflowState, nodeId: string, threadId: strin /** * Drops a node's pending work and cancels the children it leaves running. `settled` is the child - * whose result is finishing the node, which must not be cancelled. + * whose result is finishing the node, which must not be cancelled. A `wait` child run stays + * pending until it settles, so the node does not finish ahead of that cancellation. */ function withoutPendingWork( state: WorkflowState, nodeId: string, threadId: string, + runId: string, settled: string | null = null, ): WorkflowState { - const cancels = unsettledChildren(state, nodeId, threadId).filter( - (childThreadId) => childThreadId !== settled && !state.pendingCancels.includes(childThreadId), + const children = unsettledChildren(state, nodeId, threadId, runId) + + const cancels = children + .map((child) => child.target) + .filter((target) => target !== settled && !state.pendingCancels.includes(target)) + + const keptSpawns = new Set( + children + .filter( + (child) => child.execution === 'run' && child.close === 'wait' && child.target !== settled, + ) + .map((child) => child.key), + ) + + const keepMap = children.some( + (child) => child.execution === 'run' && child.close === 'wait' && child.target !== settled, ) return { ...state, pendingCancels: cancels.length === 0 ? state.pendingCancels : [...state.pendingCancels, ...cancels], - pendingSpawns: state.pendingSpawns.filter((item) => item.nodeId !== nodeId), - pendingMaps: state.pendingMaps.filter((item) => item.nodeId !== nodeId), + pendingSpawns: state.pendingSpawns.filter( + (item) => item.nodeId !== nodeId || keptSpawns.has(item.childThreadId), + ), + pendingMaps: state.pendingMaps.filter((item) => item.nodeId !== nodeId || keepMap), pendingSleeps: state.pendingSleeps.filter((item) => item.nodeId !== nodeId), pendingEffects: state.pendingEffects.filter((item) => item.nodeId !== nodeId), } } -function stopNode(state: WorkflowState, nodeId: string, threadId: string): WorkflowState { +function stopNode( + state: WorkflowState, + nodeId: string, + threadId: string, + runId: string, +): WorkflowState { return { - ...withoutPendingWork(state, nodeId, threadId), + ...withoutPendingWork(state, nodeId, threadId, runId), runningNodes: state.runningNodes.filter((id) => id !== nodeId), } } @@ -294,6 +414,7 @@ function finishNode( result: JsonValue | null, error: string | null, threadId: string, + runId: string, settled: string | null = null, ): WorkflowState { const id = `finished_${nodeId}` @@ -305,7 +426,7 @@ function finishNode( } return { - ...withoutPendingWork(state, nodeId, threadId, settled), + ...withoutPendingWork(state, nodeId, threadId, runId, settled), pendingEmits: [...state.pendingEmits.filter((item) => item.id !== id), { id, event }], } } @@ -316,23 +437,25 @@ function recordMapOutput( index: number, output: JsonValue | null, threadId: string, + runId: string, ): WorkflowState { if (String(index) in map.outputs) { return state } - const settled = mapChildThreadId(threadId, map, index) + const childThreadId = mapChildThreadId(threadId, map, index) + const settled = childTarget(runId, childThreadId, map.execution === 'run' ? 'run' : 'thread') const outputs = { ...map.outputs, [String(index)]: output } const size = utf8JsonBytes(outputs) if (size > map.limitBytes) { const error = new WorkflowValueTooLargeError(`Node ${map.nodeId} outputs`, size, map.limitBytes) - return finishNode(state, map.nodeId, null, error.message, threadId, settled) + return finishNode(state, map.nodeId, null, error.message, threadId, runId, settled) } if (Object.keys(outputs).length === map.items.length) { const ordered = map.items.map((_, i) => outputs[String(i)] ?? null) - return finishNode(state, map.nodeId, ordered, null, threadId, settled) + return finishNode(state, map.nodeId, ordered, null, threadId, runId, settled) } const nextIndex = Math.min(map.items.length, map.nextIndex + 1) @@ -437,7 +560,7 @@ export const workflowThread = workflowModule.thread({ const failed = Predicate.isString(error) && error.length > 0 return { - ...stopNode(state, nodeId, ctx.threadId), + ...stopNode(state, nodeId, ctx.threadId, ctx.runId), nodes: { ...state.nodes, [nodeId]: { @@ -457,7 +580,7 @@ export const workflowThread = workflowModule.thread({ case 'workflow.node.skipped': { const { nodeId, reason, cause } = event.payload return { - ...stopNode(state, nodeId, ctx.threadId), + ...stopNode(state, nodeId, ctx.threadId, ctx.runId), nodes: { ...state.nodes, [nodeId]: { @@ -483,7 +606,10 @@ export const workflowThread = workflowModule.thread({ ...state, pendingSpawns: [ ...state.pendingSpawns.filter((item) => item.nodeId !== nodeId), - event.payload, + { + ...event.payload, + ...childMode(event.payload.execution, event.payload.close), + }, ], } } @@ -501,6 +627,7 @@ export const workflowThread = workflowModule.thread({ ...state.pendingMaps.filter((item) => item.nodeId !== nodeId), { ...event.payload, + ...childMode(event.payload.execution, event.payload.close), nextIndex: Math.min(event.payload.concurrency, event.payload.items.length), outputs: {}, }, @@ -568,7 +695,7 @@ export const workflowThread = workflowModule.thread({ const effectId = event.payload.effectId const nodeId = Predicate.isString(effectId) - ? nodeIdForEffect(state, ctx.threadId, effectId) + ? nodeIdForEffect(state, ctx.threadId, ctx.runId, effectId) : null if (!nodeId) { @@ -578,7 +705,14 @@ export const workflowThread = workflowModule.thread({ const message = `Node ${nodeId} failed: ${error}` return { - ...finishNode(stopNode(state, nodeId, ctx.threadId), nodeId, null, message, ctx.threadId), + ...finishNode( + stopNode(state, nodeId, ctx.threadId, ctx.runId), + nodeId, + null, + message, + ctx.threadId, + ctx.runId, + ), nodes: { ...state.nodes, [nodeId]: { @@ -608,7 +742,7 @@ export const workflowThread = workflowModule.thread({ const map = state.pendingMaps.find((item) => item.nodeId === nodeId) if (error === null && map && Predicate.isNumber(tag.index)) { - return recordMapOutput(state, map, tag.index, result, ctx.threadId) + return recordMapOutput(state, map, tag.index, result, ctx.threadId, ctx.runId) } const node = state.nodes[nodeId] @@ -627,12 +761,15 @@ export const workflowThread = workflowModule.thread({ const satisfied = event.payload.event const satisfiedPayload = Predicate.isObject(satisfied) ? satisfied.payload : undefined - const settled = - Predicate.isObject(satisfiedPayload) && Predicate.isString(satisfiedPayload.threadId) - ? satisfiedPayload.threadId - : null + const settled = Predicate.isObject(satisfiedPayload) + ? Predicate.isString(satisfiedPayload.runId) + ? satisfiedPayload.runId + : Predicate.isString(satisfiedPayload.threadId) + ? satisfiedPayload.threadId + : null + : null - return finishNode(state, nodeId, result, error, ctx.threadId, settled) + return finishNode(state, nodeId, result, error, ctx.threadId, ctx.runId, settled) } default: @@ -646,7 +783,7 @@ export const workflowThread = workflowModule.thread({ effects.push(scheduleInvocation(state)) } - for (const item of nodeEffects(state, ctx.threadId)) { + for (const item of nodeEffects(state, ctx.threadId, ctx.runId)) { effects.push(item.effect) } @@ -654,8 +791,10 @@ export const workflowThread = workflowModule.thread({ effects.push(emit(emitReq.event)) } - for (const childThreadId of state.pendingCancels) { - effects.push(cancel(childThreadId)) + for (const childId of state.pendingCancels) { + effects.push( + childId.startsWith('run_') ? cancelRun(childId, `cancel_${childId}`) : cancel(childId), + ) } if (state.status === 'failed') {