From 76fdd509072d679d96e11ce2e8c4352554d7f0cd Mon Sep 17 00:00:00 2001 From: Daniel Bannert Date: Tue, 11 Aug 2026 10:13:08 +0200 Subject: [PATCH 01/10] feat: durable streams, run queue, evals, budgets Six capabilities an AI workload needs from the backend, plus two platform guarantees they rest on. Durable streams. A `.stream()` procedure declared `{ durable: true }` persists each chunk under a monotonic seq before it reaches a socket, and its producer outlives the socket that opened it. A run is keyed by (functionPath, args), so a reload replays the gap and continues, and a second client with the same arguments attaches to the same transcript instead of paying for a second generation. A run whose DO is evicted mid-flight replays its prefix and then fails STREAM_INTERRUPTED rather than silently re-running the handler, which would bill twice and duplicate the tail. Wire: `chunk.seq` out, `stream.sinceChunk` in. Agent run queue. `onConcurrentRun: "queue"` was documented and degraded to "reject". It is now a real FIFO queue (five deep): a parked run hibernates on a scoped waitForEvent, and the finishing run's completion writes its terminal status AND hands the thread to the next run in the same mutation, so there is no window in which the thread is ownerless and a third run can jump the line. The wake is a durable step; the waiter's own timeout covers a wake that can never land. Agent replies. An inbound trigger could start a run but never answer. `AgentReplyRef` rides the run params, `defineAgent({ onReply })` fires once on the final answer inside a durable step, and the email capture plus threaded send ship as `@lunora/agent/reply`. Slack/GitHub/Discord get the ref and the worker env; the framework knows where to reply, the app holds the credential. Evals. `recordEvaluation` takes an optional metrics handle and records the score as a gauge alongside the span attributes -- span attributes live in a bounded in-memory ring and vanish on hibernation, so the metric is what can back a trend. The new Studio Evals page joins both halves: durable trend, plus the individual verdicts with the trace that earned each one. Reactive RAG. `ragSyncTriggers` hangs re-indexing off the write: a table trigger schedules an internal action, which embeds once the mutation has committed. An edit that does not touch the indexed text schedules nothing; clearing it removes the chunks. Token budgets. A request limit is the wrong shape for a model call -- one call can cost a hundred tokens or a hundred thousand. `tokenBudget` makes the unit tokens (or cents), spent in arrears because that is when the cost is known. Deterministic mutations. A mutation body now runs with Date.now() frozen to the dispatch instant, Math.random() seeded, and fetch refused. Sound because a mutation holds the single-writer gate; queries interleave, so they keep the advisor lint and ctx.now instead. Components. `defineComponent` + `defineSchemaExtension` already worked and were undocumented, which for a contract is the same as not existing. Now a concepts page, including what a component explicitly does not get: no isolation from app data, no per-component config, no component-scoped RLS, no version negotiation. Not included: snapshot coherence for query subscriptions. A client-side coherence window was built, measured, and reverted -- it changes the delivery contract every adapter relies on, and only coalesces same-tick frames, which React already batches. The real fix needs a batch boundary on the wire; recorded as plans/333. BREAKING CHANGE: `fetch` throws inside a mutation (NETWORK_IN_MUTATION). Network I/O belongs in an action: a transaction that awaits a third party cannot be atomic. Move the call to an action, or schedule one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018hGhKLpXkFj3aWUYhZmWQv --- api-snapshots/agent.api.md | 61 +++- api-snapshots/ai.api.md | 31 ++ api-snapshots/client.api.md | 1 + api-snapshots/do.api.md | 3 + api-snapshots/errors.api.md | 8 + api-snapshots/lunora.api.md | 20 + api-snapshots/platform.api.md | 1 + api-snapshots/ratelimit.api.md | 15 + api-snapshots/react.api.md | 1 + api-snapshots/server.api.md | 26 +- api-snapshots/shard-engine.api.md | 106 ++++++ api-snapshots/studio.api.md | 2 +- api-snapshots/testing.api.md | 9 + .../src/content/docs/concepts/components.mdx | 119 ++++++ .../src/content/docs/concepts/realtime.mdx | 46 +++ apps/docs/src/content/docs/meta.json | 1 + apps/docs/src/content/docs/non-goals.mdx | 12 +- packages/agent/__tests__/agent-loop.test.ts | 31 +- .../agent/__tests__/agent-state-loop.test.ts | 1 + packages/agent/__tests__/component.test.ts | 1 + packages/agent/__tests__/loop-harness.ts | 24 ++ ...{reply.prototype.test.ts => reply.test.ts} | 74 +++- .../__tests__/run-queue.prototype.test.ts | 277 -------------- packages/agent/__tests__/run-queue.test.ts | 232 ++++++++++++ packages/agent/__tests__/skill.test.ts | 18 + packages/agent/package.json | 4 + packages/agent/src/agent-loop.ts | 145 +++++++- packages/agent/src/component.ts | 273 ++++++++++++-- packages/agent/src/index.ts | 3 + packages/agent/src/paths.ts | 1 + packages/agent/src/reply.prototype.ts | 92 ----- packages/agent/src/reply.ts | 79 ++++ packages/agent/src/run-queue.prototype.ts | 248 ------------- packages/agent/src/types.ts | 88 ++++- packages/ai/__tests__/rag/sync.test.ts | 78 ++++ packages/ai/src/rag/index.ts | 2 + packages/ai/src/rag/sync.ts | 145 ++++++++ packages/client/src/lunora-client.ts | 58 ++- packages/client/src/types.ts | 20 + .../codegen/__tests__/discover-agents.test.ts | 1 + .../simple/expected/_generated/shard.ts | 3 +- packages/codegen/src/emit.ts | 3 +- .../__tests__/shard-do.durable-stream.test.ts | 243 +++++++++++++ packages/do/src/shard-do.ts | 308 +++++++++++++++- packages/errors/src/catalog.ts | 2 + packages/platform/src/capabilities.ts | 16 + .../ratelimit/__tests__/token-budget.test.ts | 47 +++ packages/ratelimit/src/index.ts | 2 + packages/ratelimit/src/token-budget.ts | 91 +++++ packages/react/src/use-stream.ts | 14 +- packages/server/src/builder/index.ts | 6 + packages/server/src/builder/types.ts | 9 + packages/server/src/index.ts | 1 + packages/server/src/types.ts | 29 ++ .../__tests__/deterministic-scope.test.ts | 61 ++++ .../shard-engine/src/ctx-db-migrations.ts | 6 + .../shard-engine/src/deterministic-scope.ts | 118 ++++++ packages/shard-engine/src/durable-stream.ts | 206 +++++++++++ packages/shard-engine/src/index.ts | 13 + packages/shard-engine/src/types.ts | 12 + .../features/evals/eval-scores.test.ts | 89 +++++ packages/studio/src/app/nav-labels.ts | 2 + packages/studio/src/app/nav-types.ts | 1 + packages/studio/src/app/studio.tsx | 6 +- .../src/features/agents/agents-panel.tsx | 26 ++ .../studio/src/features/evals/eval-scores.ts | 131 +++++++ .../studio/src/features/evals/evals-panel.tsx | 145 ++++++++ packages/studio/src/locales/en.ts | 9 + packages/testing/src/agent-harness.ts | 24 ++ packages/testing/src/evaluation-telemetry.ts | 24 +- packages/testing/src/index.ts | 2 +- plans/240-agent-run-queue-design.md | 267 -------------- plans/242-agent-reply-design.md | 266 -------------- plans/245-eval-runner-design.md | 342 ------------------ plans/333-query-snapshot-coherence.md | 76 ++++ plans/README.md | 151 +++++++- protocol/README.md | 12 +- 77 files changed, 3546 insertions(+), 1574 deletions(-) create mode 100644 apps/docs/src/content/docs/concepts/components.mdx rename packages/agent/__tests__/{reply.prototype.test.ts => reply.test.ts} (60%) delete mode 100644 packages/agent/__tests__/run-queue.prototype.test.ts create mode 100644 packages/agent/__tests__/run-queue.test.ts delete mode 100644 packages/agent/src/reply.prototype.ts create mode 100644 packages/agent/src/reply.ts delete mode 100644 packages/agent/src/run-queue.prototype.ts create mode 100644 packages/ai/__tests__/rag/sync.test.ts create mode 100644 packages/ai/src/rag/sync.ts create mode 100644 packages/do/__tests__/shard-do.durable-stream.test.ts create mode 100644 packages/ratelimit/__tests__/token-budget.test.ts create mode 100644 packages/ratelimit/src/token-budget.ts create mode 100644 packages/shard-engine/__tests__/deterministic-scope.test.ts create mode 100644 packages/shard-engine/src/deterministic-scope.ts create mode 100644 packages/shard-engine/src/durable-stream.ts create mode 100644 packages/studio/__tests__/features/evals/eval-scores.test.ts create mode 100644 packages/studio/src/features/evals/eval-scores.ts create mode 100644 packages/studio/src/features/evals/evals-panel.tsx delete mode 100644 plans/240-agent-run-queue-design.md delete mode 100644 plans/242-agent-reply-design.md delete mode 100644 plans/245-eval-runner-design.md create mode 100644 plans/333-query-snapshot-coherence.md diff --git a/api-snapshots/agent.api.md b/api-snapshots/agent.api.md index fe0c33946d..a311ed3a6c 100644 --- a/api-snapshots/agent.api.md +++ b/api-snapshots/agent.api.md @@ -45,7 +45,15 @@ _Tagged `@experimental` — signature not tracked; churn here does not fail the ### `AgentEmailRun` (interface) -_Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ +```ts +interface AgentEmailRun { + input: string; + owner?: string; + replyRef?: AgentReplyRef; + threadKey: string; + title?: string; +} +``` ### `AgentFunctionPaths` (interface) @@ -119,6 +127,10 @@ _Tagged `@experimental` — signature not tracked; churn here does not fail the _Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ +### `AgentOnReply` (type) + +_Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ + ### `AgentOnStepFinish` (type) _Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ @@ -139,6 +151,14 @@ _Tagged `@experimental` — signature not tracked; churn here does not fail the _Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ +### `AgentReplyInfo` (interface) + +_Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ + +### `AgentReplyRef` (type) + +_Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ + ### `AgentRunFunction` (type) _Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ @@ -585,6 +605,45 @@ _Tagged `@experimental` — signature not tracked; churn here does not fail the _Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ +## `@lunora/agent/reply` + +### `EmailReplyRef` (type) + +```ts +type EmailReplyRef = Extract; +``` + +### `ReplyBody` (interface) + +```ts +interface ReplyBody { + subject: string; + text: string; +} +``` + +### `buildReferences` (const) + +```ts +const buildReferences: (replyRef: EmailReplyRef) => string; +``` + +### `captureEmailReplyRef` (const) + +```ts +const captureEmailReplyRef: (email: InboundEmail) => EmailReplyRef | undefined; +``` + +### `replyToEmail` (const) + +```ts +const replyToEmail: (mailer: Mailer, replyRef: EmailReplyRef, body: ReplyBody) => Promise<{ + id: string; +}>; +``` + ## `@lunora/agent/sandbox` ### `BrowserToolInput` (type) diff --git a/api-snapshots/ai.api.md b/api-snapshots/ai.api.md index b1ddaf3de1..a69344bbc2 100644 --- a/api-snapshots/ai.api.md +++ b/api-snapshots/ai.api.md @@ -167,6 +167,27 @@ _Tagged `@experimental` — signature not tracked; churn here does not fail the _Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ +### `RagSyncArgs` (interface) + +```ts +interface RagSyncArgs extends Record { + deleted?: boolean; + id: string; + text?: string; +} +``` + +### `RagSyncOptions` (interface) + +```ts +interface RagSyncOptions = Record> { + action: unknown; + delayMs?: number; + id?: (document: Document) => string; + text: (document: Document) => string | undefined; +} +``` + ### `RagTextStore` (interface) _Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ @@ -244,3 +265,13 @@ _Tagged `@experimental` — signature not tracked; churn here does not fail the ### `hybridRank` (const) _Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ + +### `ragSyncTriggers` (const) + +```ts +const ragSyncTriggers: = Record>(options: RagSyncOptions) => { + afterDelete: RagSyncHandler; + afterInsert: RagSyncHandler; + afterUpdate: RagSyncHandler; +}; +``` diff --git a/api-snapshots/client.api.md b/api-snapshots/client.api.md index df0c956c13..f4c2d536f3 100644 --- a/api-snapshots/client.api.md +++ b/api-snapshots/client.api.md @@ -744,6 +744,7 @@ class LunoraClient { shardKey?: string; }): Unsubscribe; stream>(function_: F, args: ArgsOf, options?: { + durable?: boolean; maxBuffer?: number; shardKey?: string; }): StreamIterable>; diff --git a/api-snapshots/do.api.md b/api-snapshots/do.api.md index f3fc5702f2..3308950607 100644 --- a/api-snapshots/do.api.md +++ b/api-snapshots/do.api.md @@ -376,6 +376,9 @@ abstract class ShardDO { protected recordExternalSourceError(table: string, error: unknown, trace?: TraceRefLike): void; protected recordExternalSourceWarning(table: string, message: string, trace?: TraceRefLike): void; protected executeStream(_functionPath: string, _args: Record): null | { + durable?: { + ttlMs?: number; + }; iterator: (signal: AbortSignal) => AsyncIterable; }; protected runCachedQuery(functionPath: string, args: Record, run: () => Promise): Promise; diff --git a/api-snapshots/errors.api.md b/api-snapshots/errors.api.md index 7f2e7b25eb..9a2c31622e 100644 --- a/api-snapshots/errors.api.md +++ b/api-snapshots/errors.api.md @@ -501,6 +501,10 @@ const ERROR_CATALOG: { readonly status: 500; readonly title: "Nested transaction"; }; + readonly NETWORK_IN_MUTATION: { + readonly status: 500; + readonly title: "Network call inside a mutation"; + }; readonly OUT_OF_ORDER: { readonly status: 409; readonly title: "Out-of-order mutation"; @@ -518,6 +522,10 @@ const ERROR_CATALOG: { readonly status: 500; readonly title: "SQL storage unavailable"; }; + readonly STREAM_INTERRUPTED: { + readonly status: 503; + readonly title: "Durable stream interrupted"; + }; readonly TOKEN_EXPIRED: { readonly status: 401; readonly title: "Authentication token expired"; diff --git a/api-snapshots/lunora.api.md b/api-snapshots/lunora.api.md index 3b21ce6fdb..8a64bd1571 100644 --- a/api-snapshots/lunora.api.md +++ b/api-snapshots/lunora.api.md @@ -129,6 +129,10 @@ Re-exported from `@lunora/server` — signature tracked at its source. Re-exported from `@lunora/server` — signature tracked at its source. +### `DurableStreamOptions` (interface) + +Re-exported from `@lunora/server` — signature tracked at its source. + ### `EmptyArgs` (type) Re-exported from `@lunora/server` — signature tracked at its source. @@ -3491,6 +3495,10 @@ Re-exported from `@lunora/ratelimit` — signature tracked at its source. Re-exported from `@lunora/ratelimit` — signature tracked at its source. +### `TokenBudget` (interface) + +Re-exported from `@lunora/ratelimit` — signature tracked at its source. + ### `VERSION` (const) Re-exported from `@lunora/ratelimit` — signature tracked at its source. @@ -3531,6 +3539,10 @@ Re-exported from `@lunora/ratelimit` — signature tracked at its source. Re-exported from `@lunora/ratelimit` — signature tracked at its source. +### `tokenBudget` (const) + +Re-exported from `@lunora/ratelimit` — signature tracked at its source. + ## `lunorash/runtime` ### `AdminTableResolver` (type) @@ -4619,6 +4631,10 @@ Re-exported from `@lunora/server` — signature tracked at its source. Re-exported from `@lunora/server` — signature tracked at its source. +### `DurableStreamOptions` (interface) + +Re-exported from `@lunora/server` — signature tracked at its source. + ### `EmptyArgs` (type) Re-exported from `@lunora/server` — signature tracked at its source. @@ -6669,6 +6685,10 @@ Re-exported from `@lunora/server` — signature tracked at its source. Re-exported from `@lunora/server` — signature tracked at its source. +### `DurableStreamOptions` (interface) + +Re-exported from `@lunora/server` — signature tracked at its source. + ### `ExposeConfig` (interface) Re-exported from `@lunora/server` — signature tracked at its source. diff --git a/api-snapshots/platform.api.md b/api-snapshots/platform.api.md index 1b3c2eb714..ac073849d7 100644 --- a/api-snapshots/platform.api.md +++ b/api-snapshots/platform.api.md @@ -254,6 +254,7 @@ interface PlatformCapabilities { browser?: Capability; containers?: Capability; crossShardFanout?: Capability; + durableStreams?: Capability; globalTables?: Capability; hyperdrive?: Capability; keyValueStore?: Capability; diff --git a/api-snapshots/ratelimit.api.md b/api-snapshots/ratelimit.api.md index c48abd86f0..dfd926f661 100644 --- a/api-snapshots/ratelimit.api.md +++ b/api-snapshots/ratelimit.api.md @@ -246,6 +246,15 @@ interface SqlStoreOptions { } ``` +### `TokenBudget` (interface) + +```ts +interface TokenBudget { + check: (key: string) => Promise; + record: (key: string, tokens: number) => Promise; +} +``` + ### `VERSION` (const) ```ts @@ -312,3 +321,9 @@ const rateLimit: (limiter: LimiterResolver, name: string, opti ```ts const ratelimitPlugin: (limiter: LimiterResolver) => Plugin, Context, Context & RatelimitApiContext>; ``` + +### `tokenBudget` (const) + +```ts +const tokenBudget: (limiter: RateLimiter, name: Names) => TokenBudget; +``` diff --git a/api-snapshots/react.api.md b/api-snapshots/react.api.md index 0aae362d0f..7eacc934e7 100644 --- a/api-snapshots/react.api.md +++ b/api-snapshots/react.api.md @@ -735,6 +735,7 @@ interface UseRateLimitResult { ```ts interface UseStreamOptions { + durable?: boolean; maxBuffer?: number; shardKey?: string; } diff --git a/api-snapshots/server.api.md b/api-snapshots/server.api.md index 5e4d8b8d52..b61555daa7 100644 --- a/api-snapshots/server.api.md +++ b/api-snapshots/server.api.md @@ -325,6 +325,14 @@ interface DefineStorageRuleInput { type DurableObjectJurisdiction = "eu" | "fedramp" | "us"; ``` +### `DurableStreamOptions` (interface) + +```ts +interface DurableStreamOptions { + readonly ttlMs?: number; +} +``` + ### `EmptyArgs` (type) ```ts @@ -814,7 +822,9 @@ interface InternalQueryBuilder; ctx: Context; signal: AbortSignal; - }) => AsyncGenerator | AsyncIterable) => RegisteredStream; + }) => AsyncGenerator | AsyncIterable, options?: { + durable?: boolean | DurableStreamOptions; + }) => RegisteredStream; use: (middleware: Middleware) => InternalQueryBuilder; } ``` @@ -1421,7 +1431,9 @@ interface QueryBuilder args: InferArgs; ctx: Context; signal: AbortSignal; - }) => AsyncGenerator | AsyncIterable) => RegisteredStream; + }) => AsyncGenerator | AsyncIterable, options?: { + durable?: boolean | DurableStreamOptions; + }) => RegisteredStream; use: (middleware: Middleware) => QueryBuilder; x402: (config: X402ProcedureConfig) => QueryBuilder; } @@ -1573,6 +1585,7 @@ interface RegisteredShape { readonly args: A; + readonly durable?: DurableStreamOptions; readonly handler: (context: unknown, args: InferArgs, signal: AbortSignal) => AsyncIterable; readonly kind: "stream"; readonly visibility?: FunctionVisibility; @@ -4309,6 +4322,14 @@ interface DatabaseWriter extends DatabaseReader { type DurableObjectJurisdiction = "eu" | "fedramp" | "us"; ``` +### `DurableStreamOptions` (interface) + +```ts +interface DurableStreamOptions { + readonly ttlMs?: number; +} +``` + ### `ExposeConfig` (interface) ```ts @@ -4666,6 +4687,7 @@ type RegisteredQuery = RegisteredFunction { readonly args: A; + readonly durable?: DurableStreamOptions; readonly handler: (context: unknown, args: InferArgs, signal: AbortSignal) => AsyncIterable; readonly kind: "stream"; readonly visibility?: FunctionVisibility; diff --git a/api-snapshots/shard-engine.api.md b/api-snapshots/shard-engine.api.md index b6816ea298..0a473e7641 100644 --- a/api-snapshots/shard-engine.api.md +++ b/api-snapshots/shard-engine.api.md @@ -661,6 +661,33 @@ interface DeployInfo { } ``` +### `DurableStreamChunk` (interface) + +```ts +interface DurableStreamChunk { + dataJson: string; + seq: number; +} +``` + +### `DurableStreamRun` (interface) + +```ts +interface DurableStreamRun { + error?: string; + errorCode?: string; + lastSeq: number; + startedAt: number; + status: DurableStreamStatus; +} +``` + +### `DurableStreamStatus` (type) + +```ts +type DurableStreamStatus = "complete" | "error" | "running"; +``` + ### `ExportRow` (interface) ```ts @@ -2079,6 +2106,18 @@ const SCHEMA_HISTORY_MAX_VERSIONS = 50; const SEARCH_STATE_TABLE = "__lunora_search_state"; ``` +### `STREAM_CHUNKS_TABLE` (const) + +```ts +const STREAM_CHUNKS_TABLE = "__stream_chunks"; +``` + +### `STREAM_RUNS_TABLE` (const) + +```ts +const STREAM_RUNS_TABLE = "__stream_runs"; +``` + ### `SchedulableWorkflowReferenceLike` (interface) ```ts @@ -2497,6 +2536,7 @@ interface SubscriptionEnvelope { name: string; }; sinceCheckpoint?: number; + sinceChunk?: number; sinceEpoch?: string; topic?: string; type: "ack" | "connect" | "shape_subscribe" | "shape_unsubscribe" | "stream" | "subscribe" | "unsubscribe" | "whisper" | "whisper_subscribe" | "whisper_unsubscribe"; @@ -2952,6 +2992,12 @@ const appendAuditEntry: (sql: SqlExec, entry: AppendAuditEntry) => void; const appendCdcChange: (sql: SqlExec, ts: number, table: string, id: string, op: CdcChange["op"], doc: Record | undefined) => void; ``` +### `appendStreamChunk` (const) + +```ts +const appendStreamChunk: (sql: SqlExec, runKey: string, seq: number, dataJson: string) => void; +``` + ### `applyCdcChanges` (const) ```ts @@ -3088,6 +3134,12 @@ const buildSettings: (rawEnv: unknown) => SettingsResult; const bumpCdcEpoch: (sql: SqlExec) => string; ``` +### `claimStreamRun` (const) + +```ts +const claimStreamRun: (sql: SqlExec, runKey: string, startedAt: number) => boolean; +``` + ### `clampPromotionThresholds` (const) ```ts @@ -3323,6 +3375,15 @@ const fanOutScalarCounts: (counter: (tableName: string, where?: WhereInput) => P const findStorageReferences: (sql: SqlExec, storageColumns: Record, keys: string[]) => StorageReferenceResult; ``` +### `finishStreamRun` (const) + +```ts +const finishStreamRun: (sql: SqlExec, runKey: string, status: "complete" | "error", failure?: { + code: string; + message: string; +}) => void; +``` + ### `foldAggregateTally` (const) ```ts @@ -3528,6 +3589,12 @@ const migrateCdcMeta: (sql: SqlExec) => void; const migrateClientWatermark: (sql: SqlExec) => void; ``` +### `migrateDurableStreams` (const) + +```ts +const migrateDurableStreams: (sql: SqlExec) => void; +``` + ### `migrateGlobalShapeSnapshot` (const) ```ts @@ -3552,6 +3619,12 @@ const migrateSearchState: (sql: SqlExec) => void; const minCdcSeq: (sql: SqlExec) => number | undefined; ``` +### `mulberry32` (const) + +```ts +const mulberry32: (seed: number) => (() => number); +``` + ### `nextPromotionState` (const) ```ts @@ -3799,6 +3872,18 @@ const readSchemaVersion: (sql: SqlExec, hash: string) => SchemaVersionRow | unde const readSearchBackfillState: (sql: SqlExec, companion: string) => SearchBackfillState; ``` +### `readStreamChunks` (const) + +```ts +const readStreamChunks: (sql: SqlExec, runKey: string, sinceSeq: number) => DurableStreamChunk[]; +``` + +### `readStreamRun` (const) + +```ts +const readStreamRun: (sql: SqlExec, runKey: string) => DurableStreamRun | undefined; +``` + ### `readTablePage` (const) ```ts @@ -3962,6 +4047,12 @@ const runSql: >(sql: SqlExec, query: string, ...pa const runTriggers: (options: RunTriggersOptions) => Promise; ``` +### `seedFrom` (const) + +```ts +const seedFrom: (text: string) => number; +``` + ### `selectExpiredIds` (const) ```ts @@ -4130,6 +4221,12 @@ const trimCdcChanges: (sql: SqlExec, throughSeq: number) => void; const trimIdempotent: (sql: SqlExec, olderThanTs: number) => void; ``` +### `trimStreamRuns` (const) + +```ts +const trimStreamRuns: (sql: SqlExec, olderThanTs: number) => void; +``` + ### `tryRowToDocument` (const) ```ts @@ -4154,6 +4251,15 @@ const unionAll: (branches: ReadonlyArray) => SQL; const validateImportRow: (schema: SchemaLike, table: string, record: Record) => string | undefined; ``` +### `withDeterministicScope` (const) + +```ts +const withDeterministicScope: (options: { + now: number; + seed: string; +}, body: () => Promise | T) => Promise; +``` + ### `writeGlobalShapeSnapshot` (const) ```ts diff --git a/api-snapshots/studio.api.md b/api-snapshots/studio.api.md index b025f89b55..47170ae23d 100644 --- a/api-snapshots/studio.api.md +++ b/api-snapshots/studio.api.md @@ -1090,7 +1090,7 @@ interface StudioProps { ### `StudioTab` (type) ```ts -type StudioTab = "agents" | "analytics" | "api" | "audit" | "authAudit" | "authConfig" | "authSessions" | "containers" | "dashboards" | "data" | "deploymentHealth" | "drains" | "export" | "fanout" | "files" | "flags" | "functions" | "health" | "home" | "insights" | "issues" | "kv" | "logs" | "mail" | "metrics" | "migrations" | "notifications" | "organizations" | "payments" | "permissions" | "pitr" | "queues" | "realtime" | "rls" | "schedule" | "schema" | "advisorHealth" | "security" | "settings" | "sql" | "storageRules" | "traces" | "users" | "vectors" | "workflows"; +type StudioTab = "agents" | "analytics" | "api" | "audit" | "authAudit" | "authConfig" | "authSessions" | "containers" | "dashboards" | "data" | "deploymentHealth" | "drains" | "export" | "fanout" | "files" | "flags" | "functions" | "health" | "home" | "insights" | "issues" | "kv" | "logs" | "mail" | "metrics" | "migrations" | "notifications" | "organizations" | "payments" | "permissions" | "pitr" | "queues" | "realtime" | "rls" | "schedule" | "schema" | "advisorHealth" | "evals" | "security" | "settings" | "sql" | "storageRules" | "traces" | "users" | "vectors" | "workflows"; ``` ### `TFunction` (type) diff --git a/api-snapshots/testing.api.md b/api-snapshots/testing.api.md index 636121cc1b..7c137b7ea9 100644 --- a/api-snapshots/testing.api.md +++ b/api-snapshots/testing.api.md @@ -76,6 +76,14 @@ interface EvalResult { type EvaluationAttributeValue = number | string; ``` +### `EvaluationMetrics` (interface) + +```ts +interface EvaluationMetrics { + gauge: (name: string, value: number, attributes?: Record) => void; +} +``` + ### `EvaluationSpanHandle` (interface) ```ts @@ -164,6 +172,7 @@ interface LunoraTestOptions { ```ts interface RecordEvaluationInput { label?: string; + metrics?: EvaluationMetrics; name: string; score: number; span?: EvaluationSpanHandle; diff --git a/apps/docs/src/content/docs/concepts/components.mdx b/apps/docs/src/content/docs/concepts/components.mdx new file mode 100644 index 0000000000..5daa7b30e1 --- /dev/null +++ b/apps/docs/src/content/docs/concepts/components.mdx @@ -0,0 +1,119 @@ +--- +title: Components +description: Package backend features — tables, functions, and context — as an installable unit that cannot collide with the app that installs it. +--- + +A **component** is a backend feature shipped as an npm package: its own tables, +its own functions, and (optionally) something it adds to `ctx`. The app installs +it with one `.extend(...)` call and one re-export, and the component's tables can +never collide with the app's. + +Everything below is what `@lunora/agent`, `@lunora/ratelimit`, and the built-in +presence component are made of — there is no privileged first-party path. + +## Defining one + +```ts title="my-component/src/index.ts" +import { defineComponent, defineSchemaExtension, defineTable, initLunora, v } from "@lunora/server"; + +const { mutation, query } = initLunora.dataModel().create(); + +const votes = defineTable({ subject: v.string(), value: v.number(), voter: v.string() }) + .index("bySubject", ["subject"]) + .index("byVoter", ["subject", "voter"], { unique: true }); + +export const voting = defineComponent("voting", { + // Bare table names — prefixed at merge, so this becomes `voting_votes`. + extension: defineSchemaExtension("voting", { tables: { votes } }), + + functions: { + castVote: mutation.input({ subject: v.string(), value: v.number() }).mutation(async ({ args, ctx }) => { + await ctx.db.insert("voting_votes", { ...args, voter: ctx.auth.userId ?? "anon" }); + }), + tally: query.input({ subject: v.string() }).query(async ({ args, ctx }) => { + const rows = await ctx.db + .query("voting_votes") + .withIndex("bySubject", (q) => q.eq("subject", args.subject)) + .collect(); + + return rows.reduce((total, row) => total + (row.value as number), 0); + }), + }, + + // Optional: everything after `.use(voting.middleware)` sees `ctx.voting`. + middleware: ({ ctx, next }) => next({ ctx: { ...ctx, voting: makeVotingApi(ctx) } }), +}); +``` + +Three parts, all optional except the name: + +| Part | What it contributes | +| ------------ | ----------------------------------------------------------------------------- | +| `extension` | Tables, merged under the component's key | +| `functions` | Queries / mutations / actions the app re-exports | +| `middleware` | A `ctx` addition for procedures that opt in with `.use(component.middleware)` | + +## Installing one + +```ts title="lunora/schema.ts" +import { voting } from "my-component"; + +export const schema = defineSchema({ posts: defineTable({ title: v.string() }) }).extend(voting.extension); +``` + +```ts title="lunora/voting.ts" +// One line: re-exporting is what puts the component's functions in your API. +export const { castVote, tally } = voting.functions; +``` + +That is the whole installation. Codegen chases the re-export back to the +component's registration call, so `api.voting.tally` is typed end to end exactly +like a function you wrote yourself, and `voting_votes` appears in `Doc`/`api` +types. A component whose extension lives in `node_modules` is introspected at +codegen time — you do not vendor its schema. + +## What the namespace guarantees + +Every extension table is prefixed with the component's key (`votes` → +`voting_votes`), and every intra-component reference — relation targets, +aggregate and rank index `on`, standalone vector index `table` — is rewritten to +match. So: + +- **An app can never collide with a component.** Your `votes` table and the + component's are different tables. +- **A component cannot silently retarget your data.** Its references resolve + inside its own namespace; only references to base tables are left alone. +- **Two components collide only if they pick the same key**, which is a loud + error at merge, not a silent overwrite. + +## What a component does NOT get + +Stated plainly, because a component contract is only useful if its limits are +known: + +- **No isolation from the app's data.** A component's functions run with an + ordinary `ctx.db` and can read and write app tables. The namespace prevents + accidents, not a malicious package — installing a component is as much trust as + installing any npm dependency. +- **No per-component configuration surface.** There is no `app.use(component, { +options })` layer; a component that needs configuration takes it as a factory + argument (`voting({ maxVotes: 5 })`) and closes over it. +- **No component-scoped RLS.** Policies are declared on the merged schema by the + app. A component whose tables must stay internal should say so in its docs; + `.public()` on an extension table is what makes it readable through the studio + and the generated data model. +- **No version negotiation.** A component is an npm dependency and follows its + own SemVer; there is no runtime capability handshake between it and the app. + +## Migrations + +A component's tables migrate with the app's — they are merged into one schema, so +`lunora migrate generate` sees them and one migration covers both. Upgrading a +component that added a column is an ordinary schema change in your project. + +## See also + +- [Schema](/docs/concepts/schema) — `defineSchema` and `.extend(...)` +- [Middleware](/docs/concepts/middleware) — the `ctx` extension half +- [Registry items](/docs/concepts/registry-items) — the copy-in alternative, for + code you want to own and edit rather than depend on diff --git a/apps/docs/src/content/docs/concepts/realtime.mdx b/apps/docs/src/content/docs/concepts/realtime.mdx index a245c541f4..99454e025f 100644 --- a/apps/docs/src/content/docs/concepts/realtime.mdx +++ b/apps/docs/src/content/docs/concepts/realtime.mdx @@ -50,6 +50,52 @@ The client (`@lunora/client`) uses exponential backoff with jitter and **resumes by bookmark**: it sends the last delta sequence it acknowledged so the server can replay anything that was missed during the disconnect. +## Durable streams + +A `.stream()` procedure is ephemeral by default: the producer belongs to the +socket that opened it, so closing the tab ends the run and a reconnect starts +over. That is right for a progress ticker and wrong for anything expensive — +a model's answer must not disappear because someone refreshed. + +Declare the stream durable and the run stops belonging to the socket: + +```ts +export const answer = query.input({ threadId: v.id("threads"), prompt: v.string() }).stream( + async function* ({ ctx, args }) { + for await (const token of callTheModel(args.prompt)) { + yield token; + } + }, + { durable: true }, +); +``` + +Three things change: + +- **Every chunk is persisted before it is sent**, under a monotonic `seq`. A + reconnecting client replays the frames it missed out of the shard's SQLite and + then continues live — the consumer's `for await` never sees the interruption. +- **The producer outlives the socket.** Close the tab mid-run and the run + finishes anyway; the transcript is waiting when you come back. +- **A run is shared.** Its identity is `(functionPath, args)`, so a second tab + asking the same question attaches to the same run instead of paying for a + second generation. Give a stream that must not be shared a caller-unique + argument — the argument set _is_ the run key. + +On the client, opt in per call so the reconnect resumes rather than fails: + +```tsx +const { chunks, status } = useStream(api.chat.answer, { threadId, prompt }, { durable: true }); +``` + +Transcripts are trimmed after 24 hours by default (`{ durable: { ttlMs } }` to +change it). One case is deliberately loud rather than silent: if the Durable +Object is evicted mid-run, the persisted prefix is replayed and the stream then +fails with `STREAM_INTERRUPTED` instead of quietly re-running the handler — +re-running a model call would bill twice and duplicate the tail. When you need a +producer that survives that too, the run belongs in a +[workflow](/docs/packages/workflow). + ## Why no MQTT / Pub/Sub Real-time fan-out in Lunora is entirely Durable-Object-based: hibernated diff --git a/apps/docs/src/content/docs/meta.json b/apps/docs/src/content/docs/meta.json index 046c7df8f5..2fce68412e 100644 --- a/apps/docs/src/content/docs/meta.json +++ b/apps/docs/src/content/docs/meta.json @@ -24,6 +24,7 @@ "concepts/server-clients", "concepts/caching", "concepts/middleware", + "concepts/components", "concepts/scheduling", "concepts/notifications", "concepts/error-handling", diff --git a/apps/docs/src/content/docs/non-goals.mdx b/apps/docs/src/content/docs/non-goals.mdx index a555794f3e..ae88a40b89 100644 --- a/apps/docs/src/content/docs/non-goals.mdx +++ b/apps/docs/src/content/docs/non-goals.mdx @@ -161,8 +161,16 @@ A few smaller boundaries, most of them covered in depth on answers: await the first, combine both into one mutation, or use [`@lunora/db`](/docs/packages/db) mutators, whose outbox is strictly ordered. See [Write ordering](/docs/concepts/offline-first#write-ordering). -- **No runtime enforcement of query/mutation determinism yet** — it's an advisor - lint today, not a hard runtime guard. +- **Determinism is enforced in mutations, linted in queries.** A mutation body + runs with `Date.now()` frozen to the dispatch instant, `Math.random()` seeded, + and `fetch` refused (`NETWORK_IN_MUTATION`) — sound because a mutation holds the + DO's single-writer gate, so nothing else can observe the swapped globals. + Queries interleave (a deferred subscription re-run lands between two awaits of + another query), so freezing the clock for one would hand its instant to + another; they keep the `nondeterministic_query_mutation` advisor lint plus the + `ctx.now` / `ctx.fetch` surfaces instead. The values are stable _within_ one + execution, not replayed across a retry — an idempotency key, not a replayed + clock, is what makes a retry safe. ## See also diff --git a/packages/agent/__tests__/agent-loop.test.ts b/packages/agent/__tests__/agent-loop.test.ts index 4d33cf4004..f5e83a8835 100644 --- a/packages/agent/__tests__/agent-loop.test.ts +++ b/packages/agent/__tests__/agent-loop.test.ts @@ -254,6 +254,32 @@ const memoryRuntime = (options?: { return undefined; }; + /** + * The completion mutation: terminal patch + queue handoff. This double keeps + * no queue (see `run-queue.test.ts` for those semantics), so it ends the run + * exactly as the real one does with an empty queue — including the ownership + * guard that makes a replayed completion a no-op. + */ + const completeRun = (args?: Record): unknown => { + const thread = threads.get(args?.["key"] as string); + + if (!args || !thread || thread.instanceId !== args["instanceId"]) { + return {}; + } + + thread.status = args["status"] as string; + + if (args["error"] !== undefined) { + thread.error = args["error"] as string; + } + + if (args["usage"] !== undefined) { + thread.usage = args["usage"] as StoredThread["usage"]; + } + + return {}; + }; + const memoryHandler = (args?: Record): unknown => { const result = options?.memory?.result; @@ -262,6 +288,7 @@ const memoryRuntime = (options?: { const handlers = new Map) => unknown>([ [DEFAULT_AGENT_FUNCTION_PATHS.appendMessage, appendMessage], + [DEFAULT_AGENT_FUNCTION_PATHS.completeRun, completeRun], [DEFAULT_AGENT_FUNCTION_PATHS.ensureThread, ensureThread], [DEFAULT_AGENT_FUNCTION_PATHS.listMessages, listMessages], [DEFAULT_AGENT_FUNCTION_PATHS.patchThread, patchThread], @@ -764,10 +791,10 @@ describe(runAgentLoop, () => { const journal = new DurableStepJournal(); const params = { input: "hi", owner: "user-a", threadKey: "thread-1" }; - // First attempt crashes AFTER the final turn + extraction, on the terminal patch dispatch. + // First attempt crashes AFTER the final turn + extraction, on the run-completion dispatch. let failPatch = true; const crashingRun: AgentRunFunction = async (reference, args) => { - if (failPatch && reference["__lunoraRef"] === DEFAULT_AGENT_FUNCTION_PATHS.patchThread && args?.["status"] === "idle") { + if (failPatch && reference["__lunoraRef"] === DEFAULT_AGENT_FUNCTION_PATHS.completeRun && args?.["status"] === "idle") { failPatch = false; throw new Error("crash after extract"); } diff --git a/packages/agent/__tests__/agent-state-loop.test.ts b/packages/agent/__tests__/agent-state-loop.test.ts index deac056122..e1940e82e6 100644 --- a/packages/agent/__tests__/agent-state-loop.test.ts +++ b/packages/agent/__tests__/agent-state-loop.test.ts @@ -145,6 +145,7 @@ const ownedRuntime = (userId: string): { rows: Map; run: Agen const { functions } = agentComponent(); const handlers = new Map unknown }>([ [DEFAULT_AGENT_FUNCTION_PATHS.appendMessage, functions.agentAppendMessage], + [DEFAULT_AGENT_FUNCTION_PATHS.completeRun, functions.agentCompleteRun], [DEFAULT_AGENT_FUNCTION_PATHS.ensureThread, functions.agentEnsureThread], [DEFAULT_AGENT_FUNCTION_PATHS.listMessages, functions.agentMessages], [DEFAULT_AGENT_FUNCTION_PATHS.patchThread, functions.agentPatchThread], diff --git a/packages/agent/__tests__/component.test.ts b/packages/agent/__tests__/component.test.ts index d64c90134e..75819ddd11 100644 --- a/packages/agent/__tests__/component.test.ts +++ b/packages/agent/__tests__/component.test.ts @@ -176,6 +176,7 @@ describe(agentComponent, () => { "entities", "episodes", "messages", + "run_queue", "threads", ]); }); diff --git a/packages/agent/__tests__/loop-harness.ts b/packages/agent/__tests__/loop-harness.ts index e6d767d74e..99c8d2811c 100644 --- a/packages/agent/__tests__/loop-harness.ts +++ b/packages/agent/__tests__/loop-harness.ts @@ -148,8 +148,32 @@ export const memoryRuntime = (): { return undefined; }; + /** + * The completion mutation: terminal patch plus the queue handoff. The double + * keeps no queue (the queue's own semantics are covered in + * `run-queue.test.ts`), so it only has to end the run the way the real one + * does for an empty queue — including the ownership guard, which is what + * makes a replayed completion a no-op. + */ + const completeRun = (args?: Record): unknown => { + const thread = threads.get(args?.["key"] as string); + + if (!args || !thread || thread.instanceId !== args["instanceId"]) { + return {}; + } + + thread.status = args["status"] as string; + + if (args["error"] !== undefined) { + thread.error = args["error"] as string; + } + + return {}; + }; + const handlers = new Map) => unknown>([ [DEFAULT_AGENT_FUNCTION_PATHS.appendMessage, appendMessage], + [DEFAULT_AGENT_FUNCTION_PATHS.completeRun, completeRun], [DEFAULT_AGENT_FUNCTION_PATHS.ensureThread, ensureThread], [DEFAULT_AGENT_FUNCTION_PATHS.listMessages, listMessages], [DEFAULT_AGENT_FUNCTION_PATHS.patchThread, patchThread], diff --git a/packages/agent/__tests__/reply.prototype.test.ts b/packages/agent/__tests__/reply.test.ts similarity index 60% rename from packages/agent/__tests__/reply.prototype.test.ts rename to packages/agent/__tests__/reply.test.ts index 5e8ca9e91d..b1008a7b95 100644 --- a/packages/agent/__tests__/reply.prototype.test.ts +++ b/packages/agent/__tests__/reply.test.ts @@ -3,9 +3,12 @@ import { createMailer } from "@lunora/mail"; import type { InboundEmail } from "@lunora/mail/inbound"; import { describe, expect, it } from "vitest"; -import { buildReferences, captureEmailReplyRef, replyToEmail } from "../src/reply.prototype"; +import { runAgentLoop } from "../src/agent-loop"; +import { defineAgent } from "../src/define-agent"; +import { buildReferences, captureEmailReplyRef, replyToEmail } from "../src/reply"; +import { DurableStepJournal, finalTurn, loopDefaults, scriptedGenerate } from "./loop-harness"; -/** Minimal `InboundEmail` builder — fills only the fields the prototype reads. */ +/** Minimal `InboundEmail` builder — fills only the fields the helpers read. */ const fakeInboundEmail = (overrides: Partial = {}): InboundEmail => { return { attachments: [], @@ -33,7 +36,7 @@ const fakeMailTransport = (): { sent: SendPayload[]; transport: MailTransport } }; }; -describe("reply prototype (plan 242 spike)", () => { +describe("email reply", () => { it("captures a reply ref from an inbound email and threads the reply's In-Reply-To/References", async () => { const { sent, transport } = fakeMailTransport(); const mailer = createMailer({ from: "support@lunora.sh", transport }); @@ -104,4 +107,69 @@ describe("reply prototype (plan 242 spike)", () => { // proof this went through the real @lunora/mail send path, not a stub. expect(sent[0]?.from).toBe("support@lunora.sh"); }); + + it("calls onReply once with the captured ref when the run was triggered", async () => { + expect.assertions(3); + + const calls: { channel: string; text?: string; threadKey: string }[] = []; + const agent = defineAgent({ + model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast", + onReply: ({ replyRef, result, threadKey }) => { + calls.push({ channel: replyRef.channel, text: result.text, threadKey }); + }, + }); + const journal = new DurableStepJournal(); + const replyRef = captureEmailReplyRef(fakeInboundEmail({ from: "customer@example.com", messageId: "" })); + + await runAgentLoop( + loopDefaults(agent, { + generate: scriptedGenerate([finalTurn("the answer")]), + params: { input: "hi", replyRef, threadKey: "thread-1" }, + step: journal, + }), + ); + + expect(calls).toStrictEqual([{ channel: "email", text: "the answer", threadKey: "thread-1" }]); + // Delivery is a named durable step, so a replay serves it from the memo + // instead of sending the answer a second time. + expect(journal.invoked.filter((name) => name === "agent:reply")).toHaveLength(1); + expect(journal.invoked).toContain("agent:reply"); + }); + + it("never calls onReply for an ordinary in-app run (no replyRef)", async () => { + expect.assertions(1); + + let called = 0; + const agent = defineAgent({ + model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast", + onReply: () => { + called += 1; + }, + }); + + await runAgentLoop(loopDefaults(agent, { generate: scriptedGenerate([finalTurn("the answer")]) })); + + expect(called).toBe(0); + }); + + it("a failing reply does not fail the run — the answer is already on the thread", async () => { + expect.assertions(1); + + const agent = defineAgent({ + model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast", + onReply: () => { + throw new Error("slack is down"); + }, + }); + const replyRef = captureEmailReplyRef(fakeInboundEmail({ from: "customer@example.com", messageId: "" })); + + await expect( + runAgentLoop( + loopDefaults(agent, { + generate: scriptedGenerate([finalTurn("the answer")]), + params: { input: "hi", replyRef, threadKey: "thread-1" }, + }), + ), + ).resolves.toMatchObject({ text: "the answer" }); + }); }); diff --git a/packages/agent/__tests__/run-queue.prototype.test.ts b/packages/agent/__tests__/run-queue.prototype.test.ts deleted file mode 100644 index a6feeb5c71..0000000000 --- a/packages/agent/__tests__/run-queue.prototype.test.ts +++ /dev/null @@ -1,277 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import type { PrototypeDatabase, PrototypeIndexBuilder, PrototypeIndexQuery, PrototypeRow } from "../src/run-queue.prototype"; -import { completeRunAndDequeue, ensureThreadOrQueue, MAX_QUEUE_DEPTH, QueueConflictError } from "../src/run-queue.prototype"; - -const QUEUE_FULL_PATTERN = /run queue is full/u; -const IN_FLIGHT_PATTERN = /already has a run in flight/u; - -/** Same fields the prototype's two indexes order by. */ -const ORDER_FIELDS: Record = { byThread: "position" }; - -const collectConditions = (build: (q: PrototypeIndexBuilder) => unknown): [string, unknown][] => { - const conditions: [string, unknown][] = []; - const builder: PrototypeIndexBuilder = { - eq: (field, value) => { - conditions.push([field, value]); - - return builder; - }, - }; - - build(builder); - - return conditions; -}; - -const matchesConditions = (row: PrototypeRow, conditions: [string, unknown][]): boolean => conditions.every(([field, value]) => row[field] === value); - -const sortByField = (rows: PrototypeRow[], field: string | undefined, indexName: string, direction: "asc" | "desc"): PrototypeRow[] => { - if (field === undefined) { - throw new Error(`test double: .order() is only modeled for a known-keyed index, not "${indexName}"`); - } - - return rows.toSorted((a, b) => { - const delta = ((a[field] as number | undefined) ?? 0) - ((b[field] as number | undefined) ?? 0); - - return direction === "desc" ? -delta : delta; - }); -}; - -const makePrototypeIndexQuery = (allRows: PrototypeRow[], indexName: string, conditions: [string, unknown][]): PrototypeIndexQuery => { - const matches = (): PrototypeRow[] => allRows.filter((row) => matchesConditions(row, conditions)); - - return { - collect: async () => matches(), - first: async () => matches()[0] ?? null, - order: (direction: "asc" | "desc") => { - return { - collect: async () => sortByField(matches(), ORDER_FIELDS[indexName], indexName, direction), - }; - }, - }; -}; - -/** - * Minimal `ctx.db` double for the prototype — same shape as - * `__tests__/component.test.ts`'s `fakeDatabase`, extended with `remove` (the - * prototype needs to delete a dequeued queue row). - */ -const fakePrototypeDatabase = (): { db: PrototypeDatabase; rows: Map } => { - const rows = new Map(); - let nextId = 0; - - const tableRows = (table: string): PrototypeRow[] => { - const existing = rows.get(table); - - if (existing) { - return existing; - } - - const created: PrototypeRow[] = []; - - rows.set(table, created); - - return created; - }; - - const db: PrototypeDatabase = { - insert: async (table, document) => { - const id = `id-${String(nextId)}`; - - nextId += 1; - tableRows(table).push({ ...document, _id: id }); - - return id; - }, - patch: async (id, patch) => { - for (const tableContent of rows.values()) { - const row = tableContent.find((candidate) => candidate["_id"] === id); - - if (row) { - Object.assign(row, patch); - } - } - }, - query: (table) => { - return { - withIndex: (name, build) => makePrototypeIndexQuery(tableRows(table), name, collectConditions(build)), - }; - }, - remove: async (id) => { - for (const tableContent of rows.values()) { - const index = tableContent.findIndex((candidate) => candidate["_id"] === id); - - if (index !== -1) { - tableContent.splice(index, 1); - } - } - }, - }; - - return { db, rows }; -}; - -describe("run-queue prototype (plan 240 spike)", () => { - it("parks B instead of rejecting it, then resumes B in order when A completes", async () => { - const { db } = fakePrototypeDatabase(); - - // A starts and is running. - await expect(ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-a", key: "t-1" })).resolves.toStrictEqual({ created: true }); - - // B starts while A is in flight, with onConcurrentRun: "queue" — must - // park, not throw CONFLICT. - const parked = await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-b", key: "t-1", onConcurrentRun: "queue" }); - - expect(parked).toStrictEqual({ created: false, position: 0, queued: true }); - - // A completes — dequeue-and-handoff must return B's instance id and - // stamp the thread over to B atomically. - const handoff = await completeRunAndDequeue(db, { instanceId: "wf-a", key: "t-1", terminalStatus: "idle" }); - - expect(handoff).toStrictEqual({ dequeued: "wf-b" }); - }); - - it("preserves A -> B -> C ordering: C parks behind B, and dequeues follow FIFO position, not completion order", async () => { - const { db, rows } = fakePrototypeDatabase(); - - await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-a", key: "t-1" }); - - const parkedB = await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-b", key: "t-1", onConcurrentRun: "queue" }); - const parkedC = await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-c", key: "t-1", onConcurrentRun: "queue" }); - - expect(parkedB).toStrictEqual({ created: false, position: 0, queued: true }); - expect(parkedC).toStrictEqual({ created: false, position: 1, queued: true }); - - // A completes -> hands off to B (the head, position 0). - const ownershipSequence: string[] = ["wf-a"]; - const firstHandoff = await completeRunAndDequeue(db, { instanceId: "wf-a", key: "t-1", terminalStatus: "idle" }); - - expect(firstHandoff).toStrictEqual({ dequeued: "wf-b" }); - - ownershipSequence.push("wf-b"); - - expect(rows.get("proto_agent_threads")?.[0]?.["instanceId"]).toBe("wf-b"); - - // B completes -> hands off to C (the only one left). - const secondHandoff = await completeRunAndDequeue(db, { instanceId: "wf-b", key: "t-1", terminalStatus: "idle" }); - - expect(secondHandoff).toStrictEqual({ dequeued: "wf-c" }); - - ownershipSequence.push("wf-c"); - - expect(rows.get("proto_agent_threads")?.[0]?.["instanceId"]).toBe("wf-c"); - - // C completes -> nobody left, thread goes terminal. - const thirdHandoff = await completeRunAndDequeue(db, { instanceId: "wf-c", key: "t-1", terminalStatus: "idle" }); - - expect(thirdHandoff).toStrictEqual({ dequeued: undefined }); - expect(rows.get("proto_agent_threads")?.[0]?.["status"]).toBe("idle"); - - expect(ownershipSequence).toStrictEqual(["wf-a", "wf-b", "wf-c"]); - expect(rows.get("proto_agent_run_queue")).toHaveLength(0); - }); - - it("a replay of B's still-parked bootstrap does not duplicate its queue entry or change its position", async () => { - const { db, rows } = fakePrototypeDatabase(); - - await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-a", key: "t-1" }); - await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-b", key: "t-1", onConcurrentRun: "queue" }); - // A THIRD run parks behind B before B replays, so a mis-ordered - // duplicate would be observable as a position/count change. - await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-c", key: "t-1", onConcurrentRun: "queue" }); - - expect(rows.get("proto_agent_run_queue")).toHaveLength(2); - - // B's workflow replays (re-executes ensureThread for real, since it - // runs outside step.do) while still parked, before A has completed. - const replay = await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-b", key: "t-1", onConcurrentRun: "queue" }); - - expect(replay).toStrictEqual({ created: false, position: 0, queued: true }); - expect(rows.get("proto_agent_run_queue")).toHaveLength(2); - - // Ordering still resolves B then C — the replay didn't reshuffle anything. - const firstHandoff = await completeRunAndDequeue(db, { instanceId: "wf-a", key: "t-1", terminalStatus: "idle" }); - - expect(firstHandoff).toStrictEqual({ dequeued: "wf-b" }); - - const secondHandoff = await completeRunAndDequeue(db, { instanceId: "wf-b", key: "t-1", terminalStatus: "idle" }); - - expect(secondHandoff).toStrictEqual({ dequeued: "wf-c" }); - }); - - it("a replay of A's own completion does not double-dequeue and skip C's turn", async () => { - const { db } = fakePrototypeDatabase(); - - await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-a", key: "t-1" }); - await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-b", key: "t-1", onConcurrentRun: "queue" }); - await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-c", key: "t-1", onConcurrentRun: "queue" }); - - const firstCall = await completeRunAndDequeue(db, { instanceId: "wf-a", key: "t-1", terminalStatus: "idle" }); - - expect(firstCall).toStrictEqual({ dequeued: "wf-b" }); - - // A REPLAYS its own completion step (e.g. the workflow re-runs after a - // crash before the step was durably recorded as done). The thread no - // longer belongs to "wf-a", so this must be a no-op — NOT a second - // dequeue that would hand C the thread out from under B. - const replayedCall = await completeRunAndDequeue(db, { instanceId: "wf-a", key: "t-1", terminalStatus: "idle" }); - - expect(replayedCall).toStrictEqual({ dequeued: undefined }); - - // C is still parked, waiting for B — not skipped. - const secondCall = await completeRunAndDequeue(db, { instanceId: "wf-b", key: "t-1", terminalStatus: "idle" }); - - expect(secondCall).toStrictEqual({ dequeued: "wf-c" }); - }); - - it("enforces the queue depth bound, rejecting past the cap instead of growing unbounded durable state", async () => { - const { db, rows } = fakePrototypeDatabase(); - - await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-a", key: "t-1" }); - - for (let index = 0; index < MAX_QUEUE_DEPTH; index += 1) { - // eslint-disable-next-line no-await-in-loop -- each enqueue depends on the prior one's committed depth - await ensureThreadOrQueue(db, { agent: "support", instanceId: `wf-q${String(index)}`, key: "t-1", onConcurrentRun: "queue" }); - } - - expect(rows.get("proto_agent_run_queue")).toHaveLength(MAX_QUEUE_DEPTH); - - await expect(ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-overflow", key: "t-1", onConcurrentRun: "queue" })).rejects.toThrow( - QUEUE_FULL_PATTERN, - ); - - // The overflowing attempt left no trace. - expect(rows.get("proto_agent_run_queue")).toHaveLength(MAX_QUEUE_DEPTH); - }); - - it("still throws CONFLICT for the default (reject) policy — unchanged from today", async () => { - const { db } = fakePrototypeDatabase(); - - await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-a", key: "t-1" }); - - await expect(ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-b", key: "t-1" })).rejects.toThrow(IN_FLIGHT_PATTERN); - await expect(ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-b", key: "t-1", onConcurrentRun: "reject" })).rejects.toThrow( - IN_FLIGHT_PATTERN, - ); - }); - - it("still replaces (terminating the in-flight run's ownership) — unchanged from today", async () => { - const { db, rows } = fakePrototypeDatabase(); - - await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-old", key: "t-1" }); - - const result = await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-new", key: "t-1", onConcurrentRun: "replace" }); - - expect(result).toStrictEqual({ created: false, priorInstanceId: "wf-old", replaced: true }); - expect(rows.get("proto_agent_threads")?.[0]?.["instanceId"]).toBe("wf-new"); - }); - - it("rejects an id-less dispatch under queue policy rather than parking an un-wakeable run", async () => { - const { db } = fakePrototypeDatabase(); - - await ensureThreadOrQueue(db, { agent: "support", instanceId: "wf-a", key: "t-1" }); - - await expect(ensureThreadOrQueue(db, { agent: "support", key: "t-1", onConcurrentRun: "queue" })).rejects.toThrow(QueueConflictError); - }); -}); diff --git a/packages/agent/__tests__/run-queue.test.ts b/packages/agent/__tests__/run-queue.test.ts new file mode 100644 index 0000000000..93ef7329a0 --- /dev/null +++ b/packages/agent/__tests__/run-queue.test.ts @@ -0,0 +1,232 @@ +/** + * `onConcurrentRun: "queue"` — the durable per-thread run queue. + * + * Exercises the REAL `agentEnsureThread` / `agentCompleteRun` mutations over an + * in-memory `ctx.db`, because the properties under test are all about what the + * mutations do to the tables: FIFO order, an idempotent enqueue under replay, an + * idempotent handoff under replay, and the bound. + */ +import { describe, expect, it } from "vitest"; + +import { agentComponent } from "../src/component"; + +interface FakeRow extends Record { + _id: string; +} + +/** Collect the `.eq(...)` conditions a `withIndex` callback declares. */ +const collectConditions = (build: (q: unknown) => unknown): [string, unknown][] => { + const conditions: [string, unknown][] = []; + const builder = { + eq: (field: string, value: unknown) => { + conditions.push([field, value]); + + return builder; + }, + }; + + build(builder); + + return conditions; +}; + +/** Filter by the `.eq(...)` conditions; insertion order stands in for index order. */ +const makeIndexQuery = (candidates: FakeRow[], build: (q: unknown) => unknown): { collect: () => Promise; first: () => Promise } => { + const conditions = collectConditions(build); + const matches = (): FakeRow[] => candidates.filter((row) => conditions.every(([field, value]) => row[field] === value)); + + return { + collect: async () => matches(), + first: async () => matches()[0] ?? null, + }; +}; + +/** + * An in-memory `ctx.db`. `withIndex` filters by the declared equalities and — as + * the real index read does — returns rows in insertion order, which for the + * `(threadKey, position)` index is position order. + */ +const fakeDatabase = (): { database: Record; rows: Map } => { + const rows = new Map(); + let nextId = 0; + + const tableRows = (table: string): FakeRow[] => { + const existing = rows.get(table); + + if (existing) { + return existing; + } + + const created: FakeRow[] = []; + + rows.set(table, created); + + return created; + }; + + const database = { + delete: async (id: string) => { + for (const [table, tableContent] of rows) { + rows.set( + table, + tableContent.filter((row) => row["_id"] !== id), + ); + } + }, + insert: async (table: string, document: Record) => { + const id = `id-${String(nextId)}`; + + nextId += 1; + tableRows(table).push({ ...document, _id: id }); + + return id; + }, + patch: async (id: string, patch: Record) => { + for (const tableContent of rows.values()) { + const row = tableContent.find((candidate) => candidate["_id"] === id); + + if (row) { + for (const [key, value] of Object.entries(patch)) { + if (value === undefined) { + Reflect.deleteProperty(row, key); + } else { + row[key] = value; + } + } + } + } + }, + query: (table: string) => { + return { + withIndex: (_name: string, build: (q: unknown) => unknown) => makeIndexQuery(tableRows(table), build), + }; + }, + }; + + return { database, rows }; +}; + +const setup = () => { + const { database, rows } = fakeDatabase(); + const { functions } = agentComponent(); + const context = { auth: { userId: undefined }, db: database }; + + return { + complete: async (arguments_: Record) => + (await functions.agentCompleteRun.handler(context, arguments_ as never)) as { dequeued?: string }, + queue: (): FakeRow[] => rows.get("agent_run_queue") ?? [], + start: async (instanceId?: string) => + (await functions.agentEnsureThread.handler(context, { + agent: "support", + key: "thread-1", + onConcurrentRun: "queue", + ...(instanceId === undefined ? {} : { instanceId }), + } as never)) as { created: boolean; position?: number; queued?: boolean }, + thread: (): FakeRow | undefined => (rows.get("agent_threads") ?? [])[0], + }; +}; + +describe("onConcurrentRun: queue", () => { + it("parks runs behind the one in flight and hands the thread over in FIFO order", async () => { + expect.assertions(7); + + const { complete, queue, start, thread } = setup(); + + await expect(start("wf-a")).resolves.toStrictEqual({ created: true }); + await expect(start("wf-b")).resolves.toStrictEqual({ created: false, position: 0, queued: true }); + await expect(start("wf-c")).resolves.toStrictEqual({ created: false, position: 1, queued: true }); + // A parked run must not take the thread from the one in flight. + expect(thread()?.["instanceId"]).toBe("wf-a"); + + // A finishes: the thread goes to B (not to the terminal status), and B's + // queue row is consumed in the same mutation that transfers ownership. + await expect(complete({ instanceId: "wf-a", key: "thread-1", status: "idle" })).resolves.toStrictEqual({ dequeued: "wf-b" }); + expect(thread()).toMatchObject({ instanceId: "wf-b", status: "running" }); + expect(queue().map((row) => row["instanceId"])).toStrictEqual(["wf-c"]); + }); + + it("goes idle once the queue drains", async () => { + expect.assertions(2); + + const { complete, start, thread } = setup(); + + await start("wf-a"); + await start("wf-b"); + await complete({ instanceId: "wf-a", key: "thread-1", status: "idle" }); + + await expect(complete({ instanceId: "wf-b", key: "thread-1", status: "idle" })).resolves.toStrictEqual({}); + expect(thread()).toMatchObject({ instanceId: "wf-b", status: "idle" }); + }); + + it("is idempotent under replay: a parked run re-enqueues to the same slot", async () => { + expect.assertions(2); + + const { queue, start } = setup(); + + await start("wf-a"); + await start("wf-b"); + + // A workflow replay re-runs the bootstrap for real (it is outside step.do). + await expect(start("wf-b")).resolves.toStrictEqual({ created: false, position: 0, queued: true }); + expect(queue()).toHaveLength(1); + }); + + it("is idempotent under replay: a finished run's completion never dequeues twice", async () => { + expect.assertions(3); + + const { complete, queue, start, thread } = setup(); + + await start("wf-a"); + await start("wf-b"); + await start("wf-c"); + await complete({ instanceId: "wf-a", key: "thread-1", status: "idle" }); + + // A's completion replays after ownership already moved to B. Dequeuing + // again here would skip B's turn entirely. + await expect(complete({ instanceId: "wf-a", key: "thread-1", status: "idle" })).resolves.toStrictEqual({}); + expect(thread()?.["instanceId"]).toBe("wf-b"); + expect(queue().map((row) => row["instanceId"])).toStrictEqual(["wf-c"]); + }); + + it("hands the thread on even when the finishing run errored", async () => { + expect.assertions(1); + + const { complete, start } = setup(); + + await start("wf-a"); + await start("wf-b"); + + // B is waiting for A to END, not to succeed. + await expect(complete({ error: "boom", instanceId: "wf-a", key: "thread-1", status: "error" })).resolves.toStrictEqual({ dequeued: "wf-b" }); + }); + + it("rejects past the depth cap instead of parking unboundedly", async () => { + expect.assertions(2); + + const { queue, start } = setup(); + + await start("wf-a"); + + for (const id of ["wf-b", "wf-c", "wf-d", "wf-e", "wf-f"]) { + // eslint-disable-next-line no-await-in-loop -- sequential: each enqueue must observe the previous one's row + await start(id); + } + + expect(queue()).toHaveLength(5); + await expect(start("wf-g")).rejects.toThrow("run queue is full"); + }); + + it("refuses to queue a dispatch that has no instance id to wake", async () => { + expect.assertions(2); + + const { queue, start } = setup(); + + await start("wf-a"); + + // The inbound-email / inbound-channel paths dispatch with no instanceId: + // nothing could tell two such dispatches apart later to wake the right + // one, so parking them would strand a run rather than order it. + await expect(start()).rejects.toThrow("cannot queue a dispatch with no instance id"); + expect(queue()).toHaveLength(0); + }); +}); diff --git a/packages/agent/__tests__/skill.test.ts b/packages/agent/__tests__/skill.test.ts index d36031a1e9..086e43c107 100644 --- a/packages/agent/__tests__/skill.test.ts +++ b/packages/agent/__tests__/skill.test.ts @@ -150,8 +150,26 @@ const memoryRuntime = ( return undefined; }; + /** Terminal patch + (empty-queue) handoff — see `run-queue.test.ts` for the queue itself. */ + const completeRun = (args?: Record): unknown => { + const thread = threads.get(args?.["key"] as string); + + if (!args || !thread || thread.instanceId !== args["instanceId"]) { + return {}; + } + + thread.status = args["status"] as string; + + if (args["error"] !== undefined) { + thread.error = args["error"] as string; + } + + return {}; + }; + const handlers = new Map) => unknown>([ [DEFAULT_AGENT_FUNCTION_PATHS.appendMessage, appendMessage], + [DEFAULT_AGENT_FUNCTION_PATHS.completeRun, completeRun], [DEFAULT_AGENT_FUNCTION_PATHS.ensureThread, ensureThread], [DEFAULT_AGENT_FUNCTION_PATHS.listMessages, listMessages], [DEFAULT_AGENT_FUNCTION_PATHS.patchThread, patchThread], diff --git a/packages/agent/package.json b/packages/agent/package.json index 426513c768..85cbffb2fe 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -66,6 +66,10 @@ "types": "./dist/skill-markdown.d.ts", "import": "./dist/skill-markdown.mjs" }, + "./reply": { + "types": "./dist/reply.d.ts", + "import": "./dist/reply.mjs" + }, "./sandbox": { "types": "./dist/sandbox.d.ts", "import": "./dist/sandbox.mjs" diff --git a/packages/agent/src/agent-loop.ts b/packages/agent/src/agent-loop.ts index 4a25b0c54d..39c70b8138 100644 --- a/packages/agent/src/agent-loop.ts +++ b/packages/agent/src/agent-loop.ts @@ -771,6 +771,111 @@ const terminatePriorInstance = async (env: Record, exportName: } }; +/** + * How long a parked run waits to be handed the thread before giving up. + * + * The wake event is sent from a durable step, so it survives ordinary failures; + * the timeout covers the case it can never land — the finishing run's instance + * was terminated after the handoff committed, or its binding was gone. Without + * it a parked run hibernates forever holding a queue slot. Twelve hours is far + * longer than any agent run and far shorter than "never". + */ +const DEQUEUE_TIMEOUT = "12 hours"; + +/** + * Hibernate until this run is handed the thread it queued for. + * + * The match `type` is scoped to `(threadKey, instanceId)` for the same reason + * the HITL approval scopes to a tool-call id: native CF Workflows matches a + * waiter by `type`, so a shared type would let one parked run's wake resolve a + * different one. A workflow replay memoizes the resolved wait, so a resumed run + * does not park twice. + */ +const awaitDequeue = async (step: AgentStepLike, threadKey: string, instanceId: string, queued: boolean | undefined): Promise => { + if (queued !== true) { + return; + } + + await step.waitForEvent<{ threadKey: string }>(`dequeue:${instanceId}`, { + timeout: DEQUEUE_TIMEOUT, + type: `agent-dequeue:${threadKey}:${instanceId}`, + }); +}; + +/** + * Wake the run the completion mutation just handed the thread to. + * + * Wrapped in a durable step so a transient failure is retried rather than + * stranding a run that already owns the thread; if it can never land, the + * parked run's own {@link DEQUEUE_TIMEOUT} ends it instead of hibernating + * forever. Best-effort by design — the ownership transfer already committed, + * and re-running this mutation is not an option. + */ +const wakeDequeuedRun = async ( + deps: { env: Record; exportName: string; step: AgentStepLike }, + threadKey: string, + dequeuedInstanceId: string, +): Promise => { + const binding = deps.env[agentBindingName(deps.exportName)] as AgentWorkflowBindingLike | undefined; + + if (!binding || typeof binding.get !== "function") { + return; + } + + const get = binding.get.bind(binding); + + try { + await deps.step.do(`dequeue-wake:${dequeuedInstanceId}`, async (): Promise => { + const instance = await get(dequeuedInstanceId); + + await instance.sendEvent({ payload: { threadKey }, type: `agent-dequeue:${threadKey}:${dequeuedInstanceId}` }); + + return dequeuedInstanceId; + }); + } catch { + // The dequeued instance is gone (terminated while parked), or the send + // kept failing. It owns the thread until its wait times out; a run that + // starts after that finds an ownerless thread and proceeds normally. + } +}; + +/** + * Hand a triggered run's final answer to the agent's `onReply`. + * + * No-op unless the run carries a `replyRef` (an ordinary in-app run does not) + * AND the agent declared `onReply`. Wrapped in a named durable step so the + * delivery is retried on a transient provider failure and, under a workflow + * replay, is served from the memo instead of sending the answer twice — the + * same guarantee `onStepFinish` gets. + * + * A reply that keeps failing must not turn a completed run into a failed one: + * the answer is already persisted on the thread, and re-running the loop to + * retry a Slack post would re-run the model. The failure is swallowed after the + * step's own retries, exactly as run-end memory extraction is. + */ +const deliverReply = async ( + agent: AgentDefinition, + context: { env: Record; params: AgentRunInput; step: AgentStepLike }, + result: AgentRunResult, +): Promise => { + const { onReply } = agent; + const { replyRef } = context.params; + + if (onReply === undefined || replyRef === undefined) { + return; + } + + try { + await context.step.do("agent:reply", async (): Promise => { + await onReply({ env: context.env, replyRef, result, threadKey: context.params.threadKey }); + + return replyRef.channel; + }); + } catch { + // Delivery failed past its retries — the answer is on the thread either way. + } +}; + /** How the loop proceeds after a turn: finish, stop on a condition, or continue. */ type TurnOutcome = "stopCondition" | undefined | { final: AgentRunResult }; @@ -917,6 +1022,7 @@ const runAgentLoop = async (options: AgentLoopOptions): Promise const stopConditions = normalizeStopWhen(agent.stopWhen); const appendMessage = toFunctionReference(paths.appendMessage); + const completeRun = toFunctionReference(paths.completeRun); const ensureThread = toFunctionReference(paths.ensureThread); const listMessages = toFunctionReference(paths.listMessages); const patchThread = toFunctionReference(paths.patchThread); @@ -931,6 +1037,20 @@ const runAgentLoop = async (options: AgentLoopOptions): Promise await run(patchThread, { key: params.threadKey, ...patch }); }; + /** + * End the run: write the terminal status and, in the same mutation, hand the + * thread to the next queued run. Only then is the dequeued run woken — it + * already owns the thread by that point, so there is no window in which two + * runs believe they may append. + */ + const finishRun = async (patch: { error?: string; status: "error" | "idle"; usage?: AgentUsage }): Promise => { + const outcome = (await run(completeRun, { instanceId, key: params.threadKey, ...patch })) as { dequeued?: string } | undefined; + + if (outcome?.dequeued !== undefined) { + await wakeDequeuedRun({ env, exportName, step }, params.threadKey, outcome.dequeued); + } + }; + // Synced-state closures handed to every tool ctx. They dispatch through the // same `run` seam the loop uses for history/persistence, so a call made from // inside a tool's memoized `step.do` is replay-safe (the memoized step is @@ -958,7 +1078,7 @@ const runAgentLoop = async (options: AgentLoopOptions): Promise ...(agent.onConcurrentRun === undefined ? {} : { onConcurrentRun: agent.onConcurrentRun }), ...(params.owner === undefined ? {} : { owner: params.owner }), ...(params.title === undefined ? {} : { title: params.title }), - })) as { created: boolean; priorInstanceId?: string; replaced?: boolean } | undefined; + })) as { created: boolean; position?: number; priorInstanceId?: string; queued?: boolean; replaced?: boolean } | undefined; // Replace policy: the thread has been taken over — terminate the run it was // taken from so it cannot resume and race on the shared seq counter. @@ -966,6 +1086,12 @@ const runAgentLoop = async (options: AgentLoopOptions): Promise await terminatePriorInstance(env, exportName, bootstrap.priorInstanceId); } + // Queue policy: when this run was parked behind the one in flight, hibernate + // until the finishing run's completion mutation hands us the thread — by + // which point we already own it, so the first append below lands on the + // shared seq counter with nobody else writing to it. + await awaitDequeue(step, params.threadKey, instanceId, bootstrap?.queued); + await persist({ content: params.input, messageKey: `${instanceId}:user`, role: "user" }); // Memory step: dispatch the configured retrieval action once per run and @@ -1041,7 +1167,11 @@ const runAgentLoop = async (options: AgentLoopOptions): Promise }); if (final) { - await run(patchThread, { key: params.threadKey, status: "idle", ...usagePatch }); + // Answer where the question was asked, before the thread goes idle: + // a triggered run that never replies looks, from the channel it came + // from, exactly like one that never ran. + await deliverReply(agent, { env, params, step }, { ...final, ...usagePatch }); + await finishRun({ status: "idle", ...usagePatch }); return { ...final, ...usagePatch }; } @@ -1049,20 +1179,21 @@ const runAgentLoop = async (options: AgentLoopOptions): Promise if (stoppedByCondition) { // A stop condition is a deliberate end, not a failure — leave the // thread idle so it can continue on the next run. - await run(patchThread, { key: params.threadKey, status: "idle", ...usagePatch }); + await finishRun({ status: "idle", ...usagePatch }); return { stopped: "stopCondition", turns: turnsRun, ...usagePatch }; } - await run(patchThread, { error: `agent stopped: maxTurns (${String(maxTurns)}) reached`, key: params.threadKey, status: "error", ...usagePatch }); + await finishRun({ error: `agent stopped: maxTurns (${String(maxTurns)}) reached`, status: "error", ...usagePatch }); return { stopped: "maxTurns", turns: maxTurns, ...usagePatch }; } catch (error) { // Surface the failure on the thread (observers see status:"error"), - // then rethrow so the workflow records/retries per its policy. - await run(patchThread, { + // then rethrow so the workflow records/retries per its policy. A failed + // run still hands the thread on: the next queued run is waiting for THIS + // one to end, not for it to succeed. + await finishRun({ error: error instanceof Error ? error.message : String(error), - key: params.threadKey, status: "error", ...(usageBox.value === undefined ? {} : { usage: usageBox.value }), }); diff --git a/packages/agent/src/component.ts b/packages/agent/src/component.ts index f814d8d9a4..dbc921f4a0 100644 --- a/packages/agent/src/component.ts +++ b/packages/agent/src/component.ts @@ -11,10 +11,24 @@ import { graphComponent, graphTables } from "./graph-component"; /** Bare table names — auto-prefixed with the extension key at merge time. */ const THREADS_BARE_TABLE = "threads"; const MESSAGES_BARE_TABLE = "messages"; +const RUN_QUEUE_BARE_TABLE = "run_queue"; /** The physical (merged) table names the runtime functions read/write. */ const THREADS_TABLE: "agent_threads" = `${AGENT_EXTENSION_KEY}_${THREADS_BARE_TABLE}`; const MESSAGES_TABLE: "agent_messages" = `${AGENT_EXTENSION_KEY}_${MESSAGES_BARE_TABLE}`; +const RUN_QUEUE_TABLE: "agent_run_queue" = `${AGENT_EXTENSION_KEY}_${RUN_QUEUE_BARE_TABLE}`; + +/** + * How many runs may park behind the one in flight under + * `onConcurrentRun: "queue"` before further starts are rejected. + * + * Each parked run is a live workflow instance hibernating on `waitForEvent`, so + * the cap is a real resource bound, not a formality: five is enough to absorb an + * impatient user double-sending, and small enough that a runaway trigger loop + * fails loudly instead of billing an unbounded pile of paused instances. Past + * the cap the start fails with the same `CONFLICT` `"reject"` already throws. + */ +const MAX_QUEUE_DEPTH = 5; /** * The agent thread tables, shipped as a schema extension so an app merges @@ -120,6 +134,29 @@ const agentExtension: SchemaExtension = defineSchemaExtension(AGENT_EXTENSION_KE // internal-only writes. .public(), + [RUN_QUEUE_BARE_TABLE]: defineTable({ + agent: v.string(), + enqueuedAt: v.number(), + /** The parked run's workflow instance — the wake event is scoped to it. */ + instanceId: v.string(), + + /** + * Monotonic per-thread position, allocated as `max(position) + 1` + * over the rows currently queued on this thread. Ordering by it + * rather than by `enqueuedAt` keeps FIFO exact when two runs are + * enqueued inside the same millisecond. + */ + position: v.number(), + threadKey: v.string(), + }) + // FIFO dequeue. + .index("byThread", ["threadKey", "position"]) + // Idempotent enqueue: a replay of a still-parked run's bootstrap + // finds its own row instead of taking a second slot. + .index("byThreadInstance", ["threadKey", "instanceId"], { unique: true }) + // See the threads table for why the agent tables are `.public()`. + .public(), + // The owner-scoped graph-memory tables (`agent_entities`/`agent_edges`) // live in graph-component.ts alongside the functions that read/write // them; spread in here so the app still merges one `agentExtension`. @@ -135,6 +172,131 @@ const agentExtension: SchemaExtension = defineSchemaExtension(AGENT_EXTENSION_KE // generated server inside a package), same as the presence component. const { mutation, query } = initLunora.dataModel().create(); +/** + * What `agentEnsureThread` reports back to the loop: the thread was created, + * continued, taken over from a prior instance (`replaced`), or — under + * `onConcurrentRun: "queue"` — parked behind the run in flight (`queued`), in + * which case the loop hibernates until the queue hands it the thread. + * @experimental + */ +interface EnsureThreadResult { + created: boolean; + /** FIFO position among the runs parked on this thread. Present only with `queued`. */ + position?: number; + /** The instance the thread was taken over FROM. Present only with `replaced`. */ + priorInstanceId?: string; + queued?: boolean; + replaced?: boolean; +} + +/** + * The minimal writer surface the queue helpers need. `agentEnsureThread` and + * `agentCompleteRun` are ordinary mutations, so this is just `ctx.db` — named + * so the helpers can be lifted out of the closures without dragging the whole + * generated data-model type along. + */ +type QueueDatabase = Parameters[0]>[0]["ctx"]["db"]; + +/** + * Park a run behind the one currently in flight (`onConcurrentRun: "queue"`). + * + * Three properties make this safe, and all three come from the DO executing one + * mutation at a time: the enqueue is idempotent by `(threadKey, instanceId)`, so + * a workflow replay of a still-parked run finds its own slot instead of taking a + * second; the position is allocated from the rows currently queued, so FIFO is + * exact; and the thread's ownership is untouched, so the in-flight run keeps the + * seq counter until it finishes and hands it over in one mutation. + */ +const enqueueRun = async ( + database: QueueDatabase, + args: { agent: string; instanceId: string | undefined; key: string; priorInstanceId: string }, + now: number, +): Promise => { + if (args.instanceId === undefined) { + // The id-less dispatch paths (inbound email / inbound channel) can't be + // told apart from each other later, so they can't be parked and woken by + // instance id. Rejecting is the honest answer, and the same one an + // overflowing queue gives. + throw new LunoraError( + "CONFLICT", + `@lunora/agent: thread "${args.key}" already has a run in flight (instance "${args.priorInstanceId}") — cannot queue a dispatch with no instance id`, + ); + } + + const { instanceId } = args; + const alreadyQueued = await database + .query(RUN_QUEUE_TABLE) + .withIndex("byThreadInstance", (q) => q.eq("threadKey", args.key).eq("instanceId", instanceId)) + .first(); + + if (alreadyQueued) { + return { created: false, position: alreadyQueued["position"] as number, queued: true }; + } + + const queued = await database + .query(RUN_QUEUE_TABLE) + .withIndex("byThread", (q) => q.eq("threadKey", args.key)) + .collect(); + + if (queued.length >= MAX_QUEUE_DEPTH) { + throw new LunoraError( + "CONFLICT", + `@lunora/agent: thread "${args.key}" run queue is full (depth ${String(MAX_QUEUE_DEPTH)}) — rejecting instance "${instanceId}"`, + ); + } + + const position = queued.length === 0 ? 0 : Math.max(...queued.map((row) => row["position"] as number)) + 1; + + await database.insert(RUN_QUEUE_TABLE, { agent: args.agent, enqueuedAt: now, instanceId, position, threadKey: args.key }); + + return { created: false, position, queued: true }; +}; + +/** + * Apply `onConcurrentRun` when a second run arrives on a thread another live + * instance already owns: park it, take the thread over, or reject. + * + * Split out of `agentEnsureThread` so the mutation body stays about the thread + * lifecycle (create / continue / replay) and this stays about the policy. + */ +const applyConcurrencyPolicy = async ( + database: QueueDatabase, + args: { + agent: string; + existingId: string; + instanceId: string | undefined; + key: string; + policy: "queue" | "reject" | "replace"; + priorInstanceId: string; + }, + now: number, +): Promise => { + if (args.policy === "queue") { + return enqueueRun(database, { agent: args.agent, instanceId: args.instanceId, key: args.key, priorInstanceId: args.priorInstanceId }, now); + } + + if (args.policy !== "replace") { + throw new LunoraError( + "CONFLICT", + `@lunora/agent: thread "${args.key}" already has a run in flight (instance "${args.priorInstanceId}") — onConcurrentRun="${args.policy}"`, + ); + } + + // Replace: take the thread over now (the caller terminates the prior + // instance) so the next append is attributed to this run. The incoming + // instance id may itself be absent (an id-less caller replacing a live run) + // — omit the column rather than writing an explicit `undefined`, which the + // validators reject. + await database.patch(args.existingId as never, { + error: undefined, + status: "running", + updatedAt: now, + ...(args.instanceId === undefined ? {} : { instanceId: args.instanceId }), + }); + + return { created: false, priorInstanceId: args.priorInstanceId, replaced: true }; +}; + /** * `AgentComponent` is part of the experimental `@lunora/agent` API and may change without a major version bump. * @experimental @@ -143,6 +305,7 @@ export interface AgentComponent { extension: SchemaExtension; functions: { agentAppendMessage: AgentRegisteredFunction; + agentCompleteRun: AgentRegisteredFunction; agentEnsureThread: AgentRegisteredFunction; agentEpisodeRecall: AgentRegisteredFunction; agentEpisodeUpsert: AgentRegisteredFunction; @@ -185,7 +348,7 @@ export const agentComponent = (): AgentComponent => { owner: v.optional(v.string()), title: v.optional(v.string()), }) - .mutation(async ({ args, ctx: context }): Promise<{ created: boolean; priorInstanceId?: string; replaced?: boolean }> => { + .mutation(async ({ args, ctx: context }): Promise => { const now = Date.now(); const existing = await context.db .query(THREADS_TABLE) @@ -223,30 +386,18 @@ export const agentComponent = (): AgentComponent => { (args.instanceId === undefined || args.instanceId !== priorInstanceId); if (isConcurrentRun) { - const policy = args.onConcurrentRun ?? "reject"; - - // "queue" has no durable queue yet — degrade to reject rather - // than silently interleave (tracked as a follow-up). - if (policy !== "replace") { - throw new LunoraError( - "CONFLICT", - `@lunora/agent: thread "${args.key}" already has a run in flight (instance "${priorInstanceId}") — onConcurrentRun="${policy}"`, - ); - } - - // Replace: take the thread over now (the caller terminates the - // prior instance) so the next append is attributed to this run. - // The incoming instance id may itself be absent (an id-less - // caller replacing a live run) — omit the column rather than - // writing an explicit `undefined`, which the validators reject. - await context.db.patch(existing["_id"] as never, { - error: undefined, - status: "running", - updatedAt: now, - ...(args.instanceId === undefined ? {} : { instanceId: args.instanceId }), - }); - - return { created: false, priorInstanceId, replaced: true }; + return applyConcurrencyPolicy( + context.db, + { + agent: args.agent, + existingId: existing["_id"] as string, + instanceId: args.instanceId, + key: args.key, + policy: args.onConcurrentRun ?? "reject", + priorInstanceId, + }, + now, + ); } // Replay (same instance) or a resumed idle/errored/cancelled @@ -372,6 +523,77 @@ export const agentComponent = (): AgentComponent => { }); }); + /** + * End the owning run and, in the SAME mutation, hand the thread to the next + * parked run if one is waiting. + * + * Doing both here is the whole correctness argument: a separate + * "patch terminal status" then "dequeue" pair would leave a window in which + * the thread is ownerless, and a third run arriving inside that window would + * see an idle thread and start immediately — ahead of runs that had been + * queued for their turn, and racing the one being woken. + * + * Idempotent under a workflow replay of the FINISHING run: the caller must + * still own the thread, so a second call (after ownership has already moved + * on) is a no-op rather than a second dequeue that skips someone's turn. + */ + const agentCompleteRun = mutation + .input({ + error: v.optional(v.string()), + instanceId: v.string(), + key: v.string(), + status: v.union(v.literal("idle"), v.literal("error"), v.literal("cancelled")), + usage: v.optional(v.object({ inputTokens: v.optional(v.number()), outputTokens: v.optional(v.number()), totalTokens: v.optional(v.number()) })), + }) + .mutation(async ({ args, ctx: context }): Promise<{ dequeued?: string }> => { + const now = Date.now(); + const thread = await context.db + .query(THREADS_TABLE) + .withIndex("byKey", (q) => q.eq("key", args.key)) + .first(); + + if (thread?.["instanceId"] !== args.instanceId) { + // Unknown thread, or ownership already moved on (a replay of this + // same completion) — either way not this call's to end. + return {}; + } + + // `byThread` is `(threadKey, position)`, so an index read is already + // position-ascending — the head of this list is the FIFO next. + const queued = await context.db + .query(RUN_QUEUE_TABLE) + .withIndex("byThread", (q) => q.eq("threadKey", args.key)) + .collect(); + const next = queued[0]; + + if (!next) { + await context.db.patch(thread["_id"] as never, { + status: args.status, + updatedAt: now, + ...(args.error === undefined ? {} : { error: args.error }), + ...(args.usage === undefined ? {} : { usage: args.usage }), + }); + + return {}; + } + + const nextInstanceId = next["instanceId"] as string; + + await context.db.delete(next["_id"] as never); + // Ownership transfers here, before the waking event is sent: the + // dequeued run is already the owner when it resumes, so it can append + // on the shared seq counter the moment it wakes. + await context.db.patch(thread["_id"] as never, { + error: undefined, + instanceId: nextInstanceId, + status: "running", + updatedAt: now, + ...(args.usage === undefined ? {} : { usage: args.usage }), + }); + + return { dequeued: nextInstanceId }; + }); + /** * Replace the thread's synced state (the `ctx.setState` target). INTERNAL — * only the workflow's admin-dispatch may call it, from inside a tool's @@ -687,6 +909,7 @@ export const agentComponent = (): AgentComponent => { extension: agentExtension, functions: { agentAppendMessage: asInternal(agentAppendMessage), + agentCompleteRun: asInternal(agentCompleteRun), agentEnsureThread: asInternal(agentEnsureThread), agentEpisodeRecall: episodic.agentEpisodeRecall, agentEpisodeUpsert: episodic.agentEpisodeUpsert, diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 1db36056fb..802fb5be53 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -41,11 +41,14 @@ export type { AgentMessageRow, AgentMessageStatus, AgentModelInput, + AgentOnReply, AgentOnStepFinish, AgentPrepareStep, AgentPrepareStepInput, AgentPrepareStepResult, AgentProgressEvent, + AgentReplyInfo, + AgentReplyRef, AgentRunFunction, AgentRunHandle, AgentRunInput, diff --git a/packages/agent/src/paths.ts b/packages/agent/src/paths.ts index 4657dba89e..1c825283b7 100644 --- a/packages/agent/src/paths.ts +++ b/packages/agent/src/paths.ts @@ -29,6 +29,7 @@ export const SANDBOX_INVOKE_PATH: "sandbox:invoke" = `${SANDBOX_MODULE}:invoke`; */ export const DEFAULT_AGENT_FUNCTION_PATHS: AgentFunctionPaths = { appendMessage: `${AGENT_MODULE}:agentAppendMessage`, + completeRun: `${AGENT_MODULE}:agentCompleteRun`, ensureThread: `${AGENT_MODULE}:agentEnsureThread`, episodeRecall: `${AGENT_MODULE}:agentEpisodeRecall`, episodeUpsert: `${AGENT_MODULE}:agentEpisodeUpsert`, diff --git a/packages/agent/src/reply.prototype.ts b/packages/agent/src/reply.prototype.ts deleted file mode 100644 index bf0d779622..0000000000 --- a/packages/agent/src/reply.prototype.ts +++ /dev/null @@ -1,92 +0,0 @@ -/** - * DESIGN-SPIKE PROTOTYPE — see `plans/242-agent-reply-design.md`. - * - * NOT wired into `component.ts`, `agent-loop.ts`, `inbound.ts`, `channels.ts`, - * or `types.ts`, and NOT exported from `./index`. This proves out the - * capture -> run -> threaded-reply round trip for the ONE channel (email) - * this spike prototypes; production wiring (if this direction is ratified) - * belongs in `types.ts` (`AgentReplyRef`, `AgentEmailRun.replyRef`) and - * `agent-loop.ts` (an `onReply` hook on the terminal turn), not here. - * - * Deliberately reuses the REAL `@lunora/mail` `Mailer` contract rather than - * a hand-rolled stand-in, so the prototype exercises the package's actual - * address/header validation — a fake mail transport only replaces the final - * network hop. - * @experimental - */ -import type { Mailer } from "@lunora/mail"; -import type { InboundEmail } from "@lunora/mail/inbound"; - -/** - * The email variant of the design doc's `AgentReplyRef` union — captured at - * `onEmail` mapper time from fields `@lunora/mail/inbound`'s - * `parseInboundEmail` already produced, not re-parsed from anything raw. - */ -interface EmailReplyRef { - channel: "email"; - /** Who to reply TO — the original sender. */ - from: string; - /** Becomes the reply's `In-Reply-To` header. */ - messageId: string; - /** The inbound message's own `References` chain, if any — appended to, not replaced. */ - references?: string; - /** Who the reply comes FROM — the original recipient(s) (mailbox that received it). */ - to: string[]; -} - -/** - * Capture a reply reference from an already-parsed inbound email. Returns - * `undefined` when the message carries no `Message-ID` — there is nothing to - * thread a reply against, so (mirroring plan 240's honest-reject-over- - * silent-misbehavior stance) the caller should decline to reply rather than - * send an unthreaded one. - */ -const captureEmailReplyRef = (email: InboundEmail): EmailReplyRef | undefined => { - if (email.messageId === undefined) { - return undefined; - } - - return { - channel: "email", - from: email.from, - messageId: email.messageId, - to: email.to, - ...(email.references === undefined ? {} : { references: email.references }), - }; -}; - -/** RFC 5322 threading: append the replied-to id to any existing `References` chain. */ -const buildReferences = (replyRef: EmailReplyRef): string => { - if (replyRef.references === undefined) { - return replyRef.messageId; - } - - return `${replyRef.references} ${replyRef.messageId}`; -}; - -interface ReplyBody { - subject: string; - text: string; -} - -/** - * Send a threaded reply for a captured email ref through a REAL `Mailer`. - * Standing in for what an `onReply` hook (design doc, not implemented here) - * would call once a run's final answer is ready. - * - * `replyRef.to` (the mailbox the inbound message was addressed to) is - * captured but unused here — this minimal prototype always sends from the - * mailer's configured default `from`. A multi-alias app that wants the reply - * to come from the SAME address that received it would pass `from: - * replyRef.to[0]` explicitly; that's a real-wiring decision, not a spike one. - */ -const replyToEmail = async (mailer: Mailer, replyRef: EmailReplyRef, body: ReplyBody): Promise<{ id: string }> => - mailer.send({ - headers: { "In-Reply-To": replyRef.messageId, References: buildReferences(replyRef) }, - subject: body.subject, - text: body.text, - to: replyRef.from, - }); - -export { buildReferences, captureEmailReplyRef, replyToEmail }; -export type { EmailReplyRef, ReplyBody }; diff --git a/packages/agent/src/reply.ts b/packages/agent/src/reply.ts new file mode 100644 index 0000000000..5e4162116e --- /dev/null +++ b/packages/agent/src/reply.ts @@ -0,0 +1,79 @@ +/** + * The outbound half of an inbound trigger: capture where a message came from, + * then answer in the same thread. + * + * Inbound is fully built (`./inbound` for email, `./channels` for + * Slack/GitHub/Discord) and hands every mapper a verified, parsed event. What + * this module adds is the small amount of channel knowledge needed to turn that + * event into an {@link AgentReplyRef} and, for email, to send a correctly + * threaded reply back. + * + * Only email ships a send helper. Slack, GitHub, and Discord need an OUTBOUND + * credential (a bot token, an installation token) and the framework has no + * store for one — `AgentInboundChannel.secret` is for verifying inbound + * signatures, not for authenticating outbound calls. So for those channels + * `onReply` receives `env` and the app makes the provider call with its own + * token. Inventing a credential store to hide three fetch calls would be the + * wrong trade. + */ +import type { Mailer } from "@lunora/mail"; +import type { InboundEmail } from "@lunora/mail/inbound"; + +import type { AgentReplyRef } from "./types"; + +/** The email arm of {@link AgentReplyRef}, narrowed for the helpers below. */ +type EmailReplyRef = Extract; + +/** + * Capture a reply reference from an already-parsed inbound email — call it in + * an `onEmail` mapper and put the result on the run's `replyRef`. + * + * Returns `undefined` when the message carries no `Message-ID`: there is + * nothing to thread a reply against, and an unthreaded reply lands as a new + * conversation in the recipient's client, which is worse than not replying. + */ +export const captureEmailReplyRef = (email: InboundEmail): EmailReplyRef | undefined => { + if (email.messageId === undefined) { + return undefined; + } + + return { + channel: "email", + from: email.from, + messageId: email.messageId, + to: email.to, + ...(email.references === undefined ? {} : { references: email.references }), + }; +}; + +/** RFC 5322 threading: append the replied-to id to any existing `References` chain. */ +export const buildReferences = (replyRef: EmailReplyRef): string => + (replyRef.references === undefined ? replyRef.messageId : `${replyRef.references} ${replyRef.messageId}`); + +/** Subject + body of a reply. */ +export interface ReplyBody { + subject: string; + text: string; +} + +/** + * Send a threaded reply through the app's own `@lunora/mail` mailer. + * + * The mailer is a parameter rather than something this module constructs: the + * app already configures one (API key or `send_email` binding), and passing it + * in keeps this module free of any runtime dependency on `@lunora/mail`. + * + * The reply goes to the original sender, from the mailer's configured default. + * An app running several aliases that wants the reply to come from whichever + * mailbox received the message passes `from: replyRef.to[0]` itself — a + * decision only the app can make. + */ +export const replyToEmail = async (mailer: Mailer, replyRef: EmailReplyRef, body: ReplyBody): Promise<{ id: string }> => + mailer.send({ + headers: { "In-Reply-To": replyRef.messageId, References: buildReferences(replyRef) }, + subject: body.subject, + text: body.text, + to: replyRef.from, + }); + +export type { EmailReplyRef }; diff --git a/packages/agent/src/run-queue.prototype.ts b/packages/agent/src/run-queue.prototype.ts deleted file mode 100644 index 170b58bd08..0000000000 --- a/packages/agent/src/run-queue.prototype.ts +++ /dev/null @@ -1,248 +0,0 @@ -/** - * DESIGN-SPIKE PROTOTYPE — see `plans/240-agent-run-queue-design.md`. - * - * NOT wired into `component.ts`, `agent-loop.ts`, or any trigger path, and - * NOT exported from `./index`. This proves out the ordering + seq-claim - * correctness of a durable per-thread run queue for - * `defineAgent({ onConcurrentRun: "queue" })` — today that value silently - * degrades to `"reject"` (`component.ts`'s `agentEnsureThread`). It is a - * throwaway artifact of the spike, not a shipped feature; production wiring - * (if this direction is ratified) belongs in `component.ts` alongside the - * real `agentEnsureThread`/`agentPatchThread` mutations, not here. - * - * Mirrors just enough of `agentEnsureThread`'s existing reject/replace logic - * to add a third, `"queue"` branch, plus the completion-side dequeue that - * `agent-loop.ts` would call instead of its current terminal `patchThread`. - * Both mutations assume the same single-threaded-per-DO execution model the - * real component relies on — see the design doc's "Ordering + seq claim" - * section for why that's what makes the handoff race-free. - * @experimental - */ - -/** Bounded queue depth — see design doc open question #1 (no config surface yet). */ -const MAX_QUEUE_DEPTH = 5; - -/** Minimal `ctx.db` shape this prototype needs — same shape `component.ts`'s mutations use. */ -interface PrototypeRow extends Record { - _id: string; -} - -interface PrototypeIndexQuery { - collect: () => Promise; - first: () => Promise; - order: (direction: "asc" | "desc") => { collect: () => Promise }; -} - -interface PrototypeIndexBuilder { - eq: (field: string, value: unknown) => PrototypeIndexBuilder; -} - -interface PrototypeDatabase { - insert: (table: string, document: Record) => Promise; - patch: (id: string, patch: Record) => Promise; - query: (table: string) => { withIndex: (name: string, build: (q: PrototypeIndexBuilder) => unknown) => PrototypeIndexQuery }; - remove: (id: string) => Promise; -} - -const THREADS_TABLE = "proto_agent_threads"; -const QUEUE_TABLE = "proto_agent_run_queue"; - -interface EnsureThreadOrQueueArgs { - agent: string; - instanceId?: string; - key: string; - onConcurrentRun?: "queue" | "reject" | "replace"; -} - -type EnsureThreadOrQueueResult = - { created: true } | { created: false } | { created: false; priorInstanceId: string; replaced: true } | { created: false; position: number; queued: true }; - -/** Thrown in place of the real `LunoraError("CONFLICT", ...)` — message-compatible enough for tests to pattern-match. */ -class QueueConflictError extends Error {} - -/** - * The `"queue"` branch, split out of `ensureThreadOrQueue` to keep that - * function's branching within the repo's cognitive-complexity budget. Called - * only once `isConcurrentRun` is already known true and `policy === "queue"`. - */ -const enqueueOrThrow = async ( - database: PrototypeDatabase, - args: EnsureThreadOrQueueArgs & { instanceId: string }, - existing: PrototypeRow, - now: number, -): Promise => { - // Idempotent enqueue: a replay of a still-parked run's bootstrap must - // return its existing position, not create a duplicate row. - const alreadyQueued = await database - .query(QUEUE_TABLE) - .withIndex("byThreadInstance", (q) => q.eq("threadKey", args.key).eq("instanceId", args.instanceId)) - .first(); - - if (alreadyQueued) { - return { created: false, position: alreadyQueued["position"] as number, queued: true }; - } - - const queuedRows = await database - .query(QUEUE_TABLE) - .withIndex("byThread", (q) => q.eq("threadKey", args.key)) - .collect(); - - if (queuedRows.length >= MAX_QUEUE_DEPTH) { - throw new QueueConflictError( - `@lunora/agent: thread "${args.key}" run queue is full (depth ${String(MAX_QUEUE_DEPTH)}) — rejecting instance "${args.instanceId}"`, - ); - } - - const position = existing["nextPosition"] as number; - - await database.patch(existing["_id"], { nextPosition: position + 1 }); - await database.insert(QUEUE_TABLE, { - agent: args.agent, - enqueuedAt: now, - instanceId: args.instanceId, - position, - threadKey: args.key, - }); - - return { created: false, position, queued: true }; -}; - -/** - * Mirrors `component.ts`'s `agentEnsureThread`, with the `"queue"` branch - * implemented instead of degraded to reject. See the design doc for why - * idempotent-by-`(threadKey, instanceId)` enqueue is required (a workflow - * replay of a still-parked run re-executes this mutation for real, since - * `ensureThread` — like the production one — runs outside `step.do`). - */ -const ensureThreadOrQueue = async (database: PrototypeDatabase, args: EnsureThreadOrQueueArgs, now = Date.now()): Promise => { - const existing = await database - .query(THREADS_TABLE) - .withIndex("byKey", (q) => q.eq("key", args.key)) - .first(); - - if (!existing) { - await database.insert(THREADS_TABLE, { - agent: args.agent, - key: args.key, - nextPosition: 0, - status: "running", - updatedAt: now, - ...(args.instanceId === undefined ? {} : { instanceId: args.instanceId }), - }); - - return { created: true }; - } - - const priorInstanceId = existing["instanceId"] as string | undefined; - const isConcurrentRun = - (existing["status"] === "running" || existing["status"] === "awaiting_input") && - priorInstanceId !== undefined && - (args.instanceId === undefined || args.instanceId !== priorInstanceId); - - if (!isConcurrentRun) { - // Replay (same instance) or a resumed idle/errored/cancelled thread. - await database.patch(existing["_id"], { - status: "running", - updatedAt: now, - ...(args.instanceId === undefined ? {} : { instanceId: args.instanceId }), - }); - - return { created: false }; - } - - const policy = args.onConcurrentRun ?? "reject"; - - if (policy === "replace") { - await database.patch(existing["_id"], { - status: "running", - updatedAt: now, - ...(args.instanceId === undefined ? {} : { instanceId: args.instanceId }), - }); - - return { created: false, priorInstanceId, replaced: true }; - } - - if (policy === "queue") { - if (args.instanceId === undefined) { - // The id-less inbound-email/inbound-channel dispatch path can't be - // told apart from anyone else later, so it can't be parked and - // dequeued by instance id — same honest-reject fallback as an - // overflowing queue. - throw new QueueConflictError( - `@lunora/agent: thread "${args.key}" already has a run in flight (instance "${priorInstanceId}") — cannot queue an id-less dispatch`, - ); - } - - return enqueueOrThrow(database, { ...args, instanceId: args.instanceId }, existing, now); - } - - // "reject" (default) - throw new QueueConflictError( - `@lunora/agent: thread "${args.key}" already has a run in flight (instance "${priorInstanceId}") — onConcurrentRun="${policy}"`, - ); -}; - -interface CompleteRunArgs { - instanceId: string; - key: string; - terminalStatus: "cancelled" | "error" | "idle"; -} - -type CompleteRunResult = { dequeued: undefined } | { dequeued: string }; - -/** - * Mirrors what `agent-loop.ts`'s terminal `patchThread` call would become: - * instead of unconditionally setting the terminal status, check the queue - * first and — inside this SAME mutation — hand the thread straight to the - * next parked instance if one is waiting. See the design doc for why doing - * both in one mutation is what makes the handoff race-free (no window where - * a third caller can observe an ownerless thread). - * - * Idempotent under a replay of the FINISHING run's own completion: guarded on - * `thread.instanceId === args.instanceId` first, so a second call (after - * ownership has already moved to the dequeued instance) is a no-op rather - * than dequeuing a second entry and skipping someone's turn. - */ -const completeRunAndDequeue = async (database: PrototypeDatabase, args: CompleteRunArgs, now = Date.now()): Promise => { - const thread = await database - .query(THREADS_TABLE) - .withIndex("byKey", (q) => q.eq("key", args.key)) - .first(); - - if (thread?.["instanceId"] !== args.instanceId) { - // Already handed off (a replay of this same completion) or unknown — - // either way, not this call's job to touch it again. - return { dequeued: undefined }; - } - - const queuedRows = await database - .query(QUEUE_TABLE) - .withIndex("byThread", (q) => q.eq("threadKey", args.key)) - .order("asc") - .collect(); - - const head = queuedRows[0]; - - if (!head) { - await database.patch(thread["_id"], { status: args.terminalStatus, updatedAt: now }); - - return { dequeued: undefined }; - } - - await database.remove(head["_id"]); - await database.patch(thread["_id"], { instanceId: head["instanceId"], status: "running", updatedAt: now }); - - return { dequeued: head["instanceId"] as string }; -}; - -export { completeRunAndDequeue, ensureThreadOrQueue, MAX_QUEUE_DEPTH, QueueConflictError }; -export type { - CompleteRunArgs, - CompleteRunResult, - EnsureThreadOrQueueArgs, - EnsureThreadOrQueueResult, - PrototypeDatabase, - PrototypeIndexBuilder, - PrototypeIndexQuery, - PrototypeRow, -}; diff --git a/packages/agent/src/types.ts b/packages/agent/src/types.ts index 3f1329a0be..20f52ea6a4 100644 --- a/packages/agent/src/types.ts +++ b/packages/agent/src/types.ts @@ -411,6 +411,40 @@ export interface AgentStepFinishInfo { */ export type AgentOnStepFinish = (info: AgentStepFinishInfo) => Promise | void; +/** + * What {@link AgentConfig.onReply} is called with when a triggered run reaches + * its final answer. + * @experimental + */ +export interface AgentReplyInfo { + /** The worker `env` — where an outbound credential (a bot token, a mailer binding) lives. */ + env: Record; + /** Where to reply, captured by the mapper that started this run. */ + replyRef: AgentReplyRef; + /** The run's final answer. */ + result: AgentRunResult; + /** The thread this run belongs to. */ + threadKey: string; +} + +/** + * Send a triggered run's answer back where it came from — the outbound half of + * an `onEmail` / `onInbound` trigger. + * + * Called once, automatically, when a run started with a `replyRef` produces a + * final answer; a run without one (an ordinary in-app `ctx.agents..run`) + * never calls it. It runs inside a named durable step, so a transient failure + * is retried and a workflow replay does not send the answer twice. + * + * Delivery is the app's call, because only the app has the credential: email + * has a home already (`replyToEmail(mailer, ref, body)` from + * `@lunora/agent/reply`), while Slack/GitHub/Discord bot tokens have no + * framework-owned store yet — read them from `info.env` and make the provider + * call. + * @experimental + */ +export type AgentOnReply = (info: AgentReplyInfo) => Promise | void; + /** * The input {@link AgentConfig.prepareStep} sees before a turn runs. * @experimental @@ -455,6 +489,25 @@ export type AgentPrepareStep = (input: AgentPrepareStepInput) => AgentPrepareSte * start a run from an inbound email. * @experimental */ + +/** + * Where a triggered run should send its answer back to — captured by the + * mapper, from fields the verified inbound payload already carries, and carried + * through the run so {@link AgentConfig.onReply} can answer in the same place + * the question was asked. + * + * A discriminated union rather than an opaque blob because each channel threads + * differently: email by RFC 5322 `In-Reply-To`/`References`, Slack by + * `thread_ts`, GitHub by issue number, Discord by channel (plus the triggering + * message for a true reply). + * @experimental + */ +export type AgentReplyRef = + | { channel: "discord"; channelId: string; messageId?: string } + | { channel: "email"; from: string; messageId: string; references?: string; to: string[] } + | { channel: "github"; commentId?: number; issueNumber: number; owner: string; repo: string } + | { channel: "slack"; channelId: string; threadTs: string }; + export interface AgentEmailRun { /** The user message that starts (or continues) the thread — the model's prompt. */ input: string; @@ -466,6 +519,14 @@ export interface AgentEmailRun { * `email.from`. */ owner?: string; + + /** + * Where to send the answer — see {@link AgentReplyRef}. Populate it from the + * event the mapper already has (`captureEmailReplyRef(email)` does it for + * email) and the run calls {@link AgentConfig.onReply} with it once the + * final answer is ready. Omit it and the run simply never replies. + */ + replyRef?: AgentReplyRef; /** The thread key — reuse to continue a conversation (e.g. a ticket id parsed from the subject). */ threadKey: string; /** Optional thread title, set on first creation. */ @@ -609,7 +670,16 @@ export interface AgentConfig { * * - `"reject"` (default) — fail the new run fast with a `CONFLICT` error. * - `"replace"` — terminate the in-flight instance and take the thread over. - * - `"queue"` — reserved for a future durable queue; currently degrades to `"reject"` (no queue exists yet), tracked as a follow-up. + * - `"queue"` — park the new run behind the one in flight (FIFO, up to five deep) and hibernate it until the thread is handed over. + * + * Each parked run is a live workflow instance waiting on an event, so the + * queue's depth cap is a real resource bound: past it, a start is rejected + * exactly as `"reject"` would. A `"replace"` arriving later supersedes the + * run in FLIGHT, not the queue — parked runs still take their turn after it. + * + * A dispatch with no instance id (the inbound-email / inbound-channel + * paths) cannot be parked, because nothing later can tell it apart from + * another such dispatch to wake it; `"queue"` rejects those. * * A workflow REPLAY re-enters the bootstrap under the SAME instance id and * is never a concurrent run (the guard compares the stored instance id). @@ -631,6 +701,13 @@ export interface AgentConfig { * signature over the raw body before calling `map`. */ onInbound?: AgentInboundChannel; + + /** + * Called with the final answer when the run was triggered from a channel + * that gave it a `replyRef` — see {@link AgentOnReply}. This is how an + * inbound-triggered agent answers where it was asked. + */ + onReply?: AgentOnReply; /** Called after each LLM turn — see {@link AgentOnStepFinish}. */ onStepFinish?: AgentOnStepFinish; @@ -831,6 +908,8 @@ export interface AgentRunInput { * the first run. */ owner?: string; + /** Where a triggered run sends its answer — see {@link AgentReplyRef}. */ + replyRef?: AgentReplyRef; /** The thread key — reuse to continue a conversation. */ threadKey: string; /** Optional thread title, set on first creation. */ @@ -866,6 +945,13 @@ export interface AgentRunResult { */ export interface AgentFunctionPaths { appendMessage: string; + + /** + * The internal `agents:agentCompleteRun` mutation the loop dispatches at the + * end of a run: it writes the terminal status AND hands the thread to the + * next queued run in the same mutation. + */ + completeRun: string; ensureThread: string; /** The internal `agents:agentEpisodeRecall` query the loop dispatches for an episodic-kind read. */ episodeRecall: string; diff --git a/packages/ai/__tests__/rag/sync.test.ts b/packages/ai/__tests__/rag/sync.test.ts new file mode 100644 index 0000000000..a875ae2f71 --- /dev/null +++ b/packages/ai/__tests__/rag/sync.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; + +import { ragSyncTriggers } from "../../src/rag/sync"; + +const ACTION = { __lunoraRef: "docs:reindex" }; + +/** Records what the trigger scheduled, standing in for `ctx.scheduler`. */ +const fakeContext = () => { + const scheduled: { args?: Record; delayMs: number }[] = []; + + return { + context: { + scheduler: { + runAfter: async (delayMs: number, _target: unknown, args?: Record) => { + scheduled.push({ args, delayMs }); + + return "job-1"; + }, + }, + }, + scheduled, + }; +}; + +describe("ragSyncTriggers", () => { + it("schedules a re-index for an inserted row", async () => { + expect.assertions(1); + + const { context, scheduled } = fakeContext(); + const sync = ragSyncTriggers({ action: ACTION, text: (document) => document["body"] as string }); + + await sync.afterInsert(context, { doc: { body: "hello" }, id: "doc-1" }); + + expect(scheduled).toStrictEqual([{ args: { id: "doc-1", text: "hello" }, delayMs: 0 }]); + }); + + it("skips an update that did not touch the indexed text", async () => { + expect.assertions(1); + + const { context, scheduled } = fakeContext(); + const sync = ragSyncTriggers({ action: ACTION, text: (document) => document["body"] as string }); + + // A title-only edit: the embedding would be byte-identical, so the whole + // dispatch is skipped rather than relying on the content-hash no-op. + await sync.afterUpdate(context, { doc: { body: "hello", title: "new" }, id: "doc-1", previous: { body: "hello", title: "old" } }); + + expect(scheduled).toStrictEqual([]); + }); + + it("removes the chunks when the indexed text goes away", async () => { + expect.assertions(2); + + const { context, scheduled } = fakeContext(); + const sync = ragSyncTriggers({ action: ACTION, text: (document) => document["body"] as string | undefined }); + + await sync.afterUpdate(context, { doc: { title: "kept" }, id: "doc-1", previous: { body: "hello" } }); + await sync.afterDelete(context, { id: "doc-2", previous: { body: "gone" } }); + + expect(scheduled[0]?.args).toStrictEqual({ deleted: true, id: "doc-1" }); + expect(scheduled[1]?.args).toStrictEqual({ deleted: true, id: "doc-2" }); + }); + + it("honours a custom source id and delay", async () => { + expect.assertions(1); + + const { context, scheduled } = fakeContext(); + const sync = ragSyncTriggers({ + action: ACTION, + delayMs: 500, + id: (document) => `slug:${String(document["slug"])}`, + text: (document) => document["body"] as string, + }); + + await sync.afterInsert(context, { doc: { body: "hi", slug: "intro" }, id: "row-9" }); + + expect(scheduled).toStrictEqual([{ args: { id: "slug:intro", text: "hi" }, delayMs: 500 }]); + }); +}); diff --git a/packages/ai/src/rag/index.ts b/packages/ai/src/rag/index.ts index 904c5d7027..3b36efec57 100644 --- a/packages/ai/src/rag/index.ts +++ b/packages/ai/src/rag/index.ts @@ -3,6 +3,8 @@ export { default as defineRag } from "./define-rag"; export { contentHash, guessMimeTypeFromExtension } from "./helpers"; export { default as hybridRank } from "./hybrid-rank"; export { default as bm25LexicalStore } from "./lexical-store"; +export type { RagSyncArgs, RagSyncOptions } from "./sync"; +export { ragSyncTriggers } from "./sync"; export type { IndexInput, IndexResult, diff --git a/packages/ai/src/rag/sync.ts b/packages/ai/src/rag/sync.ts new file mode 100644 index 0000000000..43a2be8951 --- /dev/null +++ b/packages/ai/src/rag/sync.ts @@ -0,0 +1,145 @@ +/** + * Keep a RAG index in step with a table, so the table IS the index. + * + * `rag.index(...)` is a manual call, which means an app that edits a document + * has to remember to re-index it — and a forgotten call is invisible: retrieval + * keeps answering, just from stale text. The fix is to hang the re-index off the + * write itself. + * + * Embedding is network I/O, so it cannot run inside the mutation that wrote the + * row. The bridge is the two seams that already exist: a table `.triggers()` + * handler runs in the write path and can `ctx.scheduler.runAfter(...)`, so the + * trigger records the intent and an internal ACTION does the embedding a moment + * later. That is what {@link ragSyncTriggers} wires up. + * + * Re-indexing unchanged text is already cheap — `rag.index` short-circuits on a + * content hash — but this skips scheduling entirely when an update didn't touch + * the indexed text, so an unrelated column edit costs nothing at all. + */ + +/** Structural slice of `ctx.scheduler` — enough to defer the re-index. */ +interface RagSyncScheduler { + runAfter: (delayMs: number, target: unknown, args?: Record) => Promise; +} + +/** Structural slice of the `TriggerCtx` a `.triggers()` handler receives. */ +interface RagSyncTriggerContext { + readonly scheduler: RagSyncScheduler; +} + +/** The trigger events this helper handles, narrowed to what it reads. */ +interface RagSyncEvent { + readonly doc?: Record; + readonly id: string; + readonly previous?: Record; +} + +/** What the scheduled action receives — one document to re-index, or one to drop. */ +interface RagSyncArgs extends Record { + /** `true` when the source row was deleted: call `rag.remove({ id })`. */ + deleted?: boolean; + /** The source id — the same `id` you pass to `rag.index`/`rag.remove`. */ + id: string; + /** The text to embed. Absent on a delete. */ + text?: string; +} + +interface RagSyncOptions = Record> { + /** + * The internal action to dispatch — it receives {@link RagSyncArgs} and calls + * `rag.index` / `rag.remove`. An action, not a mutation: embedding is network + * I/O and never runs in the deterministic write path. + */ + action: unknown; + + /** + * How long to wait before re-indexing. A small delay coalesces nothing by + * itself, but it keeps the embed off the write's own tail latency. Default + * `0` — as soon as the mutation commits. + */ + delayMs?: number; + /** The source id to index under. Defaults to the row's own id. */ + id?: (document: Document) => string; + /** The text to embed. Return `undefined` to skip the row (a draft, an empty body). */ + text: (document: Document) => string | undefined; +} + +/** One trigger definition, structurally — matches what `.triggers((t) => …)` returns. */ +type RagSyncHandler = (context: RagSyncTriggerContext, event: RagSyncEvent) => Promise; + +/** + * Build the three write-path handlers that keep a RAG index in step with a + * table. Wire them into the table's `.triggers()`: + * + * ```ts + * const sync = ragSyncTriggers({ action: internal.docs.reindex, text: (doc) => doc.body }); + * + * export const schema = defineSchema({ + * docs: defineTable({ body: v.string(), title: v.string() }).triggers((t) => ({ + * ragDelete: t.afterDelete(sync.afterDelete), + * ragInsert: t.afterInsert(sync.afterInsert), + * ragUpdate: t.afterUpdate(sync.afterUpdate), + * })), + * }); + * ``` + * + * The action on the other end is three lines: + * + * ```ts + * export const reindex = internalAction.input({ deleted: v.optional(v.boolean()), id: v.string(), text: v.optional(v.string()) }).action( + * async ({ args, ctx }) => { + * const rag = docsRag(ctx); + * + * await (args.deleted === true || args.text === undefined ? rag.remove({ id: args.id }) : rag.index({ id: args.id, text: args.text })); + * }, + * ); + * ``` + */ +const ragSyncTriggers = = Record>( + options: RagSyncOptions, +): { afterDelete: RagSyncHandler; afterInsert: RagSyncHandler; afterUpdate: RagSyncHandler } => { + const delayMs = options.delayMs ?? 0; + const sourceId = (document: Record, fallback: string): string => (options.id === undefined ? fallback : options.id(document as Document)); + const textOf = (document: Record | undefined): string | undefined => + document === undefined ? undefined : options.text(document as Document); + + const schedule = async (context: RagSyncTriggerContext, args: RagSyncArgs): Promise => { + await context.scheduler.runAfter(delayMs, options.action, args); + }; + + return { + afterDelete: async (context, event) => { + const document = event.previous ?? event.doc; + + await schedule(context, { deleted: true, id: document === undefined ? event.id : sourceId(document, event.id) }); + }, + afterInsert: async (context, event) => { + const text = textOf(event.doc); + + if (text === undefined || event.doc === undefined) { + return; + } + + await schedule(context, { id: sourceId(event.doc, event.id), text }); + }, + afterUpdate: async (context, event) => { + const text = textOf(event.doc); + const before = textOf(event.previous); + + // An edit that didn't touch the indexed text costs nothing: no action + // dispatch, no embedding call, no vector write. + if (text === before || event.doc === undefined) { + return; + } + + const id = sourceId(event.doc, event.id); + + // The text went away (cleared, or the row no longer qualifies): the + // old chunks must go too, or retrieval keeps serving deleted content. + await (text === undefined ? schedule(context, { deleted: true, id }) : schedule(context, { id, text })); + }, + }; +}; + +export type { RagSyncArgs, RagSyncOptions }; +export { ragSyncTriggers }; diff --git a/packages/client/src/lunora-client.ts b/packages/client/src/lunora-client.ts index e4d17ceba9..b9ea1a7a32 100644 --- a/packages/client/src/lunora-client.ts +++ b/packages/client/src/lunora-client.ts @@ -956,7 +956,10 @@ class LunoraClient { * tells us which socket to push the cancel frame onto when the consumer * calls `.cancel()` or the iterator is garbage-collected. */ - private readonly streams = new Map(); + private readonly streams = new Map< + string, + { durable: boolean; handle: StreamHandle; lastSeq: number; message: ClientMessage; shardKey: string | undefined } + >(); /** Live shape subscriptions (partial replication), keyed by their wire id. */ private readonly shapeSubscriptions = new Map(); @@ -3258,7 +3261,7 @@ class LunoraClient { public stream>( function_: F, args: ArgsOf, - options: { maxBuffer?: number; shardKey?: string } = {}, + options: { durable?: boolean; maxBuffer?: number; shardKey?: string } = {}, ): StreamIterable> { this.assertOpen(); @@ -3287,10 +3290,6 @@ class LunoraClient { }, }); - // Record before sending so an immediate ack/chunk reaching the dispatch - // path before we return finds its target. - this.streams.set(id, { handle: handle as StreamHandle, shardKey }); - this.ensureSocket(shardKey); const conn = this.getConnection(shardKey); @@ -3307,6 +3306,12 @@ class LunoraClient { type: "stream", }; + // Record before sending so an immediate ack/chunk reaching the dispatch + // path before we return finds its target. The start frame is kept so a + // durable stream can be re-sent verbatim (plus its resume watermark) + // when the socket comes back. + this.streams.set(id, { durable: options.durable === true, handle: handle as StreamHandle, lastSeq: 0, message, shardKey }); + // Fast path: socket is open, try to send immediately. `sendOn` can still // return `false` if the socket closed between the `wsState` check and the // `.send()` call (or `.send()` threw) — in that race the frame was never @@ -4830,20 +4835,39 @@ class LunoraClient { this.emitConnectionStatus(); this.markShardPendingAck(conn.shardKey); - // Fail every in-flight stream bound to this shard. A stream whose start - // frame was already sent lost its server-side iterator when the socket - // dropped and can't resume, so a consumer's `for await` would otherwise - // hang forever on a next() that never settles (streams are failed on - // close()/error/overflow but a socket bounce is the common termination). + // Settle every in-flight stream bound to this shard. An EPHEMERAL stream + // whose start frame was already sent lost its server-side iterator when + // the socket dropped and can't resume, so a consumer's `for await` would + // otherwise hang forever on a next() that never settles (streams are + // failed on close()/error/overflow but a socket bounce is the common + // termination). A DURABLE stream is the opposite case: the run survives + // the socket, so it is re-queued with a resume watermark instead. // Stream-start frames still queued in `pendingStreams` were never sent, // so they legitimately ride the next reconnect — exclude those ids. const pendingStreamIds = new Set((conn.pendingStreams ?? []).map((message) => (message as { id?: string }).id)); for (const [id, stream] of this.streams) { - if (connectionKey(stream.shardKey) === connectionKey(conn.shardKey) && !pendingStreamIds.has(id)) { - stream.handle.fail(new LunoraError("STREAM_DISCONNECTED", "stream terminated: WebSocket disconnected")); - this.streams.delete(id); + if (connectionKey(stream.shardKey) !== connectionKey(conn.shardKey) || pendingStreamIds.has(id)) { + continue; } + + if (stream.durable) { + // A durable run outlives the socket: the server keeps producing + // and persists every chunk, so re-send the start frame carrying + // the last seq we saw and the reconnect replays the gap. The + // consumer's `for await` never observes the interruption. + const resume = { ...(stream.message as { id: string; type: "stream" }), sinceChunk: stream.lastSeq } as ClientMessage; + + stream.message = resume; + // Replace, don't append: repeated bounces before the socket comes + // back would otherwise stack one frame per attempt for the same id. + conn.pendingStreams = [...(conn.pendingStreams ?? []).filter((queued) => (queued as { id?: string }).id !== id), resume]; + + continue; + } + + stream.handle.fail(new LunoraError("STREAM_DISCONNECTED", "stream terminated: WebSocket disconnected")); + this.streams.delete(id); } if (this.WebSocketImpl === undefined) { @@ -5021,9 +5045,13 @@ class LunoraClient { return; } case "chunk": { - const { data, id } = message; + const { data, id, seq } = message; const stream = this.streams.get(id); + if (stream && typeof seq === "number") { + stream.lastSeq = seq; + } + stream?.handle.push(decodeWire(data)); return; diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 1abd3841be..550acd7ea6 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -578,6 +578,18 @@ export interface ClientAckMessage { export interface ClientStreamMessage { id: string; query: { args?: Record; functionPath: string; shardKey?: string }; + + /** + * Resume watermark: the highest chunk `seq` this client already received. + * Only meaningful for a stream the server declared `durable` — the run + * replays everything after it and then continues live, which is what turns + * a reconnect into a resume instead of a lost generation. Omitted on a + * first attach. + * + * Named `sinceChunk`, not `sinceSeq`, because a subscribe envelope already + * carries a `query.sinceSeq` meaning the CDC cursor. + */ + sinceChunk?: number; type: "stream"; } @@ -700,6 +712,14 @@ export interface ServerCompleteMessage { export interface ServerChunkMessage { data: unknown; id: string; + + /** + * Monotonic position of this chunk within a **durable** run, starting at 1. + * The client stores the last one it saw and replays it as + * {@link ClientStreamMessage.sinceChunk} when the socket comes back. Absent + * on an ephemeral stream, which has nothing to resume from. + */ + seq?: number; type: "chunk"; } diff --git a/packages/codegen/__tests__/discover-agents.test.ts b/packages/codegen/__tests__/discover-agents.test.ts index 6f1f8cdba3..c32845eeec 100644 --- a/packages/codegen/__tests__/discover-agents.test.ts +++ b/packages/codegen/__tests__/discover-agents.test.ts @@ -554,6 +554,7 @@ describe("auto-registered agent runtime functions", () => { .toSorted((a, b) => a.localeCompare(b)), ).toStrictEqual([ "agentAppendMessage", + "agentCompleteRun", "agentEnsureThread", "agentEpisodeRecall", "agentEpisodeUpsert", diff --git a/packages/codegen/__tests__/fixtures/simple/expected/_generated/shard.ts b/packages/codegen/__tests__/fixtures/simple/expected/_generated/shard.ts index 962df74193..47607b07b6 100644 --- a/packages/codegen/__tests__/fixtures/simple/expected/_generated/shard.ts +++ b/packages/codegen/__tests__/fixtures/simple/expected/_generated/shard.ts @@ -493,7 +493,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -503,6 +503,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/packages/codegen/src/emit.ts b/packages/codegen/src/emit.ts index ba76b02728..ba50c2b148 100644 --- a/packages/codegen/src/emit.ts +++ b/packages/codegen/src/emit.ts @@ -4998,7 +4998,7 @@ ${relationFanout.override} return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -5008,6 +5008,7 @@ ${relationFanout.override} this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/packages/do/__tests__/shard-do.durable-stream.test.ts b/packages/do/__tests__/shard-do.durable-stream.test.ts new file mode 100644 index 0000000000..3ac586df60 --- /dev/null +++ b/packages/do/__tests__/shard-do.durable-stream.test.ts @@ -0,0 +1,243 @@ +/** + * Durable streams: the run outlives the socket that opened it. + * + * These are the three properties the feature exists for — a reload resumes + * mid-run instead of losing the work, a second viewer shares one producer + * instead of paying for a second generation, and a run the DO lost is reported + * rather than silently re-run. + */ +import type { SocketAttachment, SubscriptionEnvelope } from "@lunora/shard-engine"; +import { migrateDurableStreams } from "@lunora/shard-engine"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import type { ShardDOState } from "../src/shard-do"; +import { ShardDO } from "../src/shard-do"; +import createSqliteExec from "./_helpers/node-sqlite"; + +interface FakeWebSocket { + attachment: SocketAttachment | undefined; + send: (data: string) => void; + sent: string[]; + serializeAttachment: (value: unknown) => void; +} + +const createFakeWebSocket = (): FakeWebSocket => {return { + attachment: undefined, + send(data: string) { + this.sent.push(data); + }, + sent: [], + serializeAttachment(value: unknown) { + this.attachment = value as SocketAttachment | undefined; + }, +}}; + +const parseFrames = (ws: FakeWebSocket) => ws.sent.map((raw) => JSON.parse(raw) as Record); + +const waitForTerminator = async (ws: FakeWebSocket, deadlineMs = 500): Promise => { + const start = Date.now(); + + while (Date.now() - start < deadlineMs) { + if (parseFrames(ws).some((frame) => frame.type === "complete" || frame.type === "error")) { + return; + } + + // eslint-disable-next-line no-await-in-loop -- polling loop must wait between frame checks + await new Promise((resolve) => { + setTimeout(resolve, 1); + }); + } +}; + +/** A ShardDO whose streams are all declared durable, with test-supplied generators. */ +class DurableStreamShard extends ShardDO { + public registered = new Map, signal: AbortSignal) => AsyncIterable>(); + + /** How many times a generator was actually constructed — the "one producer" assertion. */ + public starts = 0; + + // eslint-disable-next-line class-methods-use-this -- override stub; these tests never dispatch a plain RPC + public override async handleRpc(): Promise { + return null; + } + + public driveMessage(ws: FakeWebSocket, envelope: SubscriptionEnvelope): Promise { + return this.webSocketMessage(ws as unknown as WebSocket, JSON.stringify(envelope)); + } + + public registerSocket(ws: FakeWebSocket, attachment: SocketAttachment): void { + this.state.acceptWebSocket(ws as unknown as WebSocket); + ws.serializeAttachment(attachment); + } + + protected override executeStream( + functionPath: string, + args: Record, + ): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { + const fn = this.registered.get(functionPath); + + if (!fn) { + return null; + } + + return { + durable: {}, + iterator: (signal) => { + this.starts += 1; + + return fn(args, signal); + }, + }; + } +} + +describe("shardDO durable streams", () => { + let database: ReturnType; + let sockets: FakeWebSocket[]; + let state: ShardDOState; + + beforeEach(() => { + database = createSqliteExec(); + // The real path creates these in the schema migration; these tests drive + // `executeStream` directly, so create them the same way it would. + migrateDurableStreams(database.sql); + + sockets = []; + state = { + acceptWebSocket(ws: WebSocket) { + sockets.push(ws as unknown as FakeWebSocket); + }, + getWebSockets(): WebSocket[] { + return sockets as unknown as WebSocket[]; + }, + id: { name: "shard-durable-stream" }, + storage: { sql: database.sql as unknown as ShardDOState["storage"]["sql"] }, + }; + }); + + afterEach(() => { + database.close(); + }); + + it("replays only the chunks after sinceChunk when a client reattaches", async () => { + expect.assertions(3); + + const shard = new DurableStreamShard(state, {}); + + shard.registered.set("chat:answer", async function* answer() { + yield "the"; + yield " quick"; + yield " fox"; + }); + + const first = createFakeWebSocket(); + + shard.registerSocket(first, { subs: {} }); + await shard.driveMessage(first, { id: "stream_1", query: { functionPath: "chat:answer" }, type: "stream" }); + await waitForTerminator(first); + + // Every chunk carries its position so a client knows what to resume from. + expect(parseFrames(first).filter((frame) => frame.type === "chunk")).toStrictEqual([ + { data: "the", id: "stream_1", seq: 1, type: "chunk" }, + { data: " quick", id: "stream_1", seq: 2, type: "chunk" }, + { data: " fox", id: "stream_1", seq: 3, type: "chunk" }, + ]); + + // A reload: same args, same run, resuming after the first chunk. + const resumed = createFakeWebSocket(); + + shard.registerSocket(resumed, { subs: {} }); + await shard.driveMessage(resumed, { id: "stream_9", query: { functionPath: "chat:answer" }, sinceChunk: 1, type: "stream" }); + await waitForTerminator(resumed); + + const frames = parseFrames(resumed); + + expect(frames.filter((frame) => frame.type === "chunk").map((frame) => frame.data)).toStrictEqual([" quick", " fox"]); + // The transcript was replayed from SQLite — the handler never ran again. + expect(shard.starts).toBe(1); + }); + + it("shares one producer between two clients on the same run", async () => { + expect.assertions(3); + + const shard = new DurableStreamShard(state, {}); + let release = (): void => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + + shard.registered.set("chat:answer", async function* answer() { + yield "first"; + await gate; + yield "second"; + }); + + const a = createFakeWebSocket(); + const b = createFakeWebSocket(); + + shard.registerSocket(a, { subs: {} }); + shard.registerSocket(b, { subs: {} }); + + await shard.driveMessage(a, { id: "stream_a", query: { functionPath: "chat:answer" }, type: "stream" }); + // Let the producer emit its first chunk before the second client attaches. + await new Promise((resolve) => { + setTimeout(resolve, 5); + }); + await shard.driveMessage(b, { id: "stream_b", query: { functionPath: "chat:answer" }, type: "stream" }); + + release(); + await waitForTerminator(a); + await waitForTerminator(b); + + expect(shard.starts).toBe(1); + // The late joiner still sees the whole answer: the prefix is replayed + // from the transcript, the rest arrives live. + expect( + parseFrames(a) + .filter((frame) => frame.type === "chunk") + .map((frame) => frame.data), + ).toStrictEqual(["first", "second"]); + expect( + parseFrames(b) + .filter((frame) => frame.type === "chunk") + .map((frame) => frame.data), + ).toStrictEqual(["first", "second"]); + }); + + it("reports an interrupted run instead of silently re-running the handler", async () => { + expect.assertions(2); + + const shard = new DurableStreamShard(state, {}); + + shard.registered.set("chat:answer", async function* answer() { + yield "half"; + // Never settles: the run is still `running` when the instance dies. + await new Promise(() => {}); + }); + + const first = createFakeWebSocket(); + + shard.registerSocket(first, { subs: {} }); + await shard.driveMessage(first, { id: "stream_1", query: { functionPath: "chat:answer" }, type: "stream" }); + await new Promise((resolve) => { + setTimeout(resolve, 5); + }); + + // A fresh instance over the same storage — the in-memory producer is gone + // but the half-written transcript survived, exactly as after an eviction. + const revived = new DurableStreamShard(state, {}); + + revived.registered = shard.registered; + + const second = createFakeWebSocket(); + + revived.registerSocket(second, { subs: {} }); + await revived.driveMessage(second, { id: "stream_2", query: { functionPath: "chat:answer" }, type: "stream" }); + await waitForTerminator(second); + + const frames = parseFrames(second); + + expect(frames.filter((frame) => frame.type === "chunk").map((frame) => frame.data)).toStrictEqual(["half"]); + expect((frames.at(-1) as { error?: { code?: string } }).error?.code).toBe("STREAM_INTERRUPTED"); + }); +}); diff --git a/packages/do/src/shard-do.ts b/packages/do/src/shard-do.ts index 5e85d0d288..dae8c34f10 100644 --- a/packages/do/src/shard-do.ts +++ b/packages/do/src/shard-do.ts @@ -128,12 +128,14 @@ import { ADMIN_FUNCTIONS, advanceClientWatermark, appendAuditEntry, + appendStreamChunk, armRestore, awaitWsDrain, buildPokeFrames, buildSettings, bumpCdcEpoch, CDC_LOG_TABLE, + claimStreamRun, clearCapturedMail, clearQueueMessages, ConflictError, @@ -150,6 +152,7 @@ import { ensureAuditTable, facetColumn, findStorageReferences, + finishStreamRun, FLAGS_FUNCTION_PREFIX, gateReplicaDispatch, handleReplicaControl, @@ -179,6 +182,8 @@ import { readMigrationStatus, readQueueMessageById, readQueueMessages, + readStreamChunks, + readStreamRun, readTablePage, recordCapturedMail, recordChangedKeys, @@ -195,12 +200,15 @@ import { sendDeltaFrames, ShardRunner, stableStringify, + stableWireKey, subscriptionListDeltas, summarizeFanoutTopics, summarizeSubscriptions, TransactionHeadroomTracker, trimIdempotent, + trimStreamRuns, trySendFrame, + withDeterministicScope, writeGlobalShapeSnapshot, writeIdempotent, writeTouchesMemo, @@ -724,6 +732,31 @@ const IDEMPOTENCY_RETENTION_MS = 86_400_000; */ const IDEMPOTENCY_GC_INTERVAL_MS = 3_600_000; +/** + * How long a durable stream's transcript is kept when the procedure does not + * set its own `ttlMs`. A day covers "I reloaded", "I reopened the tab this + * evening", and a second viewer arriving late, without keeping every + * transcript a chatty shard ever produced. + */ +const DURABLE_STREAM_DEFAULT_TTL_MS = 86_400_000; + +/** Minimum spacing between durable-stream TTL sweeps, amortized onto a stream start. */ +const DURABLE_STREAM_GC_INTERVAL_MS = 3_600_000; + +/** + * One socket attached to a durable stream run. The producer fans every chunk + * out to each attached sink and calls exactly one terminal (`complete`/`fail`); + * a sink that has detached is simply no longer in the set. + */ +interface DurableStreamSink { + /** Deliver one chunk. Returns `false` when the socket refused it (already closed). */ + readonly chunk: (chunk: { data: unknown; seq: number }) => boolean; + /** The run finished successfully — send `complete` and release the socket's stream slot. */ + readonly complete: () => void; + /** The run failed — send the (already redacted) error and release the socket's stream slot. */ + readonly fail: (failure: { code: string; message: string }) => void; +} + /** * Reserved shard name for the fallback Durable Object that hosts every * table without an explicit `.shardBy()` or `.global()` modifier. @@ -1258,6 +1291,18 @@ abstract class ShardDO { */ private readonly streamCancellers = new WeakMap>(); + /** + * Live producers for durable streams, keyed by run key. In-memory on + * purpose: it tracks who is *currently* producing, while the transcript + * itself lives in SQLite. A run whose entry is missing but whose row still + * says `running` was interrupted by an eviction — see + * {@link ShardDO.attachDurableStream}. + */ + private readonly durableStreamRuns = new Map }>(); + + /** Throttle for the durable-stream TTL sweep — at most one pass per {@link DURABLE_STREAM_GC_INTERVAL_MS}. */ + private lastDurableStreamTrimAt = 0; + /** * Lifetime request counters surfaced by the `__lunora_admin__:getMetrics` * RPC. In-memory only — they reset when the DO hibernates or restarts, which @@ -1961,15 +2006,24 @@ abstract class ShardDO { // // Only the depth bookkeeping stays here: it is `ShardDO` state, and the // nested-transaction error above reads it. - return this.runner.runInTransaction(async () => { - this.transactionDepth = 1; + // The mutation body runs in a deterministic scope: `Date.now()` frozen, + // `Math.random()` seeded, `fetch` refused. Sound here and nowhere else — + // this is inside the single-writer gate, so no other dispatch can observe + // the swapped globals. See `withDeterministicScope`. + const now = Date.now(); + const seed = `${this.runner.shardKey ?? ""}:${this.currentRequestMutationId ?? ""}:${String(now)}`; - try { - return await handler(); - } finally { - this.transactionDepth = 0; - } - }); + return this.runner.runInTransaction(async () => + withDeterministicScope({ now, seed }, async () => { + this.transactionDepth = 1; + + try { + return await handler(); + } finally { + this.transactionDepth = 0; + } + }), + ); } /** @@ -3412,7 +3466,10 @@ abstract class ShardDO { * the wire-frame loop in `handleStream`. */ // eslint-disable-next-line class-methods-use-this -- base-class override hook: the codegen subclass overrides this and uses `this` to dispatch via the generated function map - protected executeStream(_functionPath: string, _args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected executeStream( + _functionPath: string, + _args: Record, + ): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { // eslint-disable-next-line unicorn/no-null -- base default: `null` = "no such streaming function"; the codegen subclass overrides and also returns null return null; } @@ -4312,7 +4369,13 @@ abstract class ShardDO { // surface as an unhandled rejection. // Decode the wire-encoded stream args (bigint/bytes survive the WS hop) // before handing them to the stream handler — mirrors the `/rpc` path. - this.handleStream(ws, envelope.id, envelope.query.functionPath, decodeWire(envelope.query.args ?? {}) as Record).catch(() => { + this.handleStream( + ws, + envelope.id, + envelope.query.functionPath, + decodeWire(envelope.query.args ?? {}) as Record, + typeof envelope.sinceChunk === "number" ? envelope.sinceChunk : 0, + ).catch(() => { /* socket already gone; nothing to report */ }); @@ -7489,7 +7552,7 @@ abstract class ShardDO { * `{type:"error"}`. Either way drop the controller. */ - private async handleStream(ws: ShardSocketLike, id: string, functionPath: string, args: Record): Promise { + private async handleStream(ws: ShardSocketLike, id: string, functionPath: string, args: Record, sinceSeq = 0): Promise { const iterable = this.executeStream(functionPath, args); if (!iterable) { @@ -7520,6 +7583,12 @@ abstract class ShardDO { return; } + if (iterable.durable) { + await this.attachDurableStream(ws, id, functionPath, args, { durable: iterable.durable, iterator: iterable.iterator }, sinceSeq); + + return; + } + const controller = new AbortController(); cancellers.set(id, controller); @@ -7575,6 +7644,223 @@ abstract class ShardDO { } } + /** + * Attach a socket to a **durable** stream run, starting the run if this is + * the first attach. + * + * The run is keyed by `(functionPath, args)`, which is what makes the + * feature work: a reload sends the same start frame with the `seq` it last + * saw, lands on the same run, replays the gap out of SQLite, and continues + * live. A second tab asking the same question watches the same transcript + * instead of paying for a second generation. + * + * Four states, all reachable on a first frame. **No run:** claim it and + * start the producer, detached from this socket. **Running with a live + * producer:** replay the gap, then attach a sink. **Finished + * (`complete`/`error`):** replay the transcript and send the recorded + * terminal frame, with no producer. **Running with no live producer:** the + * DO was evicted mid-run — replay what survived, then fail with + * `STREAM_INTERRUPTED` rather than silently re-running the handler, which + * would bill a model call twice and duplicate the tail. + * + * **Known ceiling:** fan-out does not apply socket backpressure the way the + * ephemeral path does. A sink whose `send` throws is dropped, and the client + * resumes from its last `seq` on reconnect — the transcript is durable, so + * dropping a slow socket loses nothing. + */ + private async attachDurableStream( + ws: ShardSocketLike, + id: string, + functionPath: string, + args: Record, + registration: { durable: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable }, + sinceSeq: number, + ): Promise { + const sql = this.sql as SqlExec; + const runKey = `${functionPath}:${stableWireKey(args)}`; + const cancellers = socketMap(this.streamCancellers, ws); + const controller = new AbortController(); + + cancellers.set(id, controller); + ws.send(JSON.stringify({ id, type: "ack" })); + + // Everything below runs without awaiting: a DO handles one event at a + // time, so replay-then-attach cannot interleave with the producer and + // there is no window in which a chunk is neither replayed nor delivered. + let delivered = sinceSeq; + + const sendChunk = (chunk: { data: unknown; seq: number }): boolean => { + if (chunk.seq <= delivered) { + return true; + } + + delivered = chunk.seq; + + return trySendFrame(ws, JSON.stringify({ data: chunk.data, id, seq: chunk.seq, type: "chunk" })); + }; + + for (const chunk of readStreamChunks(sql, runKey, sinceSeq)) { + if (controller.signal.aborted || !sendChunk({ data: JSON.parse(chunk.dataJson) as unknown, seq: chunk.seq })) { + cancellers.delete(id); + + return; + } + } + + const run = readStreamRun(sql, runKey); + + if (run?.status === "complete") { + trySendFrame(ws, JSON.stringify({ id, type: "complete" })); + cancellers.delete(id); + + return; + } + + if (run?.status === "error") { + trySendFrame( + ws, + JSON.stringify({ error: { code: run.errorCode ?? "INTERNAL_SERVER_ERROR", message: run.error ?? "stream failed" }, id, type: "error" }), + ); + cancellers.delete(id); + + return; + } + + const live = this.durableStreamRuns.get(runKey); + + if (run !== undefined && live === undefined) { + trySendFrame( + ws, + JSON.stringify({ + error: { code: "STREAM_INTERRUPTED", message: "the producer for this durable stream did not survive; start a new run" }, + id, + type: "error", + }), + ); + cancellers.delete(id); + + return; + } + + const sink: DurableStreamSink = { + chunk: sendChunk, + complete: () => { + trySendFrame(ws, JSON.stringify({ id, type: "complete" })); + cancellers.delete(id); + }, + fail: (failure) => { + trySendFrame(ws, JSON.stringify({ error: failure, id, type: "error" })); + cancellers.delete(id); + }, + }; + + // A consumer cancelling a durable stream detaches it; the producer keeps + // going so the transcript completes for whoever attaches next. That is + // the whole point of declaring it durable. + controller.signal.addEventListener("abort", () => { + this.durableStreamRuns.get(runKey)?.sinks.delete(sink); + }); + + if (live) { + live.sinks.add(sink); + + return; + } + + this.trimDurableStreams(sql, registration.durable.ttlMs); + claimStreamRun(sql, runKey, Date.now()); + + const state = { sinks: new Set([sink]) }; + + this.durableStreamRuns.set(runKey, state); + + // Detached from the socket, and handed to `waitUntil` where the host + // exposes one so the run isn't cut short the moment the last socket + // closes — a durable stream that dies with its opener is just an + // ephemeral stream with extra writes. + const producing = this.produceDurableStream(runKey, registration.iterator, state); + + this.shardHost.waitUntil?.(producing); + + await producing; + } + + /** + * Drive a durable run's producer: persist each chunk under the next `seq`, + * then fan it out to whichever sockets are attached right now (possibly + * none — a run whose viewers all left still finishes and is still there for + * the next attach). + */ + private async produceDurableStream( + runKey: string, + iterator: (signal: AbortSignal) => AsyncIterable, + state: { sinks: Set }, + ): Promise { + const sql = this.sql as SqlExec; + // Not wired to any consumer's cancel: see `attachDurableStream`. The + // controller exists only to satisfy the handler's signature. + const controller = new AbortController(); + // The producer owns the sequence: it is the only writer, and every + // attach reads the persisted transcript rather than this counter. + let seq = 0; + + try { + for await (const chunk of iterator(controller.signal)) { + const data = encodeWire(chunk); + + seq += 1; + appendStreamChunk(sql, runKey, seq, JSON.stringify(data)); + + for (const sink of state.sinks) { + sink.chunk({ data, seq }); + } + } + + finishStreamRun(sql, runKey, "complete"); + + for (const sink of state.sinks) { + sink.complete(); + } + } catch (error: unknown) { + // Same redaction invariant as the ephemeral path: an intentional + // `LunoraError` keeps its message, anything else is redacted before + // it is persisted — the transcript is replayed to every later + // attach, so a leaked internal message would leak repeatedly. + const { body, redacted } = toErrorBody(error, { fallbackCode: "INTERNAL_SERVER_ERROR", redactedMessage: "internal error" }); + + if (redacted) { + // eslint-disable-next-line no-console -- server-side diagnostic for an internal/unhandled stream error + console.error("[@lunora/do] unhandled durable stream error:", error); + } + + finishStreamRun(sql, runKey, "error", { code: body.code, message: body.message }); + + for (const sink of state.sinks) { + sink.fail({ code: body.code, message: body.message }); + } + } finally { + this.durableStreamRuns.delete(runKey); + } + } + + /** Throttled TTL sweep for durable-stream transcripts, amortized onto a run start. */ + private trimDurableStreams(sql: SqlExec, ttlMs?: number): void { + const now = Date.now(); + + if (now - this.lastDurableStreamTrimAt <= DURABLE_STREAM_GC_INTERVAL_MS) { + return; + } + + this.lastDurableStreamTrimAt = now; + + try { + trimStreamRuns(sql, now - (ttlMs ?? DURABLE_STREAM_DEFAULT_TTL_MS)); + } catch { + // Best-effort GC (missing table on a pre-migration shard / stub sql + // handle) must never fail the stream that triggered it. + } + } + /** * Drain the tables written during the in-flight RPC and re-run every * subscription that depends on one of them. Called after `handleRpc` diff --git a/packages/errors/src/catalog.ts b/packages/errors/src/catalog.ts index 46a5844c9a..77d3c6cd87 100644 --- a/packages/errors/src/catalog.ts +++ b/packages/errors/src/catalog.ts @@ -327,10 +327,12 @@ export const ERROR_CATALOG = { */ EXPIRED: { status: 404, title: "Session expired" }, NESTED_TRANSACTION: { internal: true, status: 500, title: "Nested transaction" }, + NETWORK_IN_MUTATION: { status: 500, title: "Network call inside a mutation" }, OUT_OF_ORDER: { status: 409, title: "Out-of-order mutation" }, SHAPE_GLOBAL_TOO_LARGE: { status: 413, title: "Global shape too large" }, SHAPE_NOT_FOUND: { status: 404, title: "Shape not found" }, SQL_UNAVAILABLE: { internal: true, status: 500, title: "SQL storage unavailable" }, + STREAM_INTERRUPTED: { status: 503, title: "Durable stream interrupted" }, TOKEN_EXPIRED: { status: 401, title: "Authentication token expired" }, TOO_MANY_STREAMS: { status: 429, title: "Too many streams" }, UNKNOWN_ADMIN_OP: { status: 404, title: "Unknown admin operation" }, diff --git a/packages/platform/src/capabilities.ts b/packages/platform/src/capabilities.ts index 5c40d77a7a..bf08472d90 100644 --- a/packages/platform/src/capabilities.ts +++ b/packages/platform/src/capabilities.ts @@ -36,6 +36,13 @@ export interface PlatformCapabilities { containers?: Capability; /** Cross-shard fan-out queries. */ crossShardFanout?: Capability; + + /** + * Durable streams: a `.stream()` run whose chunks are persisted and + * whose producer outlives the socket that opened it, so a reconnecting + * or second client resumes the same transcript. + */ + durableStreams?: Capability; /** Global (replicated) tables backed by a SQL store. */ globalTables?: Capability; /** BYO database via connection pooling (Hyperdrive / RDS Proxy). */ @@ -46,6 +53,7 @@ export interface PlatformCapabilities { localSql?: Capability; /** Email sending (Resend / SES / etc). */ mail?: Capability; + /** Object storage (R2 / S3 / MinIO). */ objectStorage?: Capability; @@ -104,6 +112,10 @@ export const CLOUDFLARE_CAPABILITIES: PlatformCapabilities = { shardedState: { level: "native", note: "Durable Objects with SQLite" }, globalTables: { level: "native", note: "D1 with Sessions API" }, websocketHibernation: { level: "native", note: "DO WebSocket hibernation" }, + durableStreams: { + level: "emulated", + note: "Lunora persists each chunk to the shard's SQLite under a monotonic seq and keeps the producer alive past the socket via waitUntil; the platform has no streaming primitive of its own, and a run whose DO is evicted mid-flight ends as STREAM_INTERRUPTED rather than resuming", + }, localSql: { level: "native", note: "state.storage.sql (SQLite)" }, shardAlarms: { level: "native", note: "state.storage.setAlarm" }, shardPlacement: { @@ -195,6 +207,10 @@ export const NODE_CAPABILITIES: PlatformCapabilities = { level: "emulated", note: "Socket registry with attachments/tags persisted to SQLite, so subscription state survives a process restart; nothing is ever actually evicted from memory, so this is durability without hibernation's memory saving", }, + durableStreams: { + level: "emulated", + note: "Same shape as the Cloudflare host — chunks persisted per run in the shard's SQLite, producer detached from the socket. One process, so a restart mid-run ends it as STREAM_INTERRUPTED", + }, localSql: { level: "native", note: "better-sqlite3 (synchronous, embedded)" }, shardAlarms: { level: "emulated", diff --git a/packages/ratelimit/__tests__/token-budget.test.ts b/packages/ratelimit/__tests__/token-budget.test.ts new file mode 100644 index 0000000000..b9ecdb6ca7 --- /dev/null +++ b/packages/ratelimit/__tests__/token-budget.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; + +import { RateLimiter } from "../src/rate-limiter"; +import { tokenBudget } from "../src/token-budget"; + +const limiter = () => new RateLimiter({ config: { tokens: { capacity: 1000, kind: "token bucket", period: 60_000, rate: 1000 } } }); + +describe("tokenBudget", () => { + it("allows a call while the budget holds and refuses once it is spent", async () => { + expect.assertions(3); + + const budget = tokenBudget(limiter(), "tokens"); + + await expect(budget.check("user-a")).resolves.toMatchObject({ ok: true }); + + await budget.record("user-a", 900); + + await expect(budget.check("user-a")).resolves.toMatchObject({ ok: true }); + + // One oversized call takes the bucket negative — the NEXT call is refused, + // which is the only enforcement point that exists for a cost known in arrears. + await budget.record("user-a", 500); + + await expect(budget.check("user-a")).resolves.toMatchObject({ ok: false }); + }); + + it("charges nothing for a call that used no tokens", async () => { + expect.assertions(1); + + const shared = limiter(); + const budget = tokenBudget(shared, "tokens"); + + await budget.record("user-b", 0); + + await expect(shared.getValue("tokens", { key: "user-b" })).resolves.toMatchObject({ value: 1000 }); + }); + + it("keeps budgets separate per key", async () => { + expect.assertions(1); + + const budget = tokenBudget(limiter(), "tokens"); + + await budget.record("user-c", 1000); + + await expect(budget.check("user-d")).resolves.toMatchObject({ ok: true }); + }); +}); diff --git a/packages/ratelimit/src/index.ts b/packages/ratelimit/src/index.ts index d5ec243bdf..11838803b9 100644 --- a/packages/ratelimit/src/index.ts +++ b/packages/ratelimit/src/index.ts @@ -19,6 +19,8 @@ export type { SqlStoreOptions, } from "./store"; export { createDbStore, createMemoryStore, createReadOnlyDbStore, createSqlStore } from "./store"; +export type { TokenBudget } from "./token-budget"; +export { tokenBudget } from "./token-budget"; export type { RateLimitArgs, RateLimitConfig, diff --git a/packages/ratelimit/src/token-budget.ts b/packages/ratelimit/src/token-budget.ts new file mode 100644 index 0000000000..7ffd95e164 --- /dev/null +++ b/packages/ratelimit/src/token-budget.ts @@ -0,0 +1,91 @@ +/** + * Token budgets: a rate limit whose unit is model tokens (or cents) rather than + * requests. + * + * A request limit is the wrong shape for an LLM. One call can cost a hundred + * tokens or a hundred thousand, so "20 calls a minute per tenant" bounds nothing + * that matters — a single runaway prompt still empties the account. What has to + * be bounded is consumption, and consumption is only known once the call + * returns. + * + * So the budget is spent in arrears, which is also how tokens actually work: + * you cannot un-spend them, and there is nothing to refund. + * + * {@link TokenBudget.check} runs before the call: a tenant whose bucket is + * already empty (or negative, from an expensive previous call) is turned away + * before spending anything. {@link TokenBudget.record} runs after it, with the + * real usage — the bucket is allowed to go negative, so a call that blew through + * the remaining budget completes and the NEXT one is refused until the debt + * refills away. + * + * No estimate, no reservation to reconcile, and no new accounting path: this is + * a thin composition over the limiter's existing `check`/`limit(reserve)`. + * + * The unit is whatever the limit's `capacity` counts. Tokens are the obvious + * choice; cents work identically if you would rather budget money. + */ +import type { RateLimiter } from "./rate-limiter"; +import type { RateLimitStatus } from "./types"; + +/** A budget bound to one named limit — check before the call, record after it. */ +interface TokenBudget { + /** + * Peek at the budget before spending. `ok: false` means it is exhausted: + * refuse the call, and `retryAfter` says when it refills. Consumes nothing. + */ + check: (key: string) => Promise; + + /** + * Charge the tokens a call actually used. Always call it, including when the + * call THREW — a failed generation that consumed input tokens still has to be + * paid for. `tokens` of `0` is a no-op, so a call that spent nothing costs + * nothing. + * + * The charge is a reservation, so it may take the bucket negative: the tokens + * are already spent, and refusing to record them would let a single oversized + * call escape the budget entirely. + */ + record: (key: string, tokens: number) => Promise; +} + +/** + * Bind a {@link TokenBudget} to one of a limiter's named limits. + * + * ```ts + * const budget = tokenBudget(limiter, "tokens"); + * const allowed = await budget.check(userId); + * + * if (!allowed.ok) { + * throw new LunoraError("RATE_LIMITED", `token budget exhausted; retry in ${String(allowed.retryAfter)}ms`); + * } + * + * try { + * const { text, usage } = await generateText({ model: ctx.ai.model(), prompt }); + * + * await budget.record(userId, usage?.totalTokens ?? 0); + * + * return text; + * } catch (error) { + * // The prompt was still sent — charge what is known, then rethrow. + * await budget.record(userId, estimatedInputTokens); + * + * throw error; + * } + * ``` + */ +const tokenBudget = (limiter: RateLimiter, name: Names): TokenBudget => { + return { + check: async (key) => limiter.check(name, { key }), + record: async (key, tokens) => { + // Nothing spent, nothing to charge — and the limiter rejects a zero count. + if (!Number.isFinite(tokens) || tokens <= 0) { + return limiter.check(name, { key }); + } + + return limiter.limit(name, { count: Math.ceil(tokens), key, reserve: true }); + }, + }; +}; + +export type { TokenBudget }; +export { tokenBudget }; diff --git a/packages/react/src/use-stream.ts b/packages/react/src/use-stream.ts index 29e9b2ada3..e39c2549fd 100644 --- a/packages/react/src/use-stream.ts +++ b/packages/react/src/use-stream.ts @@ -18,6 +18,13 @@ interface UseStreamResult { } interface UseStreamOptions { + /** + * Opt into resume-on-reconnect for a stream the server declared `durable`. + * The hook keeps the chunks it already has and the socket re-attaches to the + * same run, so a dropped connection mid-generation continues instead of + * surfacing `STREAM_DISCONNECTED`. Has no effect on an ephemeral stream. + */ + durable?: boolean; /** Forwarded to `client.stream()` — caps the in-flight chunk buffer. */ maxBuffer?: number; shardKey?: string; @@ -67,7 +74,10 @@ const useStream = >( dispatch({ type: "reset" }); dispatch({ type: "start" }); - const { cancel, cleanup } = consumeStream(client.stream(function_, args, { maxBuffer: options.maxBuffer, shardKey: options.shardKey }), dispatch); + const { cancel, cleanup } = consumeStream( + client.stream(function_, args, { durable: options.durable, maxBuffer: options.maxBuffer, shardKey: options.shardKey }), + dispatch, + ); cancelRef.current = cancel; @@ -76,7 +86,7 @@ const useStream = >( cancelRef.current = undefined; }; // react-doctor-disable-next-line react-doctor/exhaustive-deps -- intentional: the stream re-opens on the query's stable `__lunoraRef` and the serialized args (a content hash) rather than the raw `function_`/`args` object identity, so a caller recreating them with the same value doesn't tear down and re-open the stream. `client` is provider-stable. - }, [client, function_.__lunoraRef, serialized, skipped, options.shardKey, options.maxBuffer]); + }, [client, function_.__lunoraRef, serialized, skipped, options.shardKey, options.maxBuffer, options.durable]); return { cancel: () => { diff --git a/packages/server/src/builder/index.ts b/packages/server/src/builder/index.ts index fd62618e39..cb2b05884d 100644 --- a/packages/server/src/builder/index.ts +++ b/packages/server/src/builder/index.ts @@ -260,12 +260,18 @@ const makeBuilder = (kind: FunctionKind, state: BuilderState, visibility?: "inte ctx: unknown; signal: AbortSignal; }) => AsyncGenerator | AsyncIterable, + streamOptions?: { durable?: boolean | { ttlMs?: number } }, ) => { const rls = collectRls(state.middlewares); const maskedTables = collectMask(state.middlewares); + // `durable: true` and `durable: { … }` are the same thing to + // the runtime — normalise here so it only ever sees the object + // (and `false`/omitted collapse to "not durable"). + const durable = streamOptions?.durable === true ? {} : streamOptions?.durable; return { args: state.args, + ...(durable ? { durable } : {}), ...(state.expose ? { expose: state.expose } : {}), handler: makeStreamHandler(state.args, state.middlewares, userHandler), kind: "stream" as const, diff --git a/packages/server/src/builder/types.ts b/packages/server/src/builder/types.ts index 60149d43cc..fcdb71291c 100644 --- a/packages/server/src/builder/types.ts +++ b/packages/server/src/builder/types.ts @@ -3,6 +3,7 @@ import type { Infer, Validator } from "@lunora/values"; import type { ActionCtx as ActionContext, ArgsValidator, + DurableStreamOptions, ExposeConfig, FunctionKind, InferArgs, @@ -85,9 +86,16 @@ export interface QueryBuilder( handler: (options: { args: InferArgs; ctx: Context; signal: AbortSignal }) => AsyncGenerator | AsyncIterable, + options?: { durable?: boolean | DurableStreamOptions }, ) => RegisteredStream; use: (middleware: Middleware) => QueryBuilder; @@ -205,6 +213,7 @@ export interface InternalQueryBuilder( handler: (options: { args: InferArgs; ctx: Context; signal: AbortSignal }) => AsyncGenerator | AsyncIterable, + options?: { durable?: boolean | DurableStreamOptions }, ) => RegisteredStream; use: (middleware: Middleware) => InternalQueryBuilder; } diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index fe1571e141..e0d6740ecd 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -113,6 +113,7 @@ export type { DatabaseReader, DatabaseWriter, DurableObjectJurisdiction, + DurableStreamOptions, ExposeConfig, FunctionKind, FunctionVisibility, diff --git a/packages/server/src/types.ts b/packages/server/src/types.ts index 80065bef1f..6ed6398634 100644 --- a/packages/server/src/types.ts +++ b/packages/server/src/types.ts @@ -663,11 +663,39 @@ type RegisteredLifecycleHook = RegisteredFunction, void, " */ interface RegisteredStream { readonly args: A; + + /** + * Present when the stream was declared `durable`. Chunks are persisted per + * run before they reach a socket, the producer outlives the socket that + * opened it, and a reconnecting (or second) client attaches to the same run + * and replays what it missed. See {@link DurableStreamOptions}. + */ + readonly durable?: DurableStreamOptions; readonly handler: (context: unknown, args: InferArgs, signal: AbortSignal) => AsyncIterable; readonly kind: "stream"; readonly visibility?: FunctionVisibility; } +/** + * Durability settings for a `.stream()` procedure. + * + * A run is identified by the function path plus its arguments, so two clients + * calling the same stream with the same arguments observe one producer and one + * transcript. That identity is the feature: it is what makes a reload resume + * rather than re-generate, and what lets a second tab watch the same answer. + * Give a stream that must NOT be shared a caller-unique argument (a thread id, + * a message id) — the argument set is the run key. + */ +interface DurableStreamOptions { + /** + * How long a finished transcript is kept before it is trimmed, in + * milliseconds. Defaults to 24 hours — long enough to survive a reload and + * a commute, short enough that a chatty shard doesn't accumulate + * transcripts forever. + */ + readonly ttlMs?: number; +} + // --- Context types ----------------------------------------------------------- /** The system tables `ctx.db.system` can read. */ @@ -2233,6 +2261,7 @@ export type { DatabaseReader, DatabaseWriter, DurableObjectJurisdiction, + DurableStreamOptions, ExposeConfig, ExternalSourceCursor, ExternalSourceDefinition, diff --git a/packages/shard-engine/__tests__/deterministic-scope.test.ts b/packages/shard-engine/__tests__/deterministic-scope.test.ts new file mode 100644 index 0000000000..dc114186d0 --- /dev/null +++ b/packages/shard-engine/__tests__/deterministic-scope.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; + +import { mulberry32, withDeterministicScope } from "../src/deterministic-scope"; + +describe("withDeterministicScope", () => { + it("freezes the clock and seeds randomness for the duration of the body", async () => { + expect.assertions(3); + + const run = async (seed: string) => + withDeterministicScope({ now: 1_700_000_000_000, seed }, async () => { + const first = Date.now(); + + await Promise.resolve(); + + // eslint-disable-next-line sonarjs/pseudo-random -- asserting the SCOPE's seeded PRNG is exactly the point; nothing security-sensitive derives from it + return { first, random: Math.random(), second: Date.now() }; + }); + + const seen = await run("shard:mut-1"); + + // The classic bug this closes: one handler, two clock reads, two answers. + expect([seen.first, seen.second]).toStrictEqual([1_700_000_000_000, 1_700_000_000_000]); + // Seeded, so the sequence is a function of the seed, not host entropy... + await expect(run("shard:mut-1")).resolves.toMatchObject({ random: seen.random }); + // ...and a different mutation gets a different one. + await expect(run("shard:mut-2")).resolves.not.toMatchObject({ random: seen.random }); + }); + + it("refuses network I/O inside the scope", async () => { + expect.assertions(1); + + await expect( + withDeterministicScope({ now: 1, seed: "s" }, async () => { + await globalThis.fetch("https://example.com"); + }), + ).rejects.toThrow("fetch is not available inside a mutation"); + }); + + it("restores the ambient sources, including when the body throws", async () => { + expect.assertions(3); + + const realNow = Date.now; + const realRandom = Math.random; + const realFetch = globalThis.fetch; + + await expect( + withDeterministicScope({ now: 1, seed: "s" }, () => { + throw new Error("boom"); + }), + ).rejects.toThrow("boom"); + + expect(Date.now).toBe(realNow); + expect([Math.random, globalThis.fetch]).toStrictEqual([realRandom, realFetch]); + }); + + it("gives two different seeds two different sequences", () => { + expect.assertions(1); + + expect(mulberry32(1)()).not.toBe(mulberry32(2)()); + }); +}); diff --git a/packages/shard-engine/src/ctx-db-migrations.ts b/packages/shard-engine/src/ctx-db-migrations.ts index ebd91e2cd2..8a4b563a63 100644 --- a/packages/shard-engine/src/ctx-db-migrations.ts +++ b/packages/shard-engine/src/ctx-db-migrations.ts @@ -33,6 +33,7 @@ import { migrateIdempotency } from "./ctx-db-idempotency"; import { migrateSearchState } from "./ctx-db-search-state"; import { runDrizzle } from "./do-exec"; import { AGG_COUNT, AGG_KEY, AGG_VALUE, createIndexSql, DOC_COLUMN, geoTableName, isFtsAvailable, jsonPathSql, tableColumns } from "./do-sql"; +import { migrateDurableStreams } from "./durable-stream"; import { rankTableName, sortColumnName } from "./rank"; import { recordSchemaVersion } from "./schema-history"; @@ -268,4 +269,9 @@ export const runShardMigrations = ( // empty until the first global shape is seeded, so a non-global DO pays // nothing — but persisting it lets the poll-loop diff survive hibernation. migrateGlobalShapeSnapshot(sql); + + // Always present: a `.stream()` procedure can be declared `durable` without + // a schema change, so the transcript tables have to exist before the first + // one runs. Empty (and free) on a shard that never opens a durable stream. + migrateDurableStreams(sql); }; diff --git a/packages/shard-engine/src/deterministic-scope.ts b/packages/shard-engine/src/deterministic-scope.ts new file mode 100644 index 0000000000..1e5b36c882 --- /dev/null +++ b/packages/shard-engine/src/deterministic-scope.ts @@ -0,0 +1,118 @@ +/** + * The deterministic scope a mutation body runs inside. + * + * A mutation is not an ordinary function: it may be re-executed (an OCC retry, a + * replayed dispatch), its result is cached by idempotency key, and its writes + * feed a changelog that subscriptions re-derive from. Ambient nondeterminism + * breaks all three in ways that only show up in production — `Date.now()` read + * twice in one handler yields two different "now"s, `Math.random()` makes a + * replay write different rows, and a `fetch` in the write path makes an atomic + * transaction depend on someone else's uptime. + * + * The advisor flags these statically (`nondeterministic_query_mutation`), but a + * lint only catches the call sites it can see — not the ones inside a dependency. + * So the runtime closes it too, by replacing the ambient sources for the duration + * of the mutation rather than merely forbidding them: + * + * `Date.now()` returns the instant the mutation began, so a handler reading the + * clock twice sees one value. `Math.random()` becomes a seeded PRNG, so the + * sequence is a function of the seed instead of the host's entropy. `fetch` + * throws — network I/O belongs in an action, because a transaction that awaits a + * third party cannot be atomic. + * + * **Scope of the guarantee.** Values are stable WITHIN one execution — the point + * is that a handler reading the clock twice sees one instant. They are not + * identical across two separate executions of the same mutation (a retry gets a + * fresh `now`, hence a fresh seed), which matches what the platform can honestly + * promise: an idempotency key, not a replayed clock, is what makes a retry safe. + * + * **Why only mutations.** Patching a global is only sound while nothing else can + * observe it. A mutation holds the DO's single-writer gate, so nothing else runs + * during it. Queries interleave (a deferred subscription re-run lands between two + * awaits of another query), so freezing the clock for one would hand its instant + * to the other — a worse lie than the one being fixed. Queries keep the static + * lint and the `ctx.now`/`ctx.fetch` surfaces instead. + */ +import { LunoraError } from "@lunora/errors"; + +/** Mixing increment of the published mulberry32 PRNG. */ +const MULBERRY_INCREMENT = 0x6d_2b_79_f5; + +/** 2^32 — divisor that maps the PRNG's 32-bit output into `[0, 1)`. */ +const UINT32_RANGE = 4_294_967_296; + +/* eslint-disable no-bitwise -- FNV-1a and mulberry32 ARE bit manipulation; both + are fixed published algorithms whose constants and shifts define the + distribution, so expressing them otherwise would only obscure them. Mirrors + the same disable in `shard-ring.ts`. */ + +/** FNV-1a (32-bit) over a string, so a seed derives deterministically from the dispatch identity. */ +const seedFrom = (text: string): number => { + let hash = 0x81_1c_9d_c5; + + for (const character of text) { + hash ^= character.codePointAt(0) ?? 0; + hash = Math.imul(hash, 0x01_00_01_93); + } + + return hash >>> 0; +}; + +/** A small, well-distributed 32-bit PRNG (mulberry32) — enough for a seeded `Math.random`. */ +const mulberry32 = (seed: number): (() => number) => { + let state = seed >>> 0; + + return () => { + state = (state + MULBERRY_INCREMENT) >>> 0; + + let next = state; + + next = Math.imul(next ^ (next >>> 15), next | 1); + next ^= next + Math.imul(next ^ (next >>> 7), next | 61); + + return ((next ^ (next >>> 14)) >>> 0) / UINT32_RANGE; + }; +}; + +/* eslint-enable no-bitwise */ + +/** The ambient sources this scope replaces, captured so they can be restored exactly. */ +interface AmbientSources { + fetch: typeof globalThis.fetch; + now: () => number; + random: () => number; +} + +/** + * Run `body` with `Date.now`, `Math.random`, and `fetch` replaced by their + * deterministic counterparts, restoring the originals afterwards — including when + * `body` throws. + * + * `now` is the dispatch instant (the same one `ctx.now` exposes) and `seed` should + * identify the dispatch (function path + instant + any idempotency key), so two + * different mutations never share a random sequence. + */ +const withDeterministicScope = async (options: { now: number; seed: string }, body: () => Promise | T): Promise => { + const original: AmbientSources = { fetch: globalThis.fetch, now: Date.now, random: Math.random }; + const random = mulberry32(seedFrom(options.seed)); + + Date.now = () => options.now; + Math.random = random; + globalThis.fetch = () => { + throw new LunoraError( + "NETWORK_IN_MUTATION", + "fetch is not available inside a mutation: a transaction that awaits a third party cannot be atomic. Move the call to an action (or schedule one).", + { status: 500 }, + ); + }; + + try { + return await body(); + } finally { + Date.now = original.now; + Math.random = original.random; + globalThis.fetch = original.fetch; + } +}; + +export { mulberry32, seedFrom, withDeterministicScope }; diff --git a/packages/shard-engine/src/durable-stream.ts b/packages/shard-engine/src/durable-stream.ts new file mode 100644 index 0000000000..89982df832 --- /dev/null +++ b/packages/shard-engine/src/durable-stream.ts @@ -0,0 +1,206 @@ +/** + * Durable streams: the persisted backing store for a `.stream()` procedure + * declared `durable`. + * + * An ordinary stream is a per-socket iterator — the producer is owned by the + * socket that opened it, so a reload mid-generation loses everything already + * yielded and cannot be resumed. That is fine for a progress feed and wrong for + * anything expensive: a model's answer must survive a refresh, and a second tab + * watching the same run must see the same tokens. + * + * A durable stream separates three concerns. **Identity:** a run is keyed by + * `(functionPath, args)`, so the same call from any socket attaches to the same + * run rather than starting a second one. **Durability:** every chunk is appended + * to `__stream_chunks` under a monotonic `seq` before it reaches a socket, so a + * reconnect replays from the last `seq` the client acknowledged and continues + * live from there. **Lifetime:** `__stream_runs` records the terminal state, so + * a client that attaches after the producer finished still gets the full + * transcript and a `complete` frame instead of hanging. + * + * The store is host-neutral (it touches SQLite only through {@link SqlExec}); + * driving the producer and fanning chunks out to attached sockets is the host's + * job. + * + * **Known ceiling.** A run whose producer dies with the DO instance (eviction + * mid-generation) stays `running` with no live producer. Attaching to one + * replays the persisted prefix and then fails with `STREAM_INTERRUPTED` rather + * than silently restarting the handler — re-running a model call would bill + * twice and duplicate the tail. Resumable-on-the-server producers would need + * the handler itself to be replay-safe; use `@lunora/workflow` when you need + * that guarantee today. + */ + +import { sql as dsql } from "drizzle-orm"; + +import type { SqlExec } from "./ctx-db"; +import { runDrizzle } from "./do-exec"; + +const STREAM_RUNS_TABLE = "__stream_runs"; +const STREAM_CHUNKS_TABLE = "__stream_chunks"; + +/** Terminal and non-terminal states a durable run can be in. */ +type DurableStreamStatus = "complete" | "error" | "running"; + +/** A durable run's header row — everything but the chunks themselves. */ +interface DurableStreamRun { + /** Redacted error message, present only when `status === "error"`. */ + error?: string; + /** Error code, present only when `status === "error"`. */ + errorCode?: string; + /** Highest `seq` appended so far; `0` before the first chunk. */ + lastSeq: number; + /** Wall-clock millis when the run started — drives TTL trimming. */ + startedAt: number; + status: DurableStreamStatus; +} + +/** One persisted chunk, in `seq` order. */ +interface DurableStreamChunk { + /** The wire-encoded chunk payload, JSON-stringified. */ + dataJson: string; + seq: number; +} + +/** + * Create the durable-stream tables. Both are always present (empty until the + * first durable stream runs, so a project with none pays nothing) because a + * durable stream can be declared without a schema migration. + */ +const migrateDurableStreams = (sql: SqlExec): void => { + runDrizzle( + sql, + dsql`CREATE TABLE IF NOT EXISTS ${dsql.identifier(STREAM_RUNS_TABLE)} ( + run_key TEXT PRIMARY KEY, + status TEXT NOT NULL, + last_seq INTEGER NOT NULL DEFAULT 0, + error_code TEXT, + error TEXT, + started_at REAL NOT NULL + )`, + ); + + runDrizzle( + sql, + dsql`CREATE TABLE IF NOT EXISTS ${dsql.identifier(STREAM_CHUNKS_TABLE)} ( + run_key TEXT NOT NULL, + seq INTEGER NOT NULL, + data_json TEXT NOT NULL, + PRIMARY KEY (run_key, seq) + )`, + ); +}; + +/** Read a run header, or `undefined` when the run was never started (or was trimmed). */ +const readStreamRun = (sql: SqlExec, runKey: string): DurableStreamRun | undefined => { + const rows = runDrizzle<{ error: string | null; error_code: string | null; last_seq: number; started_at: number; status: string }>( + sql, + dsql`SELECT status, last_seq, error_code, error, started_at FROM ${dsql.identifier(STREAM_RUNS_TABLE)} WHERE run_key = ${runKey} LIMIT 1`, + ).toArray(); + + const row = rows[0]; + + if (row === undefined) { + return undefined; + } + + return { + ...(row.error === null ? {} : { error: row.error }), + ...(row.error_code === null ? {} : { errorCode: row.error_code }), + lastSeq: row.last_seq, + startedAt: row.started_at, + status: row.status as DurableStreamStatus, + }; +}; + +/** + * Claim a run for production. Returns `true` when this caller created the row + * and therefore owns the producer; `false` when a run already exists, in which + * case the caller attaches to it instead of starting a second handler. + * + * The read and the insert are not a race despite being two statements: a DO + * executes one event at a time and nothing awaits between them, so no second + * attach can interleave. `RETURNING` would fold them into one statement, but + * nothing else in the engine relies on it against workerd's SQLite yet. + */ +const claimStreamRun = (sql: SqlExec, runKey: string, startedAt: number): boolean => { + if (readStreamRun(sql, runKey) !== undefined) { + return false; + } + + runDrizzle( + sql, + dsql`INSERT INTO ${dsql.identifier(STREAM_RUNS_TABLE)} (run_key, status, last_seq, started_at) VALUES (${runKey}, ${"running"}, 0, ${startedAt})`, + ); + + return true; +}; + +/** + * Append one chunk and advance the run's `last_seq`. Called before the chunk + * reaches any socket, so a client that reconnects can always ask for everything + * after the last `seq` it saw without a gap. + */ +const appendStreamChunk = (sql: SqlExec, runKey: string, seq: number, dataJson: string): void => { + runDrizzle(sql, dsql`INSERT OR IGNORE INTO ${dsql.identifier(STREAM_CHUNKS_TABLE)} (run_key, seq, data_json) VALUES (${runKey}, ${seq}, ${dataJson})`); + + runDrizzle(sql, dsql`UPDATE ${dsql.identifier(STREAM_RUNS_TABLE)} SET last_seq = ${seq} WHERE run_key = ${runKey}`); +}; + +/** + * Read the persisted chunks after `sinceSeq`, in order. `sinceSeq = 0` replays + * the whole transcript, which is what a first-time attach to a finished run + * asks for. + */ +const readStreamChunks = (sql: SqlExec, runKey: string, sinceSeq: number): DurableStreamChunk[] => + runDrizzle<{ data_json: string; seq: number }>( + sql, + dsql`SELECT seq, data_json FROM ${dsql.identifier(STREAM_CHUNKS_TABLE)} WHERE run_key = ${runKey} AND seq > ${sinceSeq} ORDER BY seq ASC`, + ) + .toArray() + .map((row) => { + return { dataJson: row.data_json, seq: row.seq }; + }); + +/** Mark a run finished. `errorCode`/`error` are written only for the `error` status. */ +const finishStreamRun = (sql: SqlExec, runKey: string, status: "complete" | "error", failure?: { code: string; message: string }): void => { + /* eslint-disable unicorn/no-null -- SQL NULL is the value being written: a run that completes must clear its error columns, and `undefined` is not bindable */ + const errorCode = failure?.code ?? null; + const errorMessage = failure?.message ?? null; + /* eslint-enable unicorn/no-null */ + + runDrizzle( + sql, + dsql`UPDATE ${dsql.identifier(STREAM_RUNS_TABLE)} + SET status = ${status}, error_code = ${errorCode}, error = ${errorMessage} + WHERE run_key = ${runKey}`, + ); +}; + +/** + * Drop runs (and their chunks) that started before `olderThanTs`. Runs are a + * cache of a transcript, not application data — the TTL is what keeps a chatty + * shard's SQLite from growing without bound. + */ +const trimStreamRuns = (sql: SqlExec, olderThanTs: number): void => { + runDrizzle( + sql, + dsql`DELETE FROM ${dsql.identifier(STREAM_CHUNKS_TABLE)} WHERE run_key IN ( + SELECT run_key FROM ${dsql.identifier(STREAM_RUNS_TABLE)} WHERE started_at < ${olderThanTs} + )`, + ); + + runDrizzle(sql, dsql`DELETE FROM ${dsql.identifier(STREAM_RUNS_TABLE)} WHERE started_at < ${olderThanTs}`); +}; + +export { + appendStreamChunk, + claimStreamRun, + finishStreamRun, + migrateDurableStreams, + readStreamChunks, + readStreamRun, + STREAM_CHUNKS_TABLE, + STREAM_RUNS_TABLE, + trimStreamRuns, +}; +export type { DurableStreamChunk, DurableStreamRun, DurableStreamStatus }; diff --git a/packages/shard-engine/src/index.ts b/packages/shard-engine/src/index.ts index 5612ecf7d4..894709eac7 100644 --- a/packages/shard-engine/src/index.ts +++ b/packages/shard-engine/src/index.ts @@ -85,6 +85,7 @@ export type { export { DATA_MIGRATION_STATE_TABLE, readMigrationStatus, runDataMigration } from "./data-migration"; export type { DependencyTracker } from "./dependency-tracker"; export { createDependencyTracker, depKey, SCAN_DEP, tableFromDepKey } from "./dependency-tracker"; +export { mulberry32, seedFrom, withDeterministicScope } from "./deterministic-scope"; export { runDrizzle, runSql } from "./do-exec"; export { AGG_COUNT, @@ -106,6 +107,18 @@ export { } from "./do-sql"; export type { RenderedSql, SqlEngine } from "./drizzle"; export { param, renderSql, sqliteInList, unionAll } from "./drizzle"; +export type { DurableStreamChunk, DurableStreamRun, DurableStreamStatus } from "./durable-stream"; +export { + appendStreamChunk, + claimStreamRun, + finishStreamRun, + migrateDurableStreams, + readStreamChunks, + readStreamRun, + STREAM_CHUNKS_TABLE, + STREAM_RUNS_TABLE, + trimStreamRuns, +} from "./durable-stream"; export type { ExternalSourceDiffResult } from "./external-source-diff"; export { diffExternalSource } from "./external-source-diff"; export { liftSourceId, normalizeSourceDocument, normalizeSourceValue } from "./external-source-lift"; diff --git a/packages/shard-engine/src/types.ts b/packages/shard-engine/src/types.ts index e0ed49c7c2..4f743a8028 100644 --- a/packages/shard-engine/src/types.ts +++ b/packages/shard-engine/src/types.ts @@ -98,6 +98,18 @@ export interface SubscriptionEnvelope { /** Resume checkpoint on a `shape_subscribe` envelope (the `__cdc_log` cursor the client's shape view is at). */ sinceCheckpoint?: number; + /** + * Resume watermark for a `stream` envelope: the highest chunk `seq` this + * client already has. Only meaningful for a stream declared `durable` — + * the run replays everything after it, then continues live. Absent or `0` + * asks for the whole transcript. + * + * Deliberately NOT named `sinceSeq`: that name is already taken, one level + * down in `query`, for the CDC cursor a live query resumes from. Two + * different watermarks on the same envelope need two different names. + */ + sinceChunk?: number; + /** CDC epoch the {@link SubscriptionEnvelope.sinceCheckpoint} belongs to. */ sinceEpoch?: string; diff --git a/packages/studio/__tests__/features/evals/eval-scores.test.ts b/packages/studio/__tests__/features/evals/eval-scores.test.ts new file mode 100644 index 0000000000..e5881cfbc2 --- /dev/null +++ b/packages/studio/__tests__/features/evals/eval-scores.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; + +import { buildEvalCards, extractEvalRuns, extractEvalTrends } from "../../../src/features/evals/eval-scores"; +import type { MetricHistoryResult, TraceSummary } from "../../../src/lib/admin"; + +const trace = (overrides: Partial & Pick): TraceSummary => { + return { + durationMs: 10, + functionPath: "chat:answer", + ok: true, + rootName: "chat:answer", + startTs: 1000, + traceId: "trace-1", + ...overrides, + }; +}; + +const span = (attributes: Record, name = "generate") => { + return { + attributes, + depth: 1, + durationMs: 5, + name, + offsetMs: 5, + ok: true, + parentSpanId: "", + spanId: "span-1", + }; +}; + +describe("eval score extraction", () => { + it("pulls the score, label, and trace hand-off off a graded span", () => { + expect.assertions(1); + + const runs = extractEvalRuns([ + trace({ spans: [span({ "gen_ai.evaluation.answer-relevance.label": "pass", "gen_ai.evaluation.answer-relevance.score": 0.8 })] }), + ]); + + expect(runs).toStrictEqual([ + { + label: "pass", + name: "answer-relevance", + score: 0.8, + spanName: "generate", + // The span carries an offset, not a wall clock — 1000 + 5. + startTs: 1005, + traceId: "trace-1", + }, + ]); + }); + + it("ignores spans and series that carry no eval score", () => { + expect.assertions(2); + + expect(extractEvalRuns([trace({ spans: [span({ "http.status": 200 })] })])).toStrictEqual([]); + expect(extractEvalTrends({ series: [{ functionPath: "f", kind: "counter", name: "orders.placed", points: [] }] })).toStrictEqual([]); + }); + + it("summarises from the durable buckets when they exist, and from the live ring otherwise", () => { + expect.assertions(4); + + const runs = extractEvalRuns([trace({ spans: [span({ "gen_ai.evaluation.tone.score": 0.5 })] })]); + const history: MetricHistoryResult = { + series: [ + { + functionPath: "chat:answer", + kind: "gauge", + name: "gen_ai.evaluation.tone.score", + points: [ + { bucketMs: 0, count: 1, last: 0.2, max: 0.2, min: 0.2, sum: 0.2 }, + { bucketMs: 60_000, count: 1, last: 1, max: 1, min: 1, sum: 1 }, + ], + }, + ], + }; + + const [withTrend] = buildEvalCards(runs, extractEvalTrends(history)); + + // Durable half wins the summary: the ring's single 0.5 is not the history. + expect(withTrend).toMatchObject({ latest: 1, max: 1, min: 0.2, name: "tone" }); + expect(withTrend?.points).toHaveLength(2); + // Its live runs still ride along — that is what makes a score clickable. + expect(withTrend?.runs).toHaveLength(1); + + const [liveOnly] = buildEvalCards(runs, []); + + expect(liveOnly).toMatchObject({ latest: 0.5, max: 0.5, mean: 0.5, min: 0.5, points: [] }); + }); +}); diff --git a/packages/studio/src/app/nav-labels.ts b/packages/studio/src/app/nav-labels.ts index 3229ab4cf9..d7fd4f2076 100644 --- a/packages/studio/src/app/nav-labels.ts +++ b/packages/studio/src/app/nav-labels.ts @@ -33,6 +33,7 @@ const useNavLabels = (): NavLabels => { return { advisorHealth: t("Health score"), agents: t("Agents"), + evals: t("Evals"), analytics: t("Analytics"), api: t("API"), audit: t("Audit"), @@ -95,6 +96,7 @@ const useNavLabels = (): NavLabels => { const tabDescription = { advisorHealth: t("Score your app's advisor findings: a grade, the coverage split, and every procedure ranked worst-first."), agents: t("Inspect agent threads, message timelines, tool calls, and token usage."), + evals: t("Scores recorded by recordEvaluation — the latest verdict per eval, its trend, and the graded runs behind it."), analytics: t("Usage and latency from Analytics Engine — request volume, p50/p95, and hot shards."), api: t("Interactive OpenAPI reference and copy-paste snippets for your functions."), audit: t("A durable log of admin state-changing operations."), diff --git a/packages/studio/src/app/nav-types.ts b/packages/studio/src/app/nav-types.ts index 2b44933ecb..5826b24bc1 100644 --- a/packages/studio/src/app/nav-types.ts +++ b/packages/studio/src/app/nav-types.ts @@ -45,6 +45,7 @@ type StudioTab = | "schedule" | "schema" | "advisorHealth" + | "evals" | "security" | "settings" | "sql" diff --git a/packages/studio/src/app/studio.tsx b/packages/studio/src/app/studio.tsx index 4b381b969c..343fa6233d 100644 --- a/packages/studio/src/app/studio.tsx +++ b/packages/studio/src/app/studio.tsx @@ -146,6 +146,7 @@ const SqlEditorPanel = lazyNamed(() => import("../features/sql/sql-editor-panel" const FileBrowser = lazyNamed(() => import("../features/storage/file-browser"), "FileBrowser"); const StorageRulesPanel = lazy(() => import("../features/storage/storage-rules-panel")); const TracesPanel = lazy(() => import("../features/traces/traces-panel")); +const EvalsPanel = lazy(() => import("../features/evals/evals-panel")); const VectorBrowser = lazyNamed(() => import("../features/vectors/vector-browser"), "VectorBrowser"); const WorkflowsPanel = lazy(() => import("../features/workflows/workflows-panel")); @@ -296,6 +297,7 @@ const TAB_ICONS: Record = { files: , flags: , functions: , + evals: , health: , home: , insights: , @@ -346,7 +348,7 @@ const NAV_GROUPS: readonly [NavGroup, ...NavGroup[]] = [ { key: "storage", tabs: ["files", "storageRules", "kv"] }, { key: "observability", - tabs: ["issues", "logs", "traces", "audit", "realtime", "fanout", "containers", "metrics", "analytics", "health", "deploymentHealth"], + tabs: ["issues", "logs", "traces", "evals", "audit", "realtime", "fanout", "containers", "metrics", "analytics", "health", "deploymentHealth"], }, { key: "advisors", tabs: ["advisorHealth", "security", "rls", "permissions", "insights"] }, { key: "operations", tabs: ["schedule", "mail", "drains", "notifications", "payments", "flags"] }, @@ -455,6 +457,7 @@ const TABS = exhaustiveRouteTabs([ "issues", "logs", "traces", + "evals", "realtime", "fanout", "mail", @@ -1001,6 +1004,7 @@ const buildRouter = ({ issues: , logs: , traces: , + evals: , metrics: , migrations: , notifications: , diff --git a/packages/studio/src/features/agents/agents-panel.tsx b/packages/studio/src/features/agents/agents-panel.tsx index 8191a35885..87bee4b164 100644 --- a/packages/studio/src/features/agents/agents-panel.tsx +++ b/packages/studio/src/features/agents/agents-panel.tsx @@ -21,6 +21,7 @@ interface AgentsPanelProps { /** The merged agent schema-extension table names the runtime reads/writes (see `@lunora/agent`'s `component.ts`). */ const THREADS_TABLE = "agent_threads"; const MESSAGES_TABLE = "agent_messages"; +const RUN_QUEUE_TABLE = "agent_run_queue"; /** Read caps — a studio observability view, not a paginated browser (the data browser owns deep paging). */ const THREAD_LIMIT = 100; @@ -139,6 +140,10 @@ const AgentsPanel = ({ initialShardKey = "" }: AgentsPanelProps): ReactElement = const loadedTables = tables !== undefined; const hasAgentTables = Array.isArray(tables) && tables.some((table) => table.name === THREADS_TABLE); + // The queue table arrived after the thread tables: a deployment whose schema + // predates it has agents but no `agent_run_queue`, so probe it separately + // rather than assuming the extension is all-or-nothing. + const hasQueueTable = Array.isArray(tables) && tables.some((table) => table.name === RUN_QUEUE_TABLE); const { data: threadsPage, @@ -156,6 +161,15 @@ const AgentsPanel = ({ initialShardKey = "" }: AgentsPanelProps): ReactElement = table: MESSAGES_TABLE, }; + // Runs parked behind the one in flight (`onConcurrentRun: "queue"`). Without + // this the queue is invisible: the thread just reads "running" while other + // runs wait their turn, and a full queue's CONFLICT looks unexplained. + const { data: queuePage } = useAdminQuery( + ADMIN_FUNCTIONS.readTablePage, + { limit: THREAD_LIMIT, offset: 0, orderBy: { column: "position", direction: "asc" }, skipCount: true, table: RUN_QUEUE_TABLE }, + { enabled: hasQueueTable, live: true, shardKey: initialShardKey }, + ); + const { data: messagesPage, error: messagesError, @@ -167,6 +181,8 @@ const AgentsPanel = ({ initialShardKey = "" }: AgentsPanelProps): ReactElement = }); const threads = Array.isArray(threadsPage?.rows) ? threadsPage.rows : []; + const queuedRuns = Array.isArray(queuePage?.rows) ? queuePage.rows : []; + const queueDepth = (threadKey: string): number => queuedRuns.filter((row) => readString(row, "threadKey") === threadKey).length; const messages = Array.isArray(messagesPage?.rows) ? messagesPage.rows : []; const selectedThread = threads.find((row) => readString(row, "key") === selectedKey); @@ -202,6 +218,7 @@ const AgentsPanel = ({ initialShardKey = "" }: AgentsPanelProps): ReactElement = {t("Agent")} {t("Key")} {t("Status")} + {t("Queued")} {t("Messages")} {t("Tokens")} {t("Updated")} @@ -223,6 +240,15 @@ const AgentsPanel = ({ initialShardKey = "" }: AgentsPanelProps): ReactElement = {status ?? "—"} + + {queueDepth(key) === 0 ? ( + — + ) : ( + + {queueDepth(key)} + + )} + {readNumber(row, "messageCount") ?? "—"} {totalTokens ?? "—"} diff --git a/packages/studio/src/features/evals/eval-scores.ts b/packages/studio/src/features/evals/eval-scores.ts new file mode 100644 index 0000000000..c3c14bc848 --- /dev/null +++ b/packages/studio/src/features/evals/eval-scores.ts @@ -0,0 +1,131 @@ +/** + * Pull eval scores out of the two places `recordEvaluation` emits them. + * + * Both halves matter and neither is redundant: the **durable** half is the + * `gen_ai.evaluation..score` metric series, persisted in per-minute + * buckets, which is the only thing that can back a trend across a hibernation. + * The **live** half is the same key as a span attribute in the shard's bounded + * trace ring — no history, but it carries the trace id, which is what makes a + * score clickable back to the generation that earned it. + */ +import type { MetricHistoryPoint, MetricHistoryResult, TraceSummary } from "../../lib/admin"; + +/** `gen_ai.evaluation..score` — the name is the capture group. */ +const SCORE_KEY = /^gen_ai\.evaluation\.(.+)\.score$/u; + +/** One score observed on a trace span: the live, clickable half. */ +export interface EvalRun { + /** Categorical verdict from the sibling `.label` attribute, when the scorer emitted one. */ + label?: string; + /** Eval name (the key's middle segment). */ + name: string; + score: number; + /** The span that carried the score — names the generation being graded. */ + spanName: string; + startTs: number; + /** Trace the span belongs to, for the "open in Traces" hand-off. */ + traceId: string; +} + +/** One eval's durable trend: its per-minute buckets, oldest first. */ +export interface EvalTrend { + name: string; + points: MetricHistoryPoint[]; +} + +/** Everything the panel shows for one eval, live and durable halves joined by name. */ +export interface EvalCard { + /** Most recent score, whichever half saw it last. */ + latest?: number; + max?: number; + mean?: number; + min?: number; + name: string; + /** Buckets for the sparkline; empty when only live scores exist. */ + points: MetricHistoryPoint[]; + /** Recent individual runs, newest first — each links to its trace. */ + runs: EvalRun[]; +} + +/** Extract every eval score carried by a trace span's attributes. */ +export const extractEvalRuns = (traces: ReadonlyArray): EvalRun[] => { + const runs: EvalRun[] = []; + + for (const trace of traces) { + for (const span of trace.spans) { + for (const [key, value] of Object.entries(span.attributes ?? {})) { + const matched = SCORE_KEY.exec(key); + + if (!matched || typeof value !== "number") { + continue; + } + + const name = matched[1] as string; + const label = span.attributes?.[`gen_ai.evaluation.${name}.label`]; + + runs.push({ + ...(typeof label === "string" ? { label } : {}), + name, + score: value, + spanName: span.name, + // Spans carry an offset from the trace anchor, not a wall clock. + startTs: trace.startTs + span.offsetMs, + traceId: trace.traceId, + }); + } + } + } + + return runs.toSorted((a, b) => b.startTs - a.startTs); +}; + +/** Extract the durable trend for every eval that has one. */ +export const extractEvalTrends = (history: MetricHistoryResult | undefined): EvalTrend[] => { + const trends: EvalTrend[] = []; + + for (const series of history?.series ?? []) { + const matched = SCORE_KEY.exec(series.name); + + if (matched) { + trends.push({ name: matched[1] as string, points: series.points }); + } + } + + return trends; +}; + +/** + * Join the two halves into one card per eval name. + * + * The summary (min/mean/max) prefers the durable buckets when they exist and + * falls back to the live ring otherwise, so an app that has not passed + * `ctx.metrics` to `recordEvaluation` still gets a useful panel — just without + * a trend that survives a restart. + */ +export const buildEvalCards = (runs: ReadonlyArray, trends: ReadonlyArray): EvalCard[] => { + const names = new Set([...runs.map((run) => run.name), ...trends.map((trend) => trend.name)]); + const cards: EvalCard[] = []; + + for (const name of names) { + const ownRuns = runs.filter((run) => run.name === name); + const points = trends.find((trend) => trend.name === name)?.points ?? []; + const values = points.length > 0 ? points.map((point) => point.last) : ownRuns.map((run) => run.score); + const latest = points.at(-1)?.last ?? ownRuns[0]?.score; + + cards.push({ + ...(latest === undefined ? {} : { latest }), + ...(values.length === 0 + ? {} + : { + max: Math.max(...values), + mean: values.reduce((total, value) => total + value, 0) / values.length, + min: Math.min(...values), + }), + name, + points, + runs: ownRuns, + }); + } + + return cards.toSorted((a, b) => a.name.localeCompare(b.name)); +}; diff --git a/packages/studio/src/features/evals/evals-panel.tsx b/packages/studio/src/features/evals/evals-panel.tsx new file mode 100644 index 0000000000..cc65d72a02 --- /dev/null +++ b/packages/studio/src/features/evals/evals-panel.tsx @@ -0,0 +1,145 @@ +import type { ReactElement } from "react"; + +import ErrorAlert from "../../components/error-alert"; +import { ShardInput } from "../../components/shard-input"; +import { Badge } from "../../components/ui/badge"; +import { Card, CardContent } from "../../components/ui/card"; +import { EmptyState } from "../../components/ui/empty-state"; +import { useAdminQuery } from "../../hooks/use-admin-query"; +import { useShardKey } from "../../hooks/use-shard-key"; +import { useT } from "../../i18n/i18n-context"; +import type { MetricHistoryResult, TracesResult, TraceSummary } from "../../lib/admin"; +import { ADMIN_FUNCTIONS } from "../../lib/admin"; +import { formatTimestamp } from "../../lib/internal"; +import { Sparkline } from "../reports/sparkline"; +import type { EvalCard } from "./eval-scores"; +import { buildEvalCards, extractEvalRuns, extractEvalTrends } from "./eval-scores"; + +/** Coerce a (possibly partial or pre-feature) `getTraces` payload into its `traces` array. */ +const tracesOf = (result: TracesResult | undefined): TraceSummary[] => (Array.isArray(result?.traces) ? result.traces : []); + +/** Scores are `[0, 1]`; two decimals is the resolution a scorer's verdict actually carries. */ +const formatScore = (score: number | undefined): string => (score === undefined ? "—" : score.toFixed(2)); + +interface EvalCardViewProps { + readonly card: EvalCard; + readonly onOpenTrace?: (traceId: string) => void; +} + +/** One eval: its headline score, the durable trend, and the recent graded runs. */ +const EvalCardView = ({ card, onOpenTrace }: EvalCardViewProps): ReactElement => { + const t = useT(); + + return ( + + +
+

+ {card.name} +

+ + {formatScore(card.latest)} + +
+ +

+ {t("min {min} · mean {mean} · max {max}", { max: formatScore(card.max), mean: formatScore(card.mean), min: formatScore(card.min) })} +

+ + {card.points.length > 1 ? ( + point.last)} + testId="ev-trend" + /> + ) : ( + // No trend rather than a misleading flat line: buckets only exist + // once `recordEvaluation` is handed `ctx.metrics`. +

+ {t("No durable history yet — pass ctx.metrics to recordEvaluation to chart a trend.")} +

+ )} + +
+ {card.runs.slice(0, 5).map((run) => ( +
+ + {formatTimestamp(run.startTs)} · {run.spanName} + + + {run.label === undefined ? null : {run.label}} + {formatScore(run.score)} + {onOpenTrace === undefined ? null : ( + + )} + +
+ ))} +
+
+
+ ); +}; + +interface EvalsPanelProps { + readonly initialShardKey?: string; + /** Open a trace by id — the drill-down from a graded run to the generation. */ + readonly onOpenTrace?: (traceId: string) => void; +} + +/** + * The Evals page: what `recordEvaluation` has scored on this deployment. + * + * Two reads, because the two halves answer different questions. `getTraces` + * gives the recent individual verdicts with the trace id that earned each one — + * clickable, but bounded by the shard's in-memory ring, so it empties on + * hibernation. `getMetricHistory` gives the durable per-minute buckets behind + * the trend line, but only for apps that pass `ctx.metrics` to + * `recordEvaluation`; the card says so plainly rather than drawing a trend the + * data can't back. + */ +const EvalsPanel = ({ initialShardKey, onOpenTrace }: EvalsPanelProps): ReactElement => { + const t = useT(); + const { queryShardKey, setShardKey, shardKey } = useShardKey(initialShardKey); + + const { data, error } = useAdminQuery(ADMIN_FUNCTIONS.getTraces, {}, { live: true, shardKey: queryShardKey }); + // The trend read's own failure is ignored: a worker predating the RPC, or an + // app that never records the metric, simply has no trend — never a broken page. + const { data: history } = useAdminQuery(ADMIN_FUNCTIONS.getMetricHistory, {}, { live: true, shardKey: queryShardKey }); + + const cards = buildEvalCards(extractEvalRuns(tracesOf(data)), extractEvalTrends(history)); + + return ( +
+ + + {error === null ? null : } + + {cards.length === 0 ? ( + + ) : ( +
+ {cards.map((card) => ( + + ))} +
+ )} +
+ ); +}; + +export default EvalsPanel; diff --git a/packages/studio/src/locales/en.ts b/packages/studio/src/locales/en.ts index 7151b97ba1..0c4bd3b325 100644 --- a/packages/studio/src/locales/en.ts +++ b/packages/studio/src/locales/en.ts @@ -1253,6 +1253,15 @@ const MESSAGE_IDS = [ // Observability — ctx.trace waterfalls "Traces", "Recent ctx.trace waterfalls for this shard — the drill-down from a log line.", + "Evals", + "Queued", + "Scores recorded by recordEvaluation — the latest verdict per eval, its trend, and the graded runs behind it.", + "min {min} · mean {mean} · max {max}", + "Score trend for {name}", + "No durable history yet — pass ctx.metrics to recordEvaluation to chart a trend.", + "Open the trace {trace}", + "No evals recorded", + "Call recordEvaluation from a graded run and its score shows up here.", "Search traces", "search trace, span, or function", "Errors only", diff --git a/packages/testing/src/agent-harness.ts b/packages/testing/src/agent-harness.ts index 1a6eb75a36..069168815e 100644 --- a/packages/testing/src/agent-harness.ts +++ b/packages/testing/src/agent-harness.ts @@ -247,8 +247,32 @@ const createRuntime = ( return undefined; }; + /** + * The run-completion mutation: terminal status plus, in production, the + * hand-off to the next queued run. The harness keeps no run queue — a test + * drives one run at a time — so it applies the terminal patch and reports an + * empty queue, including the ownership guard that makes a replayed + * completion a no-op. + */ + const completeRun = (args?: Record): unknown => { + const thread = threads.get(args?.["key"] as string); + + if (!args || !thread || thread.instanceId !== args["instanceId"]) { + return {}; + } + + for (const [field, value] of Object.entries(args)) { + if (field !== "key" && field !== "instanceId" && value !== undefined) { + (thread as unknown as Record)[field] = value; + } + } + + return {}; + }; + const handlers = new Map([ [DEFAULT_AGENT_FUNCTION_PATHS.appendMessage, appendMessage], + [DEFAULT_AGENT_FUNCTION_PATHS.completeRun, completeRun], [DEFAULT_AGENT_FUNCTION_PATHS.ensureThread, ensureThread], [DEFAULT_AGENT_FUNCTION_PATHS.listMessages, listMessages], [DEFAULT_AGENT_FUNCTION_PATHS.patchThread, patchThread], diff --git a/packages/testing/src/evaluation-telemetry.ts b/packages/testing/src/evaluation-telemetry.ts index 1621a119f2..939b18f736 100644 --- a/packages/testing/src/evaluation-telemetry.ts +++ b/packages/testing/src/evaluation-telemetry.ts @@ -30,6 +30,15 @@ interface EvaluationSpanHandle { setAttributes: (fields: Record) => void; } +/** + * Structural slice of `ctx.metrics` — enough to record a score as a durable + * series. Declared here for the same reason as {@link EvaluationSpanHandle}: no + * dependency on `@lunora/server`, and the real handle is assignable. + */ +interface EvaluationMetrics { + gauge: (name: string, value: number, attributes?: Record) => void; +} + /** One eval verdict to emit. */ interface RecordEvaluationInput { /** @@ -38,6 +47,16 @@ interface RecordEvaluationInput { */ label?: string; + /** + * Optional `ctx.metrics` handle. Passing it ALSO records the score as a + * `gen_ai.evaluation..score` gauge, which is what gives an eval a + * durable history: span attributes live in the shard's bounded in-memory + * ring and vanish on hibernation, while metrics are persisted in per-minute + * buckets and can be charted as a trend. Additive — the attributes are + * emitted either way. + */ + metrics?: EvaluationMetrics; + /** * The scorer/evaluation name — becomes the key's name segment. Any character * outside `[A-Za-z0-9._-]` is replaced with `_` so a scorer name carrying a @@ -105,9 +124,12 @@ const recordEvaluation = (input: RecordEvaluationInput): Record( -`approval:${call.id}`, { type: `agent-approval:${call.id}` })`. Resolution - (`component.ts:512-557`, `agentResolveApproval`) calls - `ctx.agents[agent].sendEvent(instanceId, { type: `agent-approval:${toolCallId}`, -payload })` — the SAME `AgentHandle.sendEvent` producer - (`types.ts:1161-1187`) a queue resume would use. -- `packages/scheduler/src/create-workpool.ts:24-34` is explicit about its own - boundary: "Why not Cloudflare Queues? ... do NOT grow multi-step - orchestration on top of this — that's Cloudflare **Workflows** (`step.do` / - `step.sleep` / `step.waitForEvent`)." A workpool is a bounded-concurrency - **action dispatcher** (fire-and-report-completion via `POST /complete`) with - no notion of "this specific already-created workflow instance is paused, - wake exactly it, and hand it durable state." An agent run **is** the - multi-step orchestration the workpool's own doc tells callers to keep off - it. - -## Recommendation: park on the agent's own DO table + `step.waitForEvent`, not the scheduler workpool - -Do **not** route `"queue"` through `@lunora/scheduler`'s workpool. Two reasons, -both load-bearing: - -1. **Wrong unit of work.** The workpool dispatches a `FunctionReference` call - and waits for `POST /complete`; it has no concept of "resume workflow - instance `wf-b` at its `waitForEvent` point." Fitting a park/resume queue - onto it means building exactly the orchestration primitive - `create-workpool.ts` says belongs in Workflows — i.e. reimplementing - `step.waitForEvent` on top of a tool whose doc comment says not to. -2. **Wrong ownership boundary.** The seq-claim correctness (below) depends on - the dequeue-and-handoff happening in the _same_ DO mutation that frees the - thread, so no third party can observe a briefly-ownerless thread. That - mutation already lives on the agent's own `agent_threads` table inside the - shard DO. Routing through a second DO (`SchedulerDO`) would split that - atomic step across two storage backends and reopen the exact race the - design has to close. - -Instead: add a bounded, per-thread durable queue **table** to the same schema -extension `agent_threads`/`agent_messages` already live in -(`packages/agent/src/component.ts:35-132`), and drive parking with -`step.waitForEvent` — the same primitive the HITL approval path already -proves out end-to-end (persist marker → patch status → wait on a scoped event -type → resume → patch status back). - -### Shape (not implemented outside the prototype) - -```ts -// A NEW bare table alongside "threads"/"messages" in the same extension. -const RUN_QUEUE_BARE_TABLE = "run_queue"; - -defineTable({ - agent: v.string(), - enqueuedAt: v.number(), - instanceId: v.string(), - // Monotonic per-thread position — see "ordering" below for why this beats - // sorting by `enqueuedAt` alone. - position: v.number(), - threadKey: v.string(), -}) - .index("byThread", ["threadKey", "position"]) // FIFO dequeue - .index("byThreadInstance", ["threadKey", "instanceId"], { unique: true }) // idempotent enqueue - .public(); -``` - -## Ordering + seq claim — the correctness core - -The STOP condition this plan was written under is: _"the per-thread seq -counter can't be claimed by a resuming run without racing the in-flight -run."_ Working through it: **it doesn't race**, provided both halves of the -handoff stay inside the DO's existing single-threaded mutation serialization -— the same property that already makes today's `"reject"`/`"replace"` branches -race-free. - -**Enqueue (on conflict, policy `"queue"`).** Inside the _same_ mutation that -today throws `CONFLICT` (`component.ts:230-235`), instead: - -- Look up an existing queue row for `(threadKey, instanceId)` (the - `byThreadInstance` unique index). If found, return its existing `position` - — **do not** insert a second row. This is the fix for the replay hazard - (below): `ensureThread` runs outside `step.do`, so a workflow replay of a - still-parked run re-executes this mutation for real, and it must be a - no-op, not a duplicate enqueue. -- Otherwise, if the queue is at its depth bound, throw `CONFLICT` (the same - message shape as `"reject"` today) — see "Bound + overflow" below. -- Otherwise, insert `{ threadKey, instanceId, agent, enqueuedAt: now, -position: nextPosition }`, where `nextPosition` is a per-thread counter - (stored on the thread row, incremented like `messageCount` already is) — - **not** `Date.now()`. Two runs parking in the same millisecond must still - get a strict, unambiguous order; a counter allocated inside the same - serialized mutation gives that for free, the same way `messageCount` already - gives `seq` its ordering. -- Return `{ created: false, queued: true, position }` instead of throwing. - -**Resume (on completion).** Inside the _same_ mutation that today patches the -finishing thread to `"idle"`/`"error"`/`"cancelled"` (there is no such -mutation yet — today's `patchThreadByKey` in `agent-loop.ts` just sets -`status`; the prototype adds the "check the queue" step to that call site): - -- Guard on `thread.instanceId === callerInstanceId` first — the same - ownership check `agentResolveApproval` already applies - (`component.ts:533-537`). This makes the dequeue step idempotent under a - replay of the _finishing_ run's own completion: a second execution sees the - thread's `instanceId` has already moved to the dequeued run and no-ops - rather than dequeuing the _next_ entry too. -- Query `byThread` ordered by `position` ascending, take the first row (FIFO). -- If none: patch the thread to the terminal status as today. -- If found: **in the same mutation**, delete the queue row and patch the - thread to `{ instanceId: head.instanceId, status: "running", updatedAt: -now }` — i.e. skip the terminal-status write entirely; ownership transfers - directly from the finishing instance to the head of the queue. Return - `{ dequeued: head.instanceId }` to the caller (the finishing run's - workflow), which then calls `ctx.agents[agent].sendEvent(head.instanceId, -{ type: `agent-dequeue:${threadKey}:${head.instanceId}`, payload: {} })`. - -Because both enqueue and dequeue-and-handoff are single mutations on a -single-threaded DO, there is no window where the thread is observably -"free": a third run — genuinely new, or a replay of anyone else — either -sees the thread still owned by the run that's about to finish, or already -owned by the dequeued instance. Nobody can wedge in between. The resumed run -never re-claims anything for its first message: by the time its -`waitForEvent` resolves, `thread.instanceId` already names it, so it goes -straight to `agentAppendMessage` (which — same as today — isn't itself -instance-gated; the gate is entirely "who is allowed to become `running`"). - -## Resume trigger - -`step.waitForEvent` on a queue-entry-scoped type: `agent-dequeue:: -` — mirrors the HITL approval's `agent-approval:` -scoping exactly (`agent-loop.ts:216-220`'s reasoning applies verbatim: native -CF Workflows matches a waiter by `type`, so an unscoped type would let one -parked run's wake event resolve a different one). No alarm or cron: delivery -piggybacks on the finishing run's own completion step, which already has a -workflow context to call `sendEvent` from (same call site `agentResolveApproval` -already demonstrates is reachable via `ctx.agents`). - -Open question (below): what happens if the DB mutation commits (ownership -already transferred) but the subsequent `sendEvent` RPC fails or the process -dies before making it — the dequeued run then owns the thread but is never -told to wake up. - -## Replay safety - -- **A parked run's own replay never double-enqueues** — the - `byThreadInstance` unique lookup makes enqueue idempotent (see above). -- **A REPLAY re-entering under the same instance id is never treated as a new - parked run** — inherited unchanged from the existing guard: `isConcurrentRun` - in `component.ts:220-223` already excludes `args.instanceId === priorInstanceId`, - and that check runs _before_ the queue branch, so it never fires for a - replay of the run that currently owns the thread. -- **A finishing run's own replay never double-dequeues** — the - `thread.instanceId === callerInstanceId` guard on the completion mutation - (above) makes the handoff idempotent, same pattern as - `agentResolveApproval`'s instance check. - -## Prototype (test-only) - -`packages/agent/src/run-queue.prototype.ts` implements exactly the two -mutations described above (`ensureThreadOrQueue`, `completeRunAndDequeue`) -against the SAME minimal `ctx.db` shape `__tests__/component.test.ts`'s -`fakeDatabase` already models (`insert`/`patch`/`query().withIndex().first()/ -.collect()`) — **not** wired into `component.ts`, `agent-loop.ts`, or any -trigger path. It exists to prove the ordering/replay claims above, not to -ship. - -`packages/agent/__tests__/run-queue.prototype.test.ts` drives: - -- A starts and runs (`"running"`). -- B starts with `onConcurrentRun: "queue"` while A is in flight → parks - (`queued: true`, not rejected — `CONFLICT` is never thrown). -- C starts the same way while A and B are both ahead of it → parks behind B - (`position` strictly greater than B's). -- A completes → dequeue-and-handoff returns B's instance id; the thread now - shows `instanceId: B`, `status: "running"`; the queue now holds only C. -- B completes → dequeue-and-handoff returns C's instance id; ordering - A → B → C is asserted end to end via the sequence of `instanceId` values - the thread took on, not just the two happy-path hops. -- A REPLAY of B's bootstrap while B is still parked (before A completes) does - **not** create a second queue row and does **not** change B's `position`. -- A REPLAY of A's completion (called twice with the same finishing instance - id) dequeues B only once — the second call is a no-op, not a double-dequeue - that would skip C's turn. -- The queue enforces its depth bound: parking past the cap throws `CONFLICT` - (the honest fallback), not an unbounded row count. - -## Bound + overflow - -The prototype hardcodes a small constant (`MAX_QUEUE_DEPTH = 5`) rather than -inventing a config surface — see open questions. Overflow throws the same -`CONFLICT` shape `"reject"` already throws today, so a caller that floods a -thread past the cap gets an honest, immediate failure instead of unbounded -durable rows or a run that silently never gets its turn. - -## Open questions (unresolved by this spike) - -1. **Queue depth + overflow policy as a real config surface.** The prototype - hardcodes 5. Production needs this as a per-agent (or per-`onConcurrentRun`) - knob — e.g. `onConcurrentRun: { mode: "queue", maxDepth: 5 }` — which is a - breaking shape change to the current flat string literal union in - `AgentConfig` (`types.ts:592`). Whether overflow should always be `"reject"` - or itself configurable (e.g. drop-oldest) is unresolved. -2. **Wake-delivery failure.** If the completion mutation commits the - ownership handoff but the follow-up `ctx.agents[agent].sendEvent(...)` call - fails or never runs (crash between the two), the dequeued instance owns the - thread but sits hibernating forever with nothing to wake it. The HITL - approval path has the same theoretical gap today (an approval mutation that - commits `sendEvent` failure) but it's user-triggered and retryable from a - client; a queue resume has no user in the loop to retry it. Needs either a - reconciliation sweep (an alarm that re-sends to any thread whose owner has - been "running" with zero messages appended for N minutes) or a stronger - guarantee that the mutation and the `sendEvent` call succeed atomically — - both out of scope for a spike. -3. **What a client observes while parked.** `AgentThreadStatus` today is - `"awaiting_input" | "cancelled" | "error" | "idle" | "running"` — none of - these mean "your run is queued behind another." While B is parked, the - thread it would attach to still shows A's status (`"running"`), since - ownership hasn't transferred. Does the _caller_ of the queued run (which - dispatched a workflow that's now parked, not the thread's live subscriber) - need a way to observe `{ queued: true, position }`? The prototype's queue - table could back a query, but adding a public query is a real feature - decision, not a spike deliverable. -4. **Interaction with `"replace"`.** If a thread has A running and B, C parked - behind it (`"queue"`), and a fourth run D calls with `onConcurrentRun: -"replace"`, does D take over from A only (leaving B, C to resume after D - finishes — B/C queued behind a run they never knew about), or does replace - flush the queue (terminate B and C too, since they were waiting for A - specifically)? Both are defensible; this spike takes no position. -5. **Durable-state cost.** Each parked run is a live Cloudflare Workflow - instance hibernating on `waitForEvent` for the parked duration — bounded by - the depth cap, but a cap of 5 during a real burst is still 5 concurrently - billed, paused Workflow instances per busy thread. Whether the default cap - in question 1 should be smaller (e.g. 2-3) pending real usage data is - unresolved. - -## Non-goals (this spike) - -- Wiring `"queue"` into `agentEnsureThread`/`agent-loop.ts` for real. -- Changing `"reject"`/`"replace"` behavior or their tests. -- Adding a `queueDepth` (or similar) field to `AgentConfig`/`defineAgent`. -- A client-facing "queued" status or query. -- The reply-out path (plan 242) — unrelated surface. diff --git a/plans/242-agent-reply-design.md b/plans/242-agent-reply-design.md deleted file mode 100644 index 124c78b750..0000000000 --- a/plans/242-agent-reply-design.md +++ /dev/null @@ -1,266 +0,0 @@ -# Plan 242 — Reply-out path for triggered agent runs (design spike) - -- **Category**: capability gap (inbound trigger, no outbound reply) -- **Status**: SPIKE — design + single-channel prototype only, no production wiring -- **Baseline**: `2d4f71511` (drift check passed — `channels.ts`/`inbound.ts`/`types.ts` - unchanged since; the only diff under `packages/agent/src` is plan 240's own - new prototype file) -- **Goal**: decide how a `defineAgent`-declared agent, triggered by an inbound - Slack mention / GitHub comment / Discord message / email, delivers its - answer back to the SAME thread/channel/conversation it was triggered from. - Produce a design + a single-channel (email) prototype proving capture → run - → threaded reply. Do **not** build the other three channels' outbound - adapters or wire anything into `component.ts`/`agent-loop.ts`/`inbound.ts`. - -## Current state (verified) - -- **Inbound is fully built, outbound does not exist.** `packages/agent/src/channels.ts` - verifies Slack (HMAC over `v0:timestamp:body`), GitHub (HMAC over the body), - and Discord (Ed25519 over `timestamp+body`) webhook signatures - (`verifySlack`/`verifyGithub`/`verifyDiscord`, `channels.ts:66-126`), then - offers the parsed, verified event to each agent's `onInbound.map` mapper - (`channels.ts:311-371`, `dispatchAgentChannel`). `packages/agent/src/inbound.ts` - does the email equivalent: DKIM/SPF/DMARC-gated, dispatching to `onEmail` - (`inbound.ts:67-112`, `dispatchAgentEmail`). -- **Grep confirms the gap.** `reply|respond|postMessage|sendEmail|outbound|threadTs|replyTo` - matches nothing in `channels.ts` or `inbound.ts` — there is no code path - from a finished run back to the channel it came from. The run's answer only - ever lands in `agent_messages` (the live-subscription thread the mapper's - OWN caller, if any, can read) — never posted back to Slack/GitHub/Discord, - never emailed back. -- **The reply handle is available at map time, not dropped by verification.** - This matters for the STOP condition this plan was written under - ("capturing the reply reference requires re-parsing the raw inbound payload - the verifier already consumed"). Checked directly: - - Email: `packages/mail/src/inbound/parse.ts:65-96` (`InboundEmail`) already - carries `messageId`, `from`, `to`, `inReplyTo`, `references` — CR/LF-checked, - fully parsed — and the WHOLE object is handed to `AgentEmailMapper` - (`types.ts:462`) unmodified. No re-parse is needed; the ref is one field - read away from where the mapper already stands. - - Channels: `InboundChannelEvent` (`types.ts:474-483`) hands the mapper - `headers` and a `json()` accessor over the ALREADY-VERIFIED raw body - (`channels.ts:328`, built once per request, after signature check). A - Slack `event_callback`'s `event.channel`/`event.ts` (or `thread_ts` for a - threaded reply), a GitHub `issue_comment`'s `repository`/`issue.number`/ - `comment.id`, and a Discord message's `channel_id`/`id` are all fields on - that same parsed body — again, no re-parse and no signature-verification - change required to read them. - - **Conclusion: this STOP condition does not trigger for any of the four - channels.** The gap is that nothing captures these fields into the - RETURNED run shape (`AgentEmailRun`/`AgentChannelRun`, - `types.ts:433-448`/`468`) — both are `{ input, owner?, threadKey, title? }` - today, with no room for a reply reference. That's a type/shape gap at the - map→run boundary, not a re-parsing problem. -- `packages/mail/src/types.ts:33-44` (`SendOptions`) already accepts arbitrary - `headers?: Record` on `mailer.send(...)` — so `In-Reply-To`/ - `References` threading is just two header entries, no new mail-package - capability needed. `@lunora/mail` is already a direct dependency of - `@lunora/agent` (`packages/agent/package.json:93`). -- `plans/README.md:627` — plan 132 ("Outbound webhook delivery on - queue/scheduler/dispatch") spiked Standard-Webhooks-over-`SchedulerDO` - delivery: signed, retried, dead-lettered dispatch to a table of REGISTERED - SUBSCRIBER endpoints. That is a fan-out primitive for "notify N URLs an - event happened," with machinery (endpoint table, redrive UX) sized for that - problem. A reply targets exactly ONE destination the inbound event already - named (one thread, one comment, one Message-ID) — there is no subscriber - list, no fan-out, and (per channel) a different, provider-specific delivery - call (Slack `chat.postMessage`, GitHub "create issue comment", `@lunora/mail` - send) rather than a generic signed webhook POST. - -## `AgentReplyRef` per channel - -A discriminated union, one variant per trigger, threaded through the run -shape (not the event — the event is already fully available to the mapper; -the ref is what survives PAST the mapper, into the run and eventually to -wherever a reply is composed): - -```ts -type AgentReplyRef = - | { channel: "discord"; channelId: string; messageId?: string } - | { channel: "email"; from: string; messageId: string; references?: string; to: string[] } - | { channel: "github"; commentId?: number; issueNumber: number; owner: string; repo: string } - | { channel: "slack"; channelId: string; threadTs: string }; -``` - -- **Email** — `from`/`to` swap for the reply (reply goes back to the sender, - from whichever mailbox received it); `messageId` becomes `In-Reply-To`; - `references` (the inbound message's OWN `References` header, if any) plus - its `messageId` becomes the reply's `References` — standard RFC 2822/5322 - threading, the same convention every mail client already implements. -- **Slack** — `channelId` + `threadTs` (the root message's `ts`, or the event's - own `ts` if it's the first reply) is exactly what `chat.postMessage`'s - `thread_ts` parameter wants. -- **GitHub** — `owner`/`repo`/`issueNumber` addresses "create a comment on this - issue/PR"; `commentId` is carried for a possible future "reply to this - specific review comment" (GraphQL reply-to-comment), not needed for a - top-level issue comment. -- **Discord** — `channelId` alone posts a new channel message; a genuine - threaded reply (Discord's `message_reference`) additionally wants the - triggering message's `id`. See the open question below — Discord's actual - reply mechanism doesn't fit this shape as cleanly as the other three (below). - -Capture point: each channel's mapper populates `replyRef` on ITS OWN turn, -from data it already has: - -- `onEmail: (email) => ({ ..., replyRef: email.messageId === undefined ? undefined : { channel: "email", from: email.from, messageId: email.messageId, references: email.references, to: email.to } })` -- `onInbound.map: (event) => { const body = event.json() as SlackEventBody; return { ..., replyRef: { channel: "slack", channelId: body.event.channel, threadTs: body.event.thread_ts ?? body.event.ts } }; }` - -Both are ORDINARY application code the mapper author writes — `AgentReplyRef` -just needs to be a valid field on `AgentEmailRun`/`AgentChannelRun` for it to -type-check and flow through to wherever a reply is triggered. This plan does -not itself change `types.ts` (see Non-goals); it establishes the shape a real -change would add. - -## Reply API: automatic callback, not a tool - -Two shapes considered: - -1. **`defineAgent({ onReply: async (result, replyRef) => {...} })`** — a new - per-agent callback, mirroring the existing `onStepFinish` (`types.ts:609`) - in how it's wired: invoked once, automatically, at the same point - `agent-loop.ts`'s terminal turn persists the final assistant message and - patches the thread status (`agent-loop.ts` — `handleTurn`'s "final answer" - branch, `~828-870`). The run doesn't need to know or care it was - triggered externally; `onReply` only fires when `replyRef` is present - (i.e., the thread WAS started from an inbound trigger, not a normal - in-app `ctx.agents..run` call). -2. **A reply tool** — give the model an explicit `reply` tool it calls - mid-conversation (like any other tool call), so the agent can post - more than once (a "still working on it" ack, then a final answer) or - choose NOT to reply. - -**Recommendation: (1), the automatic callback, as the default; leave (2) as a -future extension, not a competing default.** A triggered agent's whole point -is "answer where you were asked" — making that automatic is the same reasoning -that makes email threading automatic in a mail client, and it needs no prompt -engineering to make the model remember to call a tool. A tool-based path is -still worth having LATER for agents that want multiple/partial replies (open -question below), layered on top of `onReply` rather than replacing it — a tool -call can itself just invoke the same reply primitive `onReply` would use. - -## Transport decision: build directly, not over plan 132 - -**Build a direct, per-channel outbound call — do not route through plan 132's -Standard-Webhooks-over-scheduler transport.** Same reasoning as plan 240's -workpool rejection, mirrored: - -- Plan 132 solves "deliver a signed payload to N registered subscriber - endpoints, with retry/dead-letter/redrive for a fan-out audience." A reply - has exactly one destination, already named by the inbound event (a - `threadTs`, a `Message-ID`, an issue number) — there is no subscriber table - to look up and no generic payload to sign, because the destination isn't a - registered webhook URL at all; it's a provider API call (`chat.postMessage`, - "create comment," an outbound email). -- The four channels don't share a wire format the way plan 132's subscribers - do (one Standard-Webhooks POST shape for everyone). Reply delivery is - necessarily per-channel: `@lunora/mail` for email (already a dependency), - a Slack Web API call for Slack, GitHub's REST/GraphQL API for GitHub, and - Discord's REST API (or the interaction followup-webhook token) for Discord. - Forcing these through a shared signed-webhook transport buys nothing; each - needs its own SDK/fetch call and its own auth anyway. -- Retry IS still a real need (a transient Slack/GitHub/mail failure - shouldn't silently drop the answer), but that's a much smaller ask than - plan 132's full delivery framework — likely `step.do`'s existing at-least- - once retry (the run is already inside a Cloudflare Workflow) is enough, - not a new endpoint-table/dead-letter subsystem. - -## Per-channel auth - -- **Email — already has a home.** The reply goes out through the app's - existing `@lunora/mail` `Mailer` (`createMailer`/`createMailerFromEnv`), - configured once for the whole app (API key or Cloudflare Email Workers - `send_email` binding). No new credential surface: `onReply` for an - email-triggered run just needs a `Mailer` in scope, exactly like any other - server-side send already in the app. -- **Slack/GitHub/Discord — no home yet; reporting the gap, not inventing a - store.** `AgentInboundChannel` (`types.ts:501-513`) already carries an - INBOUND `secret: string | ((env) => string | undefined)` for verifying - signatures — but there is nothing analogous for an OUTBOUND credential (a - Slack bot token, a GitHub App installation token, a Discord bot token). - `ctx.secrets` (Cloudflare Secrets Store, per `@lunora/server`'s package - description) is the obvious place such a token would eventually live, and - an `AgentInboundChannel.replyToken` field mirroring the existing `secret` - shape is the obvious API shape — but neither exists today, and this spike - does not add either. This is a real gap a follow-up plan needs to close - before Slack/GitHub/Discord replies can ship, independent of the queue/ - transport decisions above. - -## Prototype (test-only, email channel) - -`packages/agent/src/reply.prototype.ts` — NOT wired into `inbound.ts`, -`component.ts`, `agent-loop.ts`, or `types.ts`, and NOT exported from -`./index`: - -- `EmailReplyRef` (the email variant of `AgentReplyRef` above). -- `captureEmailReplyRef(email: InboundEmail): EmailReplyRef | undefined` — - reads `messageId`/`from`/`to`/`references` straight off the ALREADY-PARSED - `InboundEmail` from `@lunora/mail/inbound` (no re-parsing); returns - `undefined` when the inbound message had no `Message-ID` (nothing to thread - against — same "can't queue an id-less run" honesty pattern as plan 240's - overflow case). -- `replyToEmail(mailer, replyRef, body)` — sends via the REAL `Mailer` contract - from `@lunora/mail` (`mailer.send(...)`), addressed back to `replyRef.from`, - with `headers: { "In-Reply-To": replyRef.messageId, References: }` — standard RFC 5322 threading. - -`packages/agent/__tests__/reply.prototype.test.ts` builds a fake `MailTransport` -(records what it was asked to send, matching the pattern `component.test.ts`'s -`fakeAgents()` already uses for recording calls) and a REAL `createMailer` -from `@lunora/mail` on top of it — proving the prototype's payload passes the -package's actual address/header validation, not just a hand-rolled assertion. -The test drives: capture a ref from a synthetic inbound email (with a prior -`References` header, so the round-trip must APPEND rather than replace) → "run" -the agent (a stub answer string, standing in for `runAgentLoop`'s real output) -→ reply → assert the fake transport recorded a message `to` the original -sender, `In-Reply-To` the original `Message-ID`, `References` containing BOTH -the prior chain and the original id, and the run's answer as the body. A -second case proves a message with NO `Message-ID` yields `undefined` (declines -to reply) rather than sending an unthreaded reply. - -`channels.ts`/`inbound.ts` are untouched by this plan — running the full -`@lunora/agent` suite (which includes the existing Slack/GitHub/Discord -signature-verification tests) is itself the proof inbound verification is -unaffected. - -## Open questions (unresolved by this spike) - -1. **`AgentReplyRef` as a real field.** This spike defines the shape but does - not add it to `AgentEmailRun`/`AgentChannelRun`/`InboundChannelEvent` in - `types.ts` — a real change needs to decide whether `AgentChannelRun` - (today `= AgentEmailRun`, `types.ts:468`) keeps that type alias (forcing - `AgentReplyRef` to be one union covering all channels, as drafted above) or - diverges into its own type now that channels and email need different - reply shapes. -2. **Per-channel outbound token storage.** Reported above as a gap, not - designed here: Slack/GitHub/Discord need a credential surface on - `AgentInboundChannel` (or elsewhere) that doesn't exist yet. -3. **Automatic vs. explicit reply.** The recommended `onReply` callback fires - once, automatically, on the final answer. Does any real use case need - multiple replies (an early ack, a follow-up), and if so is that a SECOND - automatic hook, or the tool-based path noted above layered on top? -4. **Partial-failure behavior.** The run succeeds and persists its answer to - the thread, but the reply POST/send fails (rate limit, revoked token, - transient network). Today's `agentAppendMessage`/thread state has no - "answered but couldn't tell them" status — does this need a new - `AgentThreadStatus` value, a retry (and if so, how many/how durable — see - the transport decision above), or is "the answer is safely in the thread, - the reply is best-effort" an acceptable default? -5. **Discord's reply mechanism doesn't fit the shared shape.** Slack/GitHub/ - email all reply via a persistent bot/API token to a durable - channel/issue/mailbox address. Discord INTERACTIONS (the inbound trigger - `channels.ts` verifies) instead hand back a time-limited followup-webhook - token scoped to that one interaction — a different, narrower credential - than "a bot token that can post to any channel." Whether Discord reply - support uses that followup-webhook token (simpler auth, but expires) or a - full bot token (persistent, but a bigger credential to provision) is - unresolved and may need its own mini-spike. - -## Non-goals (this spike) - -- Adding `AgentReplyRef`/`onReply`/`replyToken` to `types.ts`, or wiring - `onReply` into `agent-loop.ts`'s terminal-turn handling. -- Slack, GitHub, or Discord outbound adapters (only email is prototyped). -- Any token/secret storage productization (`ctx.secrets` wiring, a new - `AgentInboundChannel` field). -- Changing `channels.ts`'s or `inbound.ts`'s verification logic. -- The durable run queue (plan 240) — unrelated surface. diff --git a/plans/245-eval-runner-design.md b/plans/245-eval-runner-design.md deleted file mode 100644 index fb32444775..0000000000 --- a/plans/245-eval-runner-design.md +++ /dev/null @@ -1,342 +0,0 @@ -# Plan 245 Phase 0 — `lunora eval` runner + Studio Evals panel: design & prototype - -> Design + prototype deliverable for plan 245 (SPIKE). The eval kit itself -> (`evaluate`, `llmScorer`/`keywordScorer`/`regexScorer`/`exactMatchScorer`/ -> `containsScorer`, `agentHarness`, `recordEvaluation`) is unchanged by this -> work — `packages/testing/src/` has a zero-line diff. This doc defines the -> `*.eval.ts` discovery convention, the CI/`--threshold` exit-code contract, -> and the Studio panel's data source, and reports one real gap found while -> designing the panel. - -**Baseline:** `36421ad58` (2026-07-31, drift-checked against `2d4f71511` per -plan — `git diff --stat 2d4f71511..HEAD -- packages/testing/src packages/cli/src -packages/studio/src/features` shows zero changes to `packages/testing/src` or -`packages/studio/src/features`; `packages/cli/src` changed in ways unrelated to -commands/eval, confirmed by reading the diff) - -## 1. Current state (recap, verified) - -- The kit is complete and untouched by this design: - - `packages/testing/src/scorer.ts` — `evaluate(cases, produce, scorers)` → - `EvalResult { average, items: EvalItemResult[] }`; five scorers. - - `packages/testing/src/agent-harness.ts` — `agentHarness(agent, { script, -functions? })` drives `runAgentLoop` against an in-memory `agents:*` - runtime double and a `DurableStepJournal`. **No network, no Durable - Object, no `wrangler dev`** — confirmed by reading the whole file. - `agentHarness(...).run(...)` resolves in-process. - - `packages/testing/src/evaluation-telemetry.ts` — `recordEvaluation({name, -score, label?, span?})` returns `{ "gen_ai.evaluation..score": -number, "gen_ai.evaluation..label"?: string }` and, when handed a - `SpanHandle`, calls `span.setAttributes(...)` — i.e. it rides the SAME - span as the generation it grades, it does not create a new one. -- No runner: `packages/cli/src/commands/` has no `eval` directory; `cli.ts` - registers 30 commands, none named `eval`. -- No Studio surface: `packages/studio/src/features/` has no `evals/`; nothing - under `packages/studio/src` references `gen_ai.evaluation`. -- `@lunora/cli`'s `package.json` does not depend on `@lunora/testing` today — - adding the runner requires one new dependency edge. - -## 2. Existing seams this reuses (do not reinvent) - -- **Command shape**: `verify` (`packages/cli/src/commands/verify/{index,handler}.ts`) - is the closest sibling — a read-only, offline-safe command with `--format -pretty|json` (`util/output-format.ts`: `validateOutputFormat`, - `loggerForFormat`, `isJsonFormat`, `printJson`) and a testable - `run*Command(options): Promise<{code, ...}>` function wrapped by - `defineHandler` (`util/command.ts`) for the cerebro `execute`. `lunora eval` - copies this shape exactly. -- **File discovery**: `packages/codegen/src/discover-functions.ts`'s - `listLunoraSourceFiles` — `readdirSync` + `lstatSync` (never `statSync`, so - a directory symlink cycle isn't descended into), skipping `node_modules`/ - `_generated`. `discover-eval-files.ts` mirrors this walker verbatim, scoped - to `*.eval.ts`. -- **Contrast — commands that DO need a live worker**: `seed` and `insights` - both require `--url`/`--token` against a running Worker (`seed/index.ts`: - "bulk-insert it via the worker's admin endpoint"; `insights/index.ts`: - "Report... from a running Worker"). `lunora eval` is deliberately NOT this - shape — see §4. - -## 3. The `*.eval.ts` discovery convention - -### Location: top-level `evals/`, not `lunora/**/*.eval.ts` - -**Recommendation: a top-level `evals/` directory**, sibling to `lunora/`, -walked recursively (`evals/**/*.eval.ts`). Rejected alternative: -`lunora/**/*.eval.ts`. - -Why not inside `lunora/`: `lunora/` is codegen's domain — every `.ts` file in -it (barring `schema.ts` at the root and anything under `_generated/`) is -walked by `listLunoraSourceFiles` and parsed as a candidate function module -(`discover-functions.ts`). An eval fixture is not a function module, but nothing -today teaches codegen to skip a `*.eval.ts` suffix, so dropping evals inside -`lunora/` risks a `.eval.ts` file being picked up as a broken/empty function -source the first time someone imports something codegen's static walk doesn't -expect, or at minimum adds a codegen-side carve-out for a concept codegen has -no reason to know about. `evals/` at the project root mirrors the existing -`__tests__/` convention (test-shaped code lives in its own top-level -directory, sibling to the source it exercises) and needs zero codegen changes. - -### The eval module shape: a single default export, `{ name?, threshold?, run }` - -```ts -// evals/support-triage.eval.ts -import { agentHarness, containsScorer, evaluate, finalTurn } from "@lunora/testing"; -import { supportAgent } from "../lunora/agents"; - -export default { - name: "support-triage", // optional — defaults to the filename minus `.eval.ts` - threshold: 0.8, // optional — overrides `--threshold` for THIS eval only - run: async () => { - const harness = agentHarness(supportAgent, { - script: [finalTurn("Your refund was issued.")], - }); - - return evaluate([{ expected: "refund", input: "where's my refund?" }], async (input) => (await harness.run({ input, threadKey: input })).text ?? "", [ - containsScorer("refund"), - ]); - }, -}; -``` - -- **Single default export** — the repo-wide rule ("never mix default + named - exports; default is fine only as the sole export", `CLAUDE.md`) makes this - the only compliant shape that also carries per-eval metadata (`name`, - `threshold`) alongside the executable body. A bare `export default async () -=> EvalResult` (no metadata) was considered and rejected: it can't carry a - per-eval threshold without a second named export, which the mixed-export - rule forbids. -- **`run` is the ENTIRE eval body** — the runner does not call `evaluate` - itself, does not construct scorers, does not touch `agentHarness`. The file - imports what it needs from `@lunora/testing` and returns whatever - `EvalResult` its own call to `evaluate(...)` produces. This is what makes - "the kit is the product, the runner is glue" literal rather than aspirational: - the runner's contract with a `*.eval.ts` file is exactly `() => -Promise | EvalResult`, and `EvalResult` is the kit's own - existing exported type — nothing new is added to `@lunora/testing`. -- **No `defineEval` helper (yet)** — the codebase's `define*` convention - (`defineSchema`, `defineAgent`, `defineWorkflow`, …) would suggest a - `defineEval` identity-typed helper in `@lunora/testing` for authoring DX - (autocomplete, inline type errors on `run`'s return type). This is - deliberately **not** added in this spike: the prototype's `evals/*.eval.ts` - file structurally satisfies `EvalModule` (a type-only interface living in - `@lunora/cli`, not in the kit) with zero coupling back into - `packages/testing/src`, which is the strongest form of "reuses the kit - UNCHANGED" this design could produce. If the shape holds up, promoting it to - a real `defineEval` in `@lunora/testing` is a one-file, additive follow-up — - flagged as an open question in §6, not decided here. - -### Discovery mechanics (prototype) - -`packages/cli/src/commands/eval/discover-eval-files.ts` — `readdirSync` + -`lstatSync` recursive walk under `--dir` (default `evals/`), collecting files -whose name ends in `.eval.ts`, skipping `node_modules`/`_generated`/`dist`/ -`.git`, sorted for a deterministic run order. Absence of the directory is not -an error (mirrors `verify`'s "no tsconfig.json → warning, not error"): `lunora -eval` prints an info line and exits 0. - -## 4. The CI contract: `--threshold`, exit codes, `--format json` - -Mirrors `verify` exactly: - -- `lunora eval` — runs every discovered eval, prints a table (`NAME SCORE -THRESHOLD STATUS`), exits **0** if every eval ran without throwing (no - threshold set → report-only; this is the "just show me the scores" mode). -- `lunora eval --threshold 0.8` — a **global** gate: every eval's `.average` - must be `>= 0.8`, unless the eval's own `threshold` export overrides it - (per-eval wins). Exits **1** if any eval falls below its effective - threshold, or if any eval file throws while loading or running (a crash is - always a failure, independent of thresholds). -- `--format json` — mirrors `verify`'s contract bit-for-bit: every - human/progress line moves to stderr (`loggerForFormat`), stdout carries one - JSON document (`printJson`) shaped `{ code, evals: [{ name, path, average?, -threshold?, passed, error?, items? }] }`. Machine-readable, pipeable to - `jq`, and the shape a CI step greps for a specific eval's score without - re-running anything. -- **No per-eval exit code** — one process, one exit code, exactly like every - other Lunora CLI command. A per-eval breakdown lives in the printed table / - JSON body, not in the exit code's bits. -- Threshold granularity is **global by default, per-eval override** — not - "global XOR per-eval." An app with nine settled evals and one new, - known-noisy one sets `--threshold 0.85` globally and `threshold: 0.5` on the - noisy file, rather than choosing one mode for the whole run. - -## 5. Does the runner need a live worker? No — harness-only, and that is load-bearing - -`agentHarness` (§1) is a pure in-memory double: a scripted `AgentGenerate`, -an in-memory `agents:*` runtime, a `DurableStepJournal` mimicking Workflows' -`step.do` memoization. `evaluate`'s `produce` callback is caller-supplied — -nothing in the kit reaches for `fetch`, a Durable Object stub, or a Worker -binding. Verified by reading `agent-harness.ts` and `scorer.ts` in full: **zero** -references to `fetch`, `WebSocket`, or any binding type. - -This means `lunora eval` runs entirely in the CLI's own Node process — no -`wrangler dev`, no `lunora dev` running in another terminal, no `--url`/ -`--token` like `seed`/`insights` need. That is a real, load-bearing property -for the CI story: a GitHub Actions step can run `pnpm --filter exec -lunora eval --threshold 0.8` with no Worker to stand up first, no port to -wait on, no `wrangler dev` flake. This is the harness-only path the plan -asked to confirm or refute — **confirmed harness-only**, no live worker -required, and the design leans on that: nothing in §3–4 assumes a URL/token -option exists. - -(An eval whose `produce` calls `ctx.ai`/a real model provider still needs -network egress and an API key — that is a property of the eval author's own -`run()` body, identical to any Vitest test that happens to hit a real API. It -is not something the runner arranges or gates.) - -### An implementation gap this spike surfaces (not a STOP, but real) - -The prototype's discovery step (`discoverEvalFiles`) finds `*.eval.ts` files; -running them requires actually **executing** TypeScript source, not just -parsing it (unlike codegen, which only ever statically walks `lunora/*.ts` -with `ts-morph` — see `discover-functions.ts` — and never runs user code). -The prototype does a plain `await import(pathToFileURL(file).href)`. Under -Vitest (this is how the CLI's own test suite exercises the fixture — see §7) -this works transparently: Vitest's module runner transforms any `.ts` it -resolves, including one reached via a dynamic `import()` at runtime, not only -statically-imported test files. Under the **compiled** `lunora` binary running -on plain Node, it works only on Node's built-in TypeScript type-stripping, -which is opt-in (`--experimental-strip-types`) on the `^22.15.0` floor this -repo's `engines` field allows and only unflagged by default starting Node -23.6 — so on a real `^22.15.0` install, importing a `.ts` eval file from the -shipped binary throws `ERR_UNKNOWN_FILE_EXTENSION` today. This is flagged as -an open question in §6, not solved here — the prototype's own test runs under -Vitest specifically to sidestep it and prove the discovery/threshold/aggregate -logic, which is this step's actual deliverable. - -## 6. Studio Evals panel: data source, and a gap the panel design surfaces - -### Where `gen_ai.evaluation.*` actually lives - -`recordEvaluation` attaches its attributes to whatever `SpanHandle` the caller -hands it — the same post-hoc handle a `ctx.trace(name, (trace, span) => …)` -body receives (`evaluation-telemetry.ts:11-16`). That span is folded into the -shard's trace ring and surfaced today by the existing `__lunora_admin__:getTraces` -RPC (`packages/studio/src/lib/admin.ts:1014-1083`, consumed by -`TracesPanel`, `packages/studio/src/features/traces/traces-panel.tsx`) — a -`TraceSummary.spans[]` entry's `attributes` is exactly where a -`gen_ai.evaluation..score` key would show up. **No new RPC is needed for -a v1 panel** — `getTraces` already returns the data. - -### The gap: that store is a live, bounded, in-memory ring — not history - -Both `admin.ts`'s own doc comments are explicit about this, independently, for -the two candidate stores: - -- `TraceSummary` (`admin.ts:1045-1048`): "Sourced from the shard's bounded, - in-memory span ring, so it resets on hibernation/restart... a 'recent traces - on this instance' readout for local development, **NOT a durable trace - store**." -- `MetricSeries` (`admin.ts:1095-1098`): same story for `ctx.metrics` — "A - 'recent metrics on this instance' readout for local development, **NOT a - durable metric store**." - -There IS a durable, historical, per-minute-bucketed store already shipped — -`MetricHistoryPoint`/`getMetricHistory` (`admin.ts:1140-1159`, "persisted in -the shard's SQLite so it survives hibernation... ready to chart as a trend -line"). But it durably tracks `ctx.metrics.*` series (counter/gauge/ -histogram), not trace-span attributes — and `recordEvaluation` today only -ever emits span attributes, never a metric. So the plan's ask ("groups by eval -name + score over time") runs into a real seam mismatch: the durable, -time-bucketed store exists, but eval scores don't flow into it; the store eval -scores DO flow into (trace spans) has no durable history. - -**This is the STOP-shaped condition the plan called out** ("the -`gen_ai.evaluation.*` spans aren't queryable from the studio trace store the -way the panel needs") — but it resolves to a **documented gap + a v1 that -works within it**, not a hard stop, because `getTraces` genuinely does return -the data for a live/recent view; it just can't back a trend chart. - -### Recommended v1 panel (design-only in this spike, not built — see §7): live, not historical - -Model on `TracesPanel` directly (same admin query hook, same shard-scoped -live-push pattern): - -- **Query**: `useAdminQuery(ADMIN_FUNCTIONS.getTraces, {}, { live: true, -shardKey })` — identical call to what `TracesPanel` already makes. -- **Extraction**: client-side, over the returned `TraceSummary[]`. For each - span in each trace, scan `attributes` for keys matching - `/^gen_ai\.evaluation\.([^.]+)\.score$/`; the capture group is the eval - name, the value the score; a sibling `.label` key (same name) is optional - metadata. This is a pure function, exactly like `TracesPanel`'s existing - `filterTraces`/`spanBar` helpers in `trace-geometry.ts` — same file-shape - precedent. -- **Grouping/layout**: group extracted `(name, score, traceId, spanName, -startTs)` rows by eval `name` into a card per eval — most-recent score, - a small run count, a "min/mean/max over what's currently in the ring" - summary (NOT a trend line — the ring doesn't support one), and a list of - recent runs each linking to its trace (reusing the existing trace-id - hand-off `lib/trace-handoff.ts` the Traces panel's exemplar links already - use, so "open in Traces" is a real, already-built primitive, not new work). - Same `EmptyState` message pattern as `TracesPanel` for the zero-evals-seen - case. -- **Explicit framing in the UI**: reuse `TracesPanel`'s own caveat text - verbatim in spirit — "recent scores on this instance, not a durable - history" — so the panel doesn't imply a trend guarantee the data can't back. - -### Follow-up (not this spike): durable trend requires a second emission - -If a durable "score over time" chart is wanted, `recordEvaluation` would need -an ADDITIVE second path — emit through `ctx.metrics.gauge` (or a histogram) -alongside (not instead of) the span attributes, so `getMetricHistory`'s -already-durable per-minute buckets carry eval scores too and the panel can -build on `metrics-aggregate.ts`/`sparkline.tsx` (the existing trend-chart -building blocks in `packages/studio/src/features/reports/`) instead of -inventing one. This is exactly the kind of "changing the harness API" the -plan's SCOPE excludes for this spike (`recordEvaluation`'s signature/behavior -would need to grow), so it is recorded here as the concrete next step, not -attempted. - -## 7. What actually ships in this spike - -- This design doc. -- A working CLI prototype: `packages/cli/src/commands/eval/` (`index.ts`, - `handler.ts`, `discover-eval-files.ts`, `types.ts`), registered in - `cli.ts`, `@lunora/testing` added as a real dependency of `@lunora/cli`. - `runEvalCommand` is directly unit-testable (mirrors `runVerifyCommand`). -- A CLI test (`packages/cli/__tests__/commands/eval.test.ts`) that runs a real - fixture `evals/*.eval.ts` file (using actual `@lunora/testing` exports: - `evaluate`, `containsScorer`) through `runEvalCommand`, asserting both the - aggregate table/JSON output and that a below-threshold run exits non-zero. -- The Studio Evals panel is **design-only** in this spike (§6) — not - prototyped as code. Rationale: the panel's honest v1 shape depends on the - gap analysis in §6, which is itself the deliverable STEP 3 asked for - ("otherwise the design doc details the data query + layout"); building a - panel against the wrong data-source assumption (a trend chart the ring - can't support) would need to be redone once the gap was found, so the spike - stops at the design once the gap surfaced, per STEP 3's own stated - fallback. - -## 8. Open questions (STEP 4) - -1. **`defineEval` helper** — promote the `{ name?, threshold?, run }` shape - into a real `defineEval` export in `@lunora/testing` for authoring DX - (return-type inference on `run`, discoverable via the package's own - autocomplete)? This spike deliberately kept `@lunora/testing` at a - zero-line diff; a follow-up plan can add it additively once the shape has - seen real use. -2. **Threshold granularity** — resolved in §4 (global default, per-eval - override) — flagged here because it's a public CLI contract, worth a second - look before it ships (a `--threshold-mode strict|any` a la "all must pass" - vs "aggregate average must pass" was considered and rejected as premature — - the per-file `threshold` override already covers the "this one eval is - noisier" case without a second flag). -3. **Live worker** — resolved in §5: no, harness-only. The remaining open - question is narrower — the TS-execution mechanism for the compiled binary - (§5's gap): Node `--experimental-strip-types` re-exec vs. an `esbuild` - transform-then-`import()` (would promote `esbuild` from `@lunora/cli`'s - existing devDependency to a real one — no other `@lunora/*` package ships - it as a runtime dep today, confirmed by grep) vs. requiring the app's own - `vitest`/`tsx`/`jiti` be present and shelling out to it. Recommend the - `esbuild`-transform path (self-contained, no assumption about what the - consuming app has installed) as the next step's first thing to prototype. -4. **Panel: live-tail vs. historical** — resolved in §6: live/recent only for - v1 (matches what `getTraces` can actually back); a durable trend view is a - distinct follow-up gated on also emitting through `ctx.metrics` (§6's - "Follow-up" — not this spike, not decided here beyond "additive, don't - change `recordEvaluation`'s existing signature, only extend it"). -5. **`evals/` vs. per-package location for a monorepo app** — this design - assumes one `evals/` per Lunora project root. An app with multiple - `apps/*` each embedding Lunora would run `lunora eval` per-app, same as - every other per-app CLI command (`verify`, `deploy`, …) — not a special - case, but worth confirming against a real multi-app repo before this ships. diff --git a/plans/333-query-snapshot-coherence.md b/plans/333-query-snapshot-coherence.md new file mode 100644 index 0000000000..11910f5406 --- /dev/null +++ b/plans/333-query-snapshot-coherence.md @@ -0,0 +1,76 @@ +# Plan 333 — Snapshot coherence for query subscriptions + +- **Category**: correctness (user-visible torn reads) +- **Status**: DESIGN — a client-only prototype was built, measured, and reverted; the real fix needs a server-side batch boundary +- **Related**: plan 168 (cross-shard transactions), `concepts/realtime`, `non-goals.mdx` + +## The gap + +Convex guarantees that every query subscription on a client reflects **one +consistent snapshot at a single logical timestamp**, and that they advance +together. Lunora does not: each `data`/`delta` frame notifies its own +subscribers the moment it arrives. + +One mutation routinely moves more than one query — a message insert advances the +thread list AND the unread count. The shard re-runs both and writes both frames +in one pass, but the client applies them independently, so a UI reading both can +paint the new list beside the old count. Each value is individually correct; +together they describe a state the database was never in. + +This is the half of plan 168 that is **not** about cross-shard writes. It bites a +single-shard app with no `.shardBy()` at all. + +## What was tried, and why it was reverted + +A client-side coherence window: buffer server-frame updates and flush them on the +next microtask so everything from one batch lands in one commit. Optimistic +updates deliberately stayed synchronous (deferring a local write is the exact +latency optimism exists to hide). + +It worked, and it was reverted, for two reasons: + +1. **It changes the delivery contract of every subscription in the framework.** + 21 client tests failed immediately — all of them asserting a value + synchronously after simulating a frame. Those are not bad tests; they encode + the contract that a frame's value is readable as soon as the frame is handled. + Every adapter (`react`, `vue`, `svelte`, `solid`, `angular`) and `@lunora/db` + sit on that contract too. +2. **It buys little that React does not already give.** The frames it coalesces + are the ones arriving in the same tick, which React 18's auto-batching already + folds into one render. The tearing that actually reaches users is frames + arriving in _different_ ticks, which a microtask window cannot merge. + +## What the real fix looks like + +Query subscriptions need what shapes already have: an explicit batch boundary on +the wire. The poke protocol brackets a batch of shape diffs with +`pokeStart`/`pokeEnd` and the client applies the whole batch atomically +(`protocol/README.md` §5.3). Queries have no equivalent — they carry a `cursor` +but nothing says "these three frames are one commit". + +Sketch: + +1. **Server.** `flushChangedTables` already re-runs every affected subscription + for one write in a single pass. Bracket that pass: emit + `{ type: "batchStart", cursor }` before the frames and `{ type: "batchEnd", cursor }` + after, on each socket that receives at least one frame. +2. **Client.** Buffer frames between `batchStart` and `batchEnd`, then apply them + in one commit. Frames outside a batch keep today's synchronous path, so the + contract only changes where a batch is genuinely in flight — which is what + keeps the existing tests (and adapters) honest. +3. **Boundary.** The guarantee is per-shard, because that is where the atomic + commit is. A subscription set spanning two shards or D1 still advances in two + commits; `non-goals.mdx` already documents that boundary and should say so for + reads too, not just writes. + +## Open questions + +1. **Backpressure.** A slow client that never drains a batch must not let the DO + buffer indefinitely. The shape poke path already faced this — reuse its answer + rather than inventing a second one. +2. **Partial batches.** A socket that disconnects mid-batch must not apply half + of it; the client should drop an unterminated buffer on reconnect and re-seed + from the resume cursor. +3. **Is `cursor` sufficient as the batch identity**, or does a batch need its own + id the way a poke does? A single write is one cursor, but a coalesced refresh + (`flushChangedTables` merges pending tables) may span several. diff --git a/plans/README.md b/plans/README.md index 8dad5bca74..a3a823afe0 100644 --- a/plans/README.md +++ b/plans/README.md @@ -1162,16 +1162,45 @@ its §4 decisions are deliberately unmade pending a measurement) · **314** an English NLP library in every Worker (TODO, P2). Reference design docs gating unbuilt follow-on work: **052** streaming hook · +**333** query snapshot coherence (client-only prototype built, measured, and +reverted — the real fix needs a `pokeStart`/`pokeEnd`-style batch boundary for +query subscriptions; see the plan for why the microtask window was the wrong +rung) · **137** release-train rehearsal (feeds 135 Phase 3) · **162** crossTabSync relay · **234** node-host findings (spike DONE; the file was re-added by `370994075` and is kept as the host's reference) · **237** admin-auth hooks (one-adapter prototype; remaining adapters open) · **238** vector reader (codegen wiring + -`define-rag` refactor open) · **240** agent run queue · **241** in-app inbox (D1 -backend open) · **242** agent reply · **245** eval runner (CLI shipped; **Studio -Evals panel verified NOT built**) · **247** event store (not ratified) · +`define-rag` refactor open) · **241** in-app inbox (D1 +backend open) · **247** event store (not ratified) · `execution-status.md` (wave-18 dispatch tracker) · `multi-platform-portability-assessment.md`. +**245** (eval runner) SHIPPED & REMOVED: the CLI runner was already in; the +Studio **Evals** panel now exists, and the spike's "durable trend needs a second +emission" follow-up shipped with it — `recordEvaluation` takes an optional +`ctx.metrics` handle and records the score as a gauge alongside the span +attributes, so the panel charts a real trend from `getMetricHistory` instead of +a ring that empties on hibernation. Plan file deleted; record in git history. + +**242** (agent reply) SHIPPED & REMOVED: the outbound half of an inbound +trigger exists — `AgentReplyRef` (all four channels) rides the run params, +`defineAgent({ onReply })` fires once on the final answer inside the durable +`agent:reply` step, and `@lunora/agent/reply` ships the email capture + threaded +send. The spike's Slack/GitHub/Discord credential gap is closed by handing +`onReply` the worker `env` rather than inventing a token store: the framework +knows where to reply, the app holds the credential. Prototype + plan file +deleted; record in git history. + +**240** (agent run queue) SHIPPED & REMOVED: `onConcurrentRun: "queue"` is now a +real durable FIFO queue (`agent_run_queue` + `agentCompleteRun`'s +dequeue-in-the-same-mutation handoff, parked runs hibernating on a scoped +`waitForEvent`), so the value no longer degrades to `"reject"`. The spike's +open questions were closed as: depth capped at 5 with no config surface (YAGNI), +lost-wake covered by a durable send step plus the waiter's own timeout (the +`fan-out.ts` pattern), no client-facing queued status, and `"replace"` +supersedes the run in flight without flushing the queue. Prototype + plan file +deleted; record in git history. + Retired since this list was last written: **235** (progressive sharding) and **248** (runtime lints), deleted by `7c3d3f42e`; **311** (the bigint backfill), which shipped inside #365 rather than trailing it; and **265** / **302**, both @@ -1563,6 +1592,122 @@ verify`_ is YAGNI — an agent can run `pnpm test` itself; the half with real value (live function-spec parity against the deployed worker) is a candidate for a later wave. +## Wave 21 — all-package sweep (baseline `70b7451b5`, 2026-08-11) + +First all-package sweep since Wave 18 (`071c6a29c`, ten days earlier). Five read-only +audit agents (core runtime, security surfaces, tooling packages, tests+architecture, +deps/DX/docs/direction), every surviving finding re-read against live code before it +reached a plan — five subagent claims did not survive that pass and are recorded under +"considered and rejected" below. + +The headline is that the two most valuable findings are both **cache-scoping** bugs of +the same family: a client-side map keyed by less than the server keys it (316), and an +in-memory baseline with no durable backing where its own sibling has one (326). Both +were invisible to every existing test because both degrade rather than fail. + +Deliberately skipped: the uncommitted `durable-stream.ts` work in +`packages/shard-engine` and the in-flight diffs in `packages/server/src` — another +session is mid-feature there. + +| Plan | Title | Category | Pkg | Pri | Effort | Risk | Status | +| ---- | ------------------------------------------------------------- | -------------- | ----------------------- | --- | ------ | ---- | ------------------------------------------------------------ | +| 316 | Scope the client mutation watermark by identity | bug | client/db | P1 | S | LOW | TODO — [316](316-client-watermark-identity-scope.md) | +| 317 | Route every `.dev.vars` write through the atomic 0600 writer | security | cli | P1 | S | LOW | TODO — [317](317-dev-vars-atomic-write.md) | +| 318 | Stop the Vite dev loop emitting from a stale TS program | bug | vite/codegen | P1 | S | LOW | TODO — [318](318-vite-stale-codegen-project.md) | +| 319 | Bound the dispatch call; classify deterministic 4xx | bug | dispatch/workflow/queue | P1 | S–M | MED | ✅ **DONE (WS1+WS3+workflow half) — APPROVED, awaiting merge. Queue half BLOCKED by its own STOP, correctly.** Worktree `.claude/worktrees/agent-aaba9f993d37eac1d`, branch `advisor/319-dispatch-timeout-and-retry-class`, commits `bf4a23863` + `970244c04`. Reviewer re-ran both suites: dispatch 27/27, workflow 81/81. The safety invariant holds — `DETERMINISTIC_DISPATCH_STATUSES = {400,403,404,422}` with `408`/`429` excluded in the constant's own comment, and the abort error is minted at **503**, deliberately outside that set, so a timeout stays retryable. **Queue half stopped for the sanctioned reason:** `dispatchQueueBatch` unconditionally rethrows the handler's error (`if (threw) { throw handlerError; }`), `deadLettered` is post-hoc metadata rather than a mechanism, and an existing test pins the rethrow as the deliberate contract — so ack-without-retry would be a redelivery-semantics change, not a bug fix. **§9 Q1 remains unconfirmed**: this implements the docblock's contract; if the maintainer prefers today's retry-everything behaviour, drop commit `970244c04` and edit the docblock — `bf4a23863` (the timeout) stands either way. **Follow-up it surfaced:** `@lunora/queue` re-exports `RunFunctionOptions` from dispatch so it picked up `timeoutMs` automatically, but `@lunora/workflow` declares its own copy in `workflow/src/types.ts`, so `timeoutMs` is invisible on a workflow step's typed `ctx.run` even though the runtime honours it — [319](319-dispatch-timeout-and-retry-class.md) | +| 320 | Cover the Stripe/Polar money-mutating methods | tests | payment | P1 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a43c3ad10c64a298f`, branch `advisor/320-stripe-polar-money-tests`, commit `d9314a192`. 16 new tests (8 per provider), 195/195 pass; reviewer re-ran coverage independently: `stripe.ts` branches 43.39% → **65.09%**, `polar.ts` 33.67% → **60.2%**, aggregate 68.8%, floor raised 60 → 68. `packages/payment/src` untouched (verified). **It found three latent defects on the money path and correctly pinned current behaviour rather than fixing it** — each needs its own plan: (1) **Stripe `updateSubscription` with an empty `items.data` sends `id: undefined`**, which Stripe treats as *creating a new subscription item* rather than updating one — a silent billing split, not a crash, which is why the plan's "if it throws" STOP never fired (`providers/stripe.ts:406`); (2) **Polar's `refundPayment` never returns `partially_refunded`** — it has no `compareMoney` boundary check at all and always reports `refunded` (`providers/polar.ts:289-315`); (3) **Polar's `updateSubscription` silently drops `quantity`** — a quantity-only patch produces an empty `subscriptionUpdate: {}` no-op (`providers/polar.ts:338-342`). Also recommended: `resumeSubscription` coverage for Creem/Dodo (both at 0) — [320](320-stripe-polar-money-tests.md) | +| 321 | Restore the coverage floor on six packages; fix the generator | tests/dx | repo | P2 | S | LOW | TODO — [321](321-coverage-floor-gaps.md) | +| 322 | Make the randomized test seed actually randomize | tests | repo | P2 | S→M | MED | TODO — [322](322-vitest-shuffle.md) | +| 323 | Run `dist:check` on pull requests | dx/ci | ci | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a7678ba2e1d9b517e`, branch `advisor/323-dist-check-pr-gate`, commit `9c81e74d6`, one file. **The gate was proven to fail before it was wired:** `dist:check` against a development build exits **1** with **180 files** flagged (`react/jsx-dev-runtime` and `jsxDEV` markers in `@lunora/react` and `@lunora/studio` output); against a clean production build it exits **0** ("1282 shipped files across 54 packages carry no development-build markers"). So the alpha.31 defect class does reach a dev build today and currently escapes every PR job. **Accepted deviation, and it is the difference between working and decorative:** the executor also added `dist-production` to `lint-required-check`'s `needs` — reviewer verified the aggregate now lists it. Without that the job would report and never block, and this repo's branch ruleset requires only the aggregate checks. WS3 was deliberately **not** done: `CLAUDE.md:42`'s claim that both gates have CI jobs became true once the job landed, so editing it would be churn. **Unverified by construction:** the job has never been observed running in CI, and no `api-surface`-vs-`dist-production` duration comparison exists — local timings were taken under wildly different load and are not comparable — [323](323-dist-check-pr-gate.md) | +| 324 | Test the shared client mutation path; floor `@lunora/client` | tests | client | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a3e993f1a89ad1ba9`, branch `advisor/324-client-mutation-path-tests`, commit `0979c777c`. Reviewer re-ran: 38 files, 657 passed + 1 expected-fail. `mutation-runner.ts` and `snapshot-precondition.ts` both at 100% branches; floor pinned at 84/75/76/84 with `src/sw/**` excluded (§9 Q1 — a real ~11%-stmt gap, excluded with its reason in the config so later sw work doesn't inherit a floor it must fight). **Both proofs were actually performed, not asserted:** the ref-count test reds when `inFlight` is replaced with a boolean, and bumping `branches` to 99 fails the run with the threshold error. **Trap for anyone re-measuring:** the printed coverage table silently omits rows for both new files — read `coverage/lcov.info` (`BRH==BRF`), not the table. §9 Q2: `src/auth/index.ts` (0% across 95 lines of client auth surface) deserves its own plan — [324](324-client-mutation-path-tests.md) | +| 325 | Test the native-signup email gate | tests/security | auth | P2 | S | LOW | TODO — [325](325-email-gate-tests.md) | +| 326 | Make the shape poke baseline survive hibernation | perf | do/shard-engine | P2 | M | MED | TODO — [326](326-shape-poke-cursor-durability.md) | +| 327 | Manifest hygiene: unused dep, prod-vs-dev dep, catalogs, hono | deps | repo | P3 | S | LOW | TODO — [327](327-manifest-hygiene.md) | +| 328 | Stop recording an attacker-chosen IP in the auth audit trail | security | auth | P3 | S | LOW | TODO — [328](328-audit-log-ip-fallback.md) | +| 329 | Publish the Node host's capability matrix | docs | platform-node | P3 | S | LOW | TODO — [329](329-platform-node-docs.md) | +| 330 | Let Angular produce the preloaded query it can consume | direction | angular | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-ae58d3596d59bc8a0`, branch `advisor/330-angular-preload-query`, commit `30ea1ffe4`. `packages/angular/src/server.ts` is a pure re-export of `@lunora/client/ssr` (verified against that module's live surface, not Solid's copy), `./server` added to the exports map, `dist/server.{mjs,d.ts}` emitted, 108 tests pass, key-order gate clean, `api:check` unchanged (angular is untiered). Reviewer re-ran the criteria independently. **Deviation, accepted:** the export-parity test asserts against `@lunora/client/ssr` rather than `@lunora/solid/server` — strictly stronger (it catches Solid drifting too) and avoids a first-of-its-kind adapter-to-adapter dependency edge. **`templates/analog/` was documented, not wired** — its demo still uses a hand-rolled `LunoraService` over the vanilla client, and swapping it is a separate migration. Two follow-ups opened: an API-snapshot tier decision for `@lunora/angular` now that it publishes a second entry point, and `angular.mdx`'s "Deliberately small surface" callout, which undersells an adapter that already exports `paginatedQuery`/`presence`/`subscription`/`rateLimit`/`stream`/`agent*`/`flag*` — [330](330-angular-preload-query.md) | +| 331 | Resolve the `/worker` subpath contradiction | direction | vue/svelte | P3 | S–M | LOW | **PHASE 0 DONE — awaiting a maintainer decision before Phase 1A.** Evidence in §9, reviewer-verified against the live tree: the subpath has **zero** real importers (the only hit repo-wide is its own docblock example), and `withFrameworkWorker` itself is reached only by codegen's `buildFrameworkWorker` → `@lunora/runtime`, never through a framework `/worker` subpath — the Astro and SvelteKit templates both bypass it, leaving `templates/sveltekit`'s prose stale. `cb6df0762` shows the same defect was already caught and half-fixed once. Recommendation: **Option A** (delete both re-exports, promote `withFrameworkWorker` as the documented BYO-framework entry). Phase 1A removes a published surface, so it is not an executor's call — [331](331-worker-subpath-decision.md) | +| 332 | Spike: what would a payment conformance suite assert? | direction | payment | P3 | S | LOW | TODO (spike) — [332](332-payment-conformance-spike.md) | + +### Dependency notes + +- **332 depends on 320.** The shared-assertion inventory is far easier to read once all + six providers have comparable coverage, and 320's cases are half its input. +- **321 before 324's floor step and before 322.** 321 owns floor policy; 324 adds one + package's thresholds and 322 changes what a green run means. Landing them out of + order produces two conflicting edits to the same configs. +- **326 has a gating measurement (WS0).** If the cold-memo rescan turns out cheap at + realistic CDC retention, the plan's own STOP fires and the one-line + attachment-`sinceSeq` fallback is the whole fix. Do not start WS1 before WS0. +- **331 is a decision, not an implementation.** Its Phase 0 is three greps and a + maintainer call; the two outcomes touch different files. +- 316 and 324 both touch `@lunora/client` but not the same modules (`lunora-client.ts` + vs `mutation-runner.ts` + `vitest.config.ts`). They can run in parallel; 316 first if + they must be serialized. + +### Findings considered and rejected (Wave 21) + +Vetted against live code and dropped — recorded so they aren't re-audited: + +- **"Jurisdiction handling is duplicated across five packages and has already + drifted."** The duplication is real and **deliberate**, and it is guarded: + `packages/runtime/__tests__/jurisdiction-union-consistency.test.ts` pins all five + copies (`runtime`, `scheduler`, `server`, `mail`, `container`) to the canonical + `"eu" | "fedramp" | "us"` literal, with the decoupling rationale in its header + comment. The "widening union" cited as evidence of drift was `ShardJurisdiction` in + `@lunora/platform` — a different type. The residual sliver (five hand-rolled + `namespace.get(idFromName(…))` adapters that bypass `ShardDirectory`) is a real + portability observation, but it is a decoupling trade the repo made knowingly, not a + defect. +- **"The `hono@4.12.34` pin is in the wrong YAML key."** It sits under + `minimumReleaseAgeExclude`, which is exactly the key that waives the release-age + hold; `undici`, `js-yaml` and `dompurify` are there for the same reason. Only the + stale lockfile is real, and that is plan 327 WS4. +- **`applyPlanToCollections` partial-apply** (`packages/db/src/apply-plan.ts:133`). It + is an exported helper, its keyless-insert throw is already tested + (`db/__tests__/apply-plan.test.ts:88`), and the half-applied state requires calling + it outside a TanStack optimistic transaction. A one-line reorder if anyone is in the + file; not a plan. +- **"~14 peer ranges hardcode a catalogued version."** A peer range is a _supported_ + range, deliberately independent of the version we build against — `react ^19.2.7` + already admits the catalogued `19.2.8`, so nothing lies. Narrowing peers to catalog + values would shrink what consumers may install. `@lunora/auth` and `@lunora/vite` + using `catalog:` in peers is a per-case judgement, not a rule to generalise. +- **"No watch mode in 54 of 55 packages."** MED confidence, no evidence the loop is + actually painful; the repo's cross-package loop goes through `vis` anyway. +- **`pg`/`mysql2`/`postgres` `*` peer ranges in `@lunora/hyperdrive`.** Thin — the peers + are already `optional: true`, so a narrower range produces a warning, not a fix. +- **Two transitive high advisories, checked and un-upgradable from this repo:** + `ip-address` 10.2.0 reaches `mcp`/`agent` only via + `@modelcontextprotocol/sdk > express-rate-limit`, and Lunora serves MCP through + `WebStandardStreamableHTTPServerTransport` (`packages/mcp/src/serve-stateless.ts:14`), + never the Express path; `image-size` 2.0.2 reaches `storage` via + `@visulima/storage > @netlify/blobs`, and Lunora instantiates `AwsLightStorage` + (`packages/storage/src/upload-handler.ts:22`), never the Netlify provider. +- **Verified clean under direct read** (no findings): the wire codec round-trip + (`shared/wire-codec.ts`, `sql-projection.ts`, `serialize-sql.ts`, `index-key-codec.ts` + — args stay wire-tagged worker→shard and are only `encodeWire`'d on the + server-initiated path, so no double-encode); OCC/idempotency transaction boundaries + and the batch replay path; read-set/range invalidation (every uncertainty path is + conservative); the client socket lifecycle (superseded-socket guards, heartbeat and + timer cleanup); `session-do.ts`; `resolve-shard.ts`; the whole `/_lunora/admin/*` + plane (every extracted route cluster reaches `assertAdminAuthorized`, including the + clusters with no textual `isAdmin` hit, and `auth-admin-routes.ts:694` gates _before_ + the method check so route topology does not leak); identity-header smuggling on both + the fetch and WS-upgrade paths; constant-time token comparison throughout; payment + webhook verification on both schemes; x402's fail-closed pay-side policy; + `@lunora/storage` signed-URL binding with CR/LF canonical-injection guards; + `@lunora/cloudflare-access` RS256 pinning; CLI path handling (`fileSegment`, + `assertSafeItemName`, registry `from`/`to` traversal rejection); no committed secrets. +- **Not filed, noted only:** `packages/react/src/payment.tsx:6-23` hand-mirrors + `Subscription` with `provider: "polar" | "stripe"` while `payment/src/types.ts:31` + has five providers, and omits `currentPeriodStart`. Real drift, invisible to + `api:check` and `lint:types` because both sides compile in isolation — it motivates + plan 332 and is called out in that plan's out-of-scope list. Worth a two-line fix + next time anyone is in that file. Also: `packages/queue/src/metric-history.ts` writes + a `last_ts` column nothing reads (dead, not a defect), and `verifyDiscord` lacks the + replay-window check `verifySlack` has (thin — Discord signs the timestamp). + ## Filed outside a wave | Plan | Title | Category | Pkg | Pri | Effort | Risk | Status | diff --git a/protocol/README.md b/protocol/README.md index 1f94d6b076..ac3a568c1b 100644 --- a/protocol/README.md +++ b/protocol/README.md @@ -182,7 +182,7 @@ are ignored by the client parser. | `unsubscribe` | `{ type, id }` | | `shape_subscribe` | `{ type, id, shape: { name, args? }, sinceCheckpoint?, sinceEpoch? }` | | `shape_unsubscribe` | `{ type, id }` | -| `stream` | `{ type, id, query: { functionPath, args?, shardKey? } }` | +| `stream` | `{ type, id, query: { functionPath, args?, shardKey? }, sinceChunk? }` | | `whisper_subscribe` / `whisper_unsubscribe` | `{ type, topic }` | | `whisper` | `{ type, topic, data? }` | @@ -191,6 +191,14 @@ are ignored by the client parser. `sinceEpoch` ride along only on a resume. Subscription ids are conventionally `sub_`; shape ids `shape_`; stream ids `stream_`. +`stream.sinceChunk` is the **durable-stream** resume watermark and is unrelated +to `subscribe.query.sinceSeq` (a CDC cursor): it is the highest `chunk.seq` the +client already holds for this run. A run is identified by `(functionPath, args)`, +so re-sending the same start frame with `sinceChunk` re-attaches to the run +already in flight, replays the chunks after that watermark from the server's +transcript, and then continues live. A server that did not declare the procedure +durable ignores `sinceChunk` and emits chunks without `seq`. + The `connect` frame is sent once per socket open, **before** resubscribing, so `onConnect`/`onDisconnect` lifecycle hooks fire symmetrically. @@ -205,7 +213,7 @@ The `connect` frame is sent once per socket open, **before** resubscribing, so | `settled` | `{ type, id, cursor?, epoch?, lastMutationId? }` | write touched tables, byte-identical result; advance only | | `error` | `{ type, id?, error: { code?, message? }, message? }` | subscription/stream-scoped error (`4001`/`TOKEN_EXPIRED` = token expired) | | `complete` | `{ type, id }` | subscription/stream closed server-side | -| `chunk` | `{ type, id, data: }` | one streaming-query chunk | +| `chunk` | `{ type, id, data: , seq? }` | one streaming-query chunk (`seq` on a durable run only) | | `whisper` | `{ type, topic, data: , from? }` | ephemeral relay | ### 5.3 Shape poke protocol (partial replication) From b2b837a0799ebcefd61c2cbe692c8f65f4278def Mon Sep 17 00:00:00 2001 From: Daniel Bannert Date: Tue, 11 Aug 2026 11:06:14 +0200 Subject: [PATCH 02/10] fix: address the review findings on this branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts one feature and closes three leak paths the audit found. Deterministic mutations, reverted. `withDeterministicScope` captured and restored `globalThis` around every mutation. The soundness argument — a mutation holds the DO's single-writer gate — is per Durable Object, and workerd hosts many DOs of one class per isolate. Two interleaved scopes therefore corrupt the save/restore stack and leave the isolate permanently patched: a frozen clock and a throwing `fetch` for every shard, query, alarm, and telemetry flush on it. Reproduced, then reverted whole; the sound shape (per-context injection) is recorded in plans/334. Durable streams no longer cache an answer. A finished run was replayed to any later caller with the same arguments, which made the feature a 24-hour response cache — including for a failed run, re-served with no retry path. Sharing is now scoped to a run still IN FLIGHT; a caller resuming a transcript it already holds still gets the tail. An eviction no longer wedges the key either: a dead run with nobody resuming it is reclaimed rather than answering STREAM_INTERRUPTED until its TTL expires. Durable stream runs are per-identity. The run key now includes the socket's verified identity, so two users asking the same question never share a transcript. An attach never drives the handler, so a shared run would also skip the procedure's RLS middleware. Durable stream storage is bounded. Each run stores its own `ttl_ms` — one procedure's sweep was deleting every other procedure's transcripts — chunks are capped, the per-chunk run-row UPDATE is gone (nothing read it mid-run), and a sink whose socket refused a frame is dropped instead of retried per chunk. The agent run queue no longer leaks. A run that ended while still parked kept its slot forever, and five of those exhausted the depth cap permanently; the completion mutation now releases a non-owner's row. Ownership transfers before the wake is sent, so an instance terminated while parked left the thread owned by a workflow that would never resume; a thread untouched past the dequeue window is now reclaimable. The park moved inside the try block so a timed-out wait still runs the completion, and a failing run's error survives the handoff instead of being cleared. Token budgets charge oversized calls. `record` forwarded the raw count to the limiter, which throws past capacity — so the one call a budget exists to catch escaped it entirely and replaced the caller's error. Clamped to capacity. Also: the Node capability entry claimed durable streams it does not implement (now `unsupported`), the client trusts the server's `seq` rather than the caller's `durable` flag, `sinceChunk` is validated before it can truncate a stream, the RAG sync action is typed instead of `unknown`, and the Evals trace drill-down is actually wired. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018hGhKLpXkFj3aWUYhZmWQv --- api-snapshots/ai.api.md | 10 +- api-snapshots/errors.api.md | 8 +- api-snapshots/shard-engine.api.md | 60 +---- .../src/content/docs/concepts/realtime.mdx | 25 ++- apps/docs/src/content/docs/non-goals.mdx | 17 +- packages/agent/__tests__/run-queue.test.ts | 49 ++++ packages/agent/src/agent-loop.ts | 26 ++- packages/agent/src/component.ts | 47 +++- packages/ai/__tests__/rag/sync.test.ts | 2 +- packages/ai/src/rag/index.ts | 2 +- packages/ai/src/rag/sync.ts | 14 +- packages/client/src/lunora-client.ts | 10 + .../__tests__/shard-do.durable-stream.test.ts | 135 +++++++++-- packages/do/src/shard-do.ts | 211 +++++++++++++----- packages/errors/src/catalog.ts | 2 +- packages/platform/src/capabilities.ts | 4 +- .../ratelimit/__tests__/token-budget.test.ts | 13 ++ packages/ratelimit/src/token-budget.ts | 10 +- packages/server/src/types.ts | 16 +- .../__tests__/deterministic-scope.test.ts | 61 ----- .../shard-engine/src/deterministic-scope.ts | 118 ---------- packages/shard-engine/src/durable-stream.ts | 54 +++-- packages/shard-engine/src/index.ts | 6 +- .../studio/src/features/evals/evals-panel.tsx | 10 +- plans/334-mutation-determinism.md | 71 ++++++ plans/README.md | 38 ++-- 26 files changed, 625 insertions(+), 394 deletions(-) delete mode 100644 packages/shard-engine/__tests__/deterministic-scope.test.ts delete mode 100644 packages/shard-engine/src/deterministic-scope.ts create mode 100644 plans/334-mutation-determinism.md diff --git a/api-snapshots/ai.api.md b/api-snapshots/ai.api.md index a69344bbc2..7c193bdb65 100644 --- a/api-snapshots/ai.api.md +++ b/api-snapshots/ai.api.md @@ -167,6 +167,14 @@ _Tagged `@experimental` — signature not tracked; churn here does not fail the _Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ +### `RagSyncActionReference` (interface) + +```ts +interface RagSyncActionReference { + readonly __lunoraRef: string; +} +``` + ### `RagSyncArgs` (interface) ```ts @@ -181,7 +189,7 @@ interface RagSyncArgs extends Record { ```ts interface RagSyncOptions = Record> { - action: unknown; + action: RagSyncActionReference; delayMs?: number; id?: (document: Document) => string; text: (document: Document) => string | undefined; diff --git a/api-snapshots/errors.api.md b/api-snapshots/errors.api.md index 9a2c31622e..55ccf7d15c 100644 --- a/api-snapshots/errors.api.md +++ b/api-snapshots/errors.api.md @@ -501,10 +501,6 @@ const ERROR_CATALOG: { readonly status: 500; readonly title: "Nested transaction"; }; - readonly NETWORK_IN_MUTATION: { - readonly status: 500; - readonly title: "Network call inside a mutation"; - }; readonly OUT_OF_ORDER: { readonly status: 409; readonly title: "Out-of-order mutation"; @@ -526,6 +522,10 @@ const ERROR_CATALOG: { readonly status: 503; readonly title: "Durable stream interrupted"; }; + readonly STREAM_TOO_LONG: { + readonly status: 507; + readonly title: "Durable stream exceeded its chunk ceiling"; + }; readonly TOKEN_EXPIRED: { readonly status: 401; readonly title: "Authentication token expired"; diff --git a/api-snapshots/shard-engine.api.md b/api-snapshots/shard-engine.api.md index 0a473e7641..5b475823ca 100644 --- a/api-snapshots/shard-engine.api.md +++ b/api-snapshots/shard-engine.api.md @@ -661,15 +661,6 @@ interface DeployInfo { } ``` -### `DurableStreamChunk` (interface) - -```ts -interface DurableStreamChunk { - dataJson: string; - seq: number; -} -``` - ### `DurableStreamRun` (interface) ```ts @@ -682,12 +673,6 @@ interface DurableStreamRun { } ``` -### `DurableStreamStatus` (type) - -```ts -type DurableStreamStatus = "complete" | "error" | "running"; -``` - ### `ExportRow` (interface) ```ts @@ -2106,18 +2091,6 @@ const SCHEMA_HISTORY_MAX_VERSIONS = 50; const SEARCH_STATE_TABLE = "__lunora_search_state"; ``` -### `STREAM_CHUNKS_TABLE` (const) - -```ts -const STREAM_CHUNKS_TABLE = "__stream_chunks"; -``` - -### `STREAM_RUNS_TABLE` (const) - -```ts -const STREAM_RUNS_TABLE = "__stream_runs"; -``` - ### `SchedulableWorkflowReferenceLike` (interface) ```ts @@ -3137,7 +3110,7 @@ const bumpCdcEpoch: (sql: SqlExec) => string; ### `claimStreamRun` (const) ```ts -const claimStreamRun: (sql: SqlExec, runKey: string, startedAt: number) => boolean; +const claimStreamRun: (sql: SqlExec, runKey: string, startedAt: number, ttlMs: number) => boolean; ``` ### `clampPromotionThresholds` (const) @@ -3270,6 +3243,12 @@ const deleteGlobalShapeSnapshot: (sql: SqlExec, connectionId: string, subId: str const deleteGlobalShapeSnapshotsForConnection: (sql: SqlExec, connectionId: string) => void; ``` +### `deleteStreamRun` (const) + +```ts +const deleteStreamRun: (sql: SqlExec, runKey: string) => void; +``` + ### `depKey` (const) ```ts @@ -3378,7 +3357,7 @@ const findStorageReferences: (sql: SqlExec, storageColumns: Record void; @@ -3619,12 +3598,6 @@ const migrateSearchState: (sql: SqlExec) => void; const minCdcSeq: (sql: SqlExec) => number | undefined; ``` -### `mulberry32` (const) - -```ts -const mulberry32: (seed: number) => (() => number); -``` - ### `nextPromotionState` (const) ```ts @@ -4047,12 +4020,6 @@ const runSql: >(sql: SqlExec, query: string, ...pa const runTriggers: (options: RunTriggersOptions) => Promise; ``` -### `seedFrom` (const) - -```ts -const seedFrom: (text: string) => number; -``` - ### `selectExpiredIds` (const) ```ts @@ -4224,7 +4191,7 @@ const trimIdempotent: (sql: SqlExec, olderThanTs: number) => void; ### `trimStreamRuns` (const) ```ts -const trimStreamRuns: (sql: SqlExec, olderThanTs: number) => void; +const trimStreamRuns: (sql: SqlExec, now: number) => void; ``` ### `tryRowToDocument` (const) @@ -4251,15 +4218,6 @@ const unionAll: (branches: ReadonlyArray) => SQL; const validateImportRow: (schema: SchemaLike, table: string, record: Record) => string | undefined; ``` -### `withDeterministicScope` (const) - -```ts -const withDeterministicScope: (options: { - now: number; - seed: string; -}, body: () => Promise | T) => Promise; -``` - ### `writeGlobalShapeSnapshot` (const) ```ts diff --git a/apps/docs/src/content/docs/concepts/realtime.mdx b/apps/docs/src/content/docs/concepts/realtime.mdx index 99454e025f..639e153976 100644 --- a/apps/docs/src/content/docs/concepts/realtime.mdx +++ b/apps/docs/src/content/docs/concepts/realtime.mdx @@ -77,10 +77,12 @@ Three things change: then continues live — the consumer's `for await` never sees the interruption. - **The producer outlives the socket.** Close the tab mid-run and the run finishes anyway; the transcript is waiting when you come back. -- **A run is shared.** Its identity is `(functionPath, args)`, so a second tab - asking the same question attaches to the same run instead of paying for a - second generation. Give a stream that must not be shared a caller-unique - argument — the argument set _is_ the run key. +- **A live run is shared.** Its identity is `(identity, functionPath, args)`, so + a second tab of the _same signed-in user_ attaches to the run already in flight + instead of paying for a second generation. A different identity always gets its + own run. A **finished** run is not shared: a later caller asking the same + question gets a fresh answer, because a transcript is the record of one + execution, not a cached response. On the client, opt in per call so the reconnect resumes rather than fails: @@ -88,12 +90,15 @@ On the client, opt in per call so the reconnect resumes rather than fails: const { chunks, status } = useStream(api.chat.answer, { threadId, prompt }, { durable: true }); ``` -Transcripts are trimmed after 24 hours by default (`{ durable: { ttlMs } }` to -change it). One case is deliberately loud rather than silent: if the Durable -Object is evicted mid-run, the persisted prefix is replayed and the stream then -fails with `STREAM_INTERRUPTED` instead of quietly re-running the handler — -re-running a model call would bill twice and duplicate the tail. When you need a -producer that survives that too, the run belongs in a +Transcripts are trimmed after 24 hours by default, per procedure +(`{ durable: { ttlMs } }` to change it), and a run is capped at 50,000 chunks so +a runaway generator cannot fill the shard's storage. + +If the Durable Object is evicted mid-run, what happens depends on who asks next. +A client **resuming** that transcript gets `STREAM_INTERRUPTED` — its tail cannot +be spliced back on, and re-generating would duplicate what it already has. A +client asking **fresh** simply gets a new run: the dead one has no claim on its +key. When you need a producer that survives eviction itself, the run belongs in a [workflow](/docs/packages/workflow). ## Why no MQTT / Pub/Sub diff --git a/apps/docs/src/content/docs/non-goals.mdx b/apps/docs/src/content/docs/non-goals.mdx index ae88a40b89..f92a58c89d 100644 --- a/apps/docs/src/content/docs/non-goals.mdx +++ b/apps/docs/src/content/docs/non-goals.mdx @@ -161,16 +161,13 @@ A few smaller boundaries, most of them covered in depth on answers: await the first, combine both into one mutation, or use [`@lunora/db`](/docs/packages/db) mutators, whose outbox is strictly ordered. See [Write ordering](/docs/concepts/offline-first#write-ordering). -- **Determinism is enforced in mutations, linted in queries.** A mutation body - runs with `Date.now()` frozen to the dispatch instant, `Math.random()` seeded, - and `fetch` refused (`NETWORK_IN_MUTATION`) — sound because a mutation holds the - DO's single-writer gate, so nothing else can observe the swapped globals. - Queries interleave (a deferred subscription re-run lands between two awaits of - another query), so freezing the clock for one would hand its instant to - another; they keep the `nondeterministic_query_mutation` advisor lint plus the - `ctx.now` / `ctx.fetch` surfaces instead. The values are stable _within_ one - execution, not replayed across a retry — an idempotency key, not a replayed - clock, is what makes a retry safe. +- **No runtime enforcement of query/mutation determinism yet** — it's an advisor + lint (`nondeterministic_query_mutation`) plus `ctx.now`, not a hard runtime + guard. Swapping the ambient `Date.now`/`Math.random`/`fetch` for the duration of + a mutation looks tempting and is unsound here: workerd hosts many Durable + Objects in one isolate, so `blockConcurrencyWhile` gates one shard's writes, not + the isolate's globals — two shards mutating concurrently would see each other's + patched clock. A sound version injects per-context instead; see `plans/334`. ## See also diff --git a/packages/agent/__tests__/run-queue.test.ts b/packages/agent/__tests__/run-queue.test.ts index 93ef7329a0..27ec3fd3cd 100644 --- a/packages/agent/__tests__/run-queue.test.ts +++ b/packages/agent/__tests__/run-queue.test.ts @@ -216,6 +216,55 @@ describe("onConcurrentRun: queue", () => { await expect(start("wf-g")).rejects.toThrow("run queue is full"); }); + it("releases the slot of a run that ends while still parked", async () => { + expect.assertions(2); + + const { complete, queue, start } = setup(); + + await start("wf-a"); + await start("wf-b"); + await start("wf-c"); + + // B's 12h wait elapsed (or it threw before its turn): it is not the + // thread's owner, but it still holds a queue slot. Without releasing it, + // five abandoned runs exhaust the depth cap and every later start on this + // thread is refused forever. + await expect(complete({ instanceId: "wf-b", key: "thread-1", status: "error" })).resolves.toStrictEqual({}); + expect(queue().map((row) => row["instanceId"])).toStrictEqual(["wf-c"]); + }); + + it("keeps a failed run's error visible when it hands the thread on", async () => { + expect.assertions(1); + + const { complete, start, thread } = setup(); + + await start("wf-a"); + await start("wf-b"); + await complete({ error: "model refused", instanceId: "wf-a", key: "thread-1", status: "error" }); + + // The thread moves straight to B's run; clearing the error here would + // erase the only record that A failed at all. + expect(thread()).toMatchObject({ error: "model refused", instanceId: "wf-b", status: "running" }); + }); + + it("reclaims a thread whose owner was terminated while parked", async () => { + expect.assertions(2); + + const { start, thread } = setup(); + + await start("wf-a"); + await start("wf-b"); + + // Ownership transfers before the wake is sent, so an instance terminated + // while parked leaves the thread pointing at a workflow that never + // resumes. Age the row past the abandonment window. + Object.assign(thread() ?? {}, { updatedAt: Date.now() - 14 * 60 * 60 * 1000 }); + + // A new run takes it rather than CONFLICTing against a corpse forever. + await expect(start("wf-z")).resolves.toStrictEqual({ created: false }); + expect(thread()?.["instanceId"]).toBe("wf-z"); + }); + it("refuses to queue a dispatch that has no instance id to wake", async () => { expect.assertions(2); diff --git a/packages/agent/src/agent-loop.ts b/packages/agent/src/agent-loop.ts index 39c70b8138..6291ad6e08 100644 --- a/packages/agent/src/agent-loop.ts +++ b/packages/agent/src/agent-loop.ts @@ -1043,6 +1043,11 @@ const runAgentLoop = async (options: AgentLoopOptions): Promise * already owns the thread by that point, so there is no window in which two * runs believe they may append. */ + /** The user turn that opens the run. Runs after any queue wait, so a parked run appends only once it owns the thread. */ + const persistUserTurn = async (): Promise => { + await persist({ content: params.input, messageKey: `${instanceId}:user`, role: "user" }); + }; + const finishRun = async (patch: { error?: string; status: "error" | "idle"; usage?: AgentUsage }): Promise => { const outcome = (await run(completeRun, { instanceId, key: params.threadKey, ...patch })) as { dequeued?: string } | undefined; @@ -1086,13 +1091,7 @@ const runAgentLoop = async (options: AgentLoopOptions): Promise await terminatePriorInstance(env, exportName, bootstrap.priorInstanceId); } - // Queue policy: when this run was parked behind the one in flight, hibernate - // until the finishing run's completion mutation hands us the thread — by - // which point we already own it, so the first append below lands on the - // shared seq counter with nobody else writing to it. - await awaitDequeue(step, params.threadKey, instanceId, bootstrap?.queued); - - await persist({ content: params.input, messageKey: `${instanceId}:user`, role: "user" }); + await persistUserTurn(); // Memory step: dispatch the configured retrieval action once per run and // inject the assembled context into every turn's prompt. @@ -1130,6 +1129,19 @@ const runAgentLoop = async (options: AgentLoopOptions): Promise const usageBox: UsageBox = { value: undefined }; try { + // Queue policy: when this run was parked behind the one in flight, + // hibernate until the finishing run's completion hands us the thread — by + // which point we already own it, so the first append lands on the shared + // seq counter with nobody else writing to it. + // + // INSIDE the try on purpose: `waitForEvent` throws when its timeout + // elapses (a wake that could never land), and outside the try that throw + // escaped without running `finishRun` — leaving this run's queue row + // behind forever. Five of those exhaust the depth cap and every later + // start on the thread is refused. + await awaitDequeue(step, params.threadKey, instanceId, bootstrap?.queued); + await persistUserTurn(); + const { final, stoppedByCondition, turnsRun } = await runTurns(turnContext, maxTurns, stopConditions, usageBox); const usagePatch = usageBox.value === undefined ? {} : { usage: usageBox.value }; diff --git a/packages/agent/src/component.ts b/packages/agent/src/component.ts index dbc921f4a0..575ad104ba 100644 --- a/packages/agent/src/component.ts +++ b/packages/agent/src/component.ts @@ -30,6 +30,18 @@ const RUN_QUEUE_TABLE: "agent_run_queue" = `${AGENT_EXTENSION_KEY}_${RUN_QUEUE_B */ const MAX_QUEUE_DEPTH = 5; +/** + * How long a thread may sit untouched under a live-looking run before a new run + * may take it. + * + * Ownership moves to a dequeued run before its wake event is sent, so an + * instance terminated while parked leaves the thread pointing at a workflow that + * will never resume — and nothing else would ever free it. The window is longer + * than the loop's own dequeue timeout so a genuinely parked run is never + * reclaimed out from under itself. + */ +const ABANDONED_RUN_MS = 13 * 60 * 60 * 1000; + /** * The agent thread tables, shipped as a schema extension so an app merges * them with one call and they can never collide with app tables: @@ -380,7 +392,17 @@ export const agentComponent = (): AgentComponent => { // under the still-hibernating prior instance. So only a *matching* // instance id is exempt; missing OR differing both trip the policy. const priorInstanceId = existing["instanceId"] as string | undefined; + // Staleness reclaim. Ownership transfers to a dequeued run BEFORE + // its wake event is sent, so an instance terminated while parked + // leaves the thread owned by a workflow that will never resume. + // Nothing else reaps that: under `"reject"` every later run + // CONFLICTs, and under `"queue"` every later run parks behind a + // corpse. A thread untouched for longer than any run could + // plausibly hold it is treated as free. + const updatedAt = typeof existing["updatedAt"] === "number" ? existing["updatedAt"] : 0; + const abandoned = now - updatedAt > ABANDONED_RUN_MS; const isConcurrentRun = + !abandoned && (existing["status"] === "running" || existing["status"] === "awaiting_input") && priorInstanceId !== undefined && (args.instanceId === undefined || args.instanceId !== priorInstanceId); @@ -553,8 +575,21 @@ export const agentComponent = (): AgentComponent => { .first(); if (thread?.["instanceId"] !== args.instanceId) { - // Unknown thread, or ownership already moved on (a replay of this - // same completion) — either way not this call's to end. + // Not the owner: either a replay of a completion whose handoff + // already happened, or a run that ended while still PARKED — its + // wait timed out, or it threw before its turn came. The parked + // case must still release the slot it holds, or an abandoned run + // occupies a queue position forever and the depth cap eventually + // refuses every new start on this thread. + const parked = await context.db + .query(RUN_QUEUE_TABLE) + .withIndex("byThreadInstance", (q) => q.eq("threadKey", args.key).eq("instanceId", args.instanceId)) + .first(); + + if (parked) { + await context.db.delete(parked["_id"] as never); + } + return {}; } @@ -583,11 +618,17 @@ export const agentComponent = (): AgentComponent => { // Ownership transfers here, before the waking event is sent: the // dequeued run is already the owner when it resumes, so it can append // on the shared seq counter the moment it wakes. + // + // The finishing run's `error` is CARRIED, not cleared: a run that + // failed with another queued behind it would otherwise vanish without + // trace — the thread goes straight from one run to the next and + // nothing records that the first one failed. The incoming run clears + // it through its own bootstrap. await context.db.patch(thread["_id"] as never, { - error: undefined, instanceId: nextInstanceId, status: "running", updatedAt: now, + ...(args.error === undefined ? {} : { error: args.error }), ...(args.usage === undefined ? {} : { usage: args.usage }), }); diff --git a/packages/ai/__tests__/rag/sync.test.ts b/packages/ai/__tests__/rag/sync.test.ts index a875ae2f71..664d9b19c3 100644 --- a/packages/ai/__tests__/rag/sync.test.ts +++ b/packages/ai/__tests__/rag/sync.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { ragSyncTriggers } from "../../src/rag/sync"; -const ACTION = { __lunoraRef: "docs:reindex" }; +const ACTION = { __lunoraRef: "docs:reindex" } as const; /** Records what the trigger scheduled, standing in for `ctx.scheduler`. */ const fakeContext = () => { diff --git a/packages/ai/src/rag/index.ts b/packages/ai/src/rag/index.ts index 3b36efec57..2302aaaac5 100644 --- a/packages/ai/src/rag/index.ts +++ b/packages/ai/src/rag/index.ts @@ -3,7 +3,7 @@ export { default as defineRag } from "./define-rag"; export { contentHash, guessMimeTypeFromExtension } from "./helpers"; export { default as hybridRank } from "./hybrid-rank"; export { default as bm25LexicalStore } from "./lexical-store"; -export type { RagSyncArgs, RagSyncOptions } from "./sync"; +export type { RagSyncActionReference, RagSyncArgs, RagSyncOptions } from "./sync"; export { ragSyncTriggers } from "./sync"; export type { IndexInput, diff --git a/packages/ai/src/rag/sync.ts b/packages/ai/src/rag/sync.ts index 43a2be8951..f977b74dec 100644 --- a/packages/ai/src/rag/sync.ts +++ b/packages/ai/src/rag/sync.ts @@ -17,6 +17,16 @@ * the indexed text, so an unrelated column edit costs nothing at all. */ +/** + * A dispatchable function reference — the `internal.docs.reindex` you pass as + * `action`. Typed structurally (rather than `unknown`) so passing the wrong + * thing is a compile error: mis-wiring the action is the mistake this API is + * most likely to see, and it would otherwise surface as a silent no-op. + */ +interface RagSyncActionReference { + readonly __lunoraRef: string; +} + /** Structural slice of `ctx.scheduler` — enough to defer the re-index. */ interface RagSyncScheduler { runAfter: (delayMs: number, target: unknown, args?: Record) => Promise; @@ -50,7 +60,7 @@ interface RagSyncOptions = Record = Record unknown; send: (data: string) => void; sent: string[]; serializeAttachment: (value: unknown) => void; } -const createFakeWebSocket = (): FakeWebSocket => {return { - attachment: undefined, - send(data: string) { - this.sent.push(data); - }, - sent: [], - serializeAttachment(value: unknown) { - this.attachment = value as SocketAttachment | undefined; - }, -}}; +const createFakeWebSocket = (): FakeWebSocket => { + return { + attachment: undefined, + deserializeAttachment() { + return this.attachment; + }, + send(data: string) { + this.sent.push(data); + }, + sent: [], + serializeAttachment(value: unknown) { + this.attachment = value as SocketAttachment | undefined; + }, + }; +}; const parseFrames = (ws: FakeWebSocket) => ws.sent.map((raw) => JSON.parse(raw) as Record); @@ -204,8 +210,8 @@ describe("shardDO durable streams", () => { ).toStrictEqual(["first", "second"]); }); - it("reports an interrupted run instead of silently re-running the handler", async () => { - expect.assertions(2); + it("reports an interrupted run to a client holding half of it, and reclaims it for a fresh one", async () => { + expect.assertions(4); const shard = new DurableStreamShard(state, {}); @@ -229,15 +235,106 @@ describe("shardDO durable streams", () => { revived.registered = shard.registered; - const second = createFakeWebSocket(); + // A client RESUMING that transcript cannot be spliced back onto it: the + // tail would have to be re-generated, which duplicates. + const resumed = createFakeWebSocket(); + + revived.registerSocket(resumed, { subs: {} }); + await revived.driveMessage(resumed, { id: "stream_2", query: { functionPath: "chat:answer" }, sinceChunk: 1, type: "stream" }); + await waitForTerminator(resumed); + + expect((parseFrames(resumed).at(-1) as { error?: { code?: string } }).error?.code).toBe("STREAM_INTERRUPTED"); + + // A client asking fresh has no partial transcript to protect, so the dead + // run is reclaimed and a new one starts — an eviction must not wedge the + // key until its TTL expires. + revived.registered.set("chat:answer", async function* answer() { + yield "whole"; + }); + + const fresh = createFakeWebSocket(); + + revived.registerSocket(fresh, { subs: {} }); + await revived.driveMessage(fresh, { id: "stream_3", query: { functionPath: "chat:answer" }, type: "stream" }); + await waitForTerminator(fresh); + + const frames = parseFrames(fresh); + + expect(frames.filter((frame) => frame.type === "chunk").map((frame) => frame.data)).toStrictEqual(["whole"]); + expect(frames.at(-1)?.type).toBe("complete"); + // One start on the revived instance: the resuming attach refused rather + // than re-running, and only the fresh one produced. + expect(revived.starts).toBe(1); + }); + + it("re-runs for a later caller instead of serving a finished transcript as a cache", async () => { + expect.assertions(2); + + const shard = new DurableStreamShard(state, {}); + let answers = 0; + + shard.registered.set("chat:answer", async function* answer() { + answers += 1; + yield `answer-${String(answers)}`; + }); + + const first = createFakeWebSocket(); + + shard.registerSocket(first, { subs: {} }); + await shard.driveMessage(first, { id: "stream_1", query: { functionPath: "chat:answer" }, type: "stream" }); + await waitForTerminator(first); + + // Same arguments, but this caller is not resuming anything — it is asking + // a question. Replaying the stored answer would make a durable stream a + // response cache with a 24-hour TTL, including for a failed run. + const later = createFakeWebSocket(); - revived.registerSocket(second, { subs: {} }); - await revived.driveMessage(second, { id: "stream_2", query: { functionPath: "chat:answer" }, type: "stream" }); - await waitForTerminator(second); + shard.registerSocket(later, { subs: {} }); + await shard.driveMessage(later, { id: "stream_2", query: { functionPath: "chat:answer" }, type: "stream" }); + await waitForTerminator(later); - const frames = parseFrames(second); + expect( + parseFrames(later) + .filter((frame) => frame.type === "chunk") + .map((frame) => frame.data), + ).toStrictEqual(["answer-2"]); + expect(shard.starts).toBe(2); + }); + + it("never shares a run across identities", async () => { + expect.assertions(2); + + const shard = new DurableStreamShard(state, {}); + let release = (): void => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + + shard.registered.set("chat:answer", async function* answer() { + yield "secret"; + await gate; + }); + + const alice = createFakeWebSocket(); + const bob = createFakeWebSocket(); + + shard.registerSocket(alice, { subs: {}, userId: "alice" }); + shard.registerSocket(bob, { subs: {}, userId: "bob" }); + + await shard.driveMessage(alice, { id: "stream_a", query: { functionPath: "chat:answer" }, type: "stream" }); + await new Promise((resolve) => { + setTimeout(resolve, 5); + }); + // Same function, same args, different verified identity: Bob must get his + // own run, not a replay of Alice's transcript — an attach never drives the + // procedure, so it would also skip its RLS middleware. + await shard.driveMessage(bob, { id: "stream_b", query: { functionPath: "chat:answer" }, type: "stream" }); + await new Promise((resolve) => { + setTimeout(resolve, 5); + }); + release(); - expect(frames.filter((frame) => frame.type === "chunk").map((frame) => frame.data)).toStrictEqual(["half"]); - expect((frames.at(-1) as { error?: { code?: string } }).error?.code).toBe("STREAM_INTERRUPTED"); + expect(shard.starts).toBe(2); + expect(parseFrames(bob).filter((frame) => frame.type === "chunk")).toHaveLength(1); }); }); diff --git a/packages/do/src/shard-do.ts b/packages/do/src/shard-do.ts index dae8c34f10..fd3055d5c9 100644 --- a/packages/do/src/shard-do.ts +++ b/packages/do/src/shard-do.ts @@ -77,6 +77,7 @@ import type { ColumnMeta, CreateWorkflowInstanceResult, DependencyTracker, + DurableStreamRun, ExportRow, FanoutMetricsResult, FlagsResult, @@ -148,6 +149,7 @@ import { DEFAULT_MAX_RELAYS, deleteGlobalShapeSnapshot, deleteGlobalShapeSnapshotsForConnection, + deleteStreamRun, diffGlobalMembership, ensureAuditTable, facetColumn, @@ -208,7 +210,6 @@ import { trimIdempotent, trimStreamRuns, trySendFrame, - withDeterministicScope, writeGlobalShapeSnapshot, writeIdempotent, writeTouchesMemo, @@ -743,6 +744,48 @@ const DURABLE_STREAM_DEFAULT_TTL_MS = 86_400_000; /** Minimum spacing between durable-stream TTL sweeps, amortized onto a stream start. */ const DURABLE_STREAM_GC_INTERVAL_MS = 3_600_000; +/** + * Hard ceiling on the chunks one durable run may persist. + * + * A token-at-a-time generation is thousands of chunks, which is fine; a runaway + * generator is unbounded, which is not — the transcript lives in the shard's + * SQLite alongside application data, so "grow until the storage limit" takes the + * whole shard down. 50k covers any real answer with room to spare. + */ +const MAX_DURABLE_STREAM_CHUNKS = 50_000; + +/** + * What an attach should do about the run row it found. + * + * Pulled out of `attachDurableStream` because the rule is the interesting part + * and it is pure: whether a stored run may serve this caller turns entirely on + * whether the caller is RESUMING one it already holds part of, or asking fresh. + * + * `replay-terminal` is a finished run whose watcher came back: hand back the tail + * plus the recorded outcome. `interrupted` is a producer that died mid-run while + * this caller holds a prefix — the tail cannot be spliced on and re-generating + * would duplicate it, so say so rather than pretend. `reclaim` is a stored run + * this caller has no claim on (finished, or dead with nobody resuming): drop it + * and produce, because a transcript is the record of one execution rather than a + * cached response, and an eviction must not wedge the key until its TTL expires. + * `attach` joins the live producer, or starts one when there is no run. + */ +const decideDurableAttach = ( + run: DurableStreamRun | undefined, + context: { live: boolean; resuming: boolean }, +): "attach" | "interrupted" | "reclaim" | "replay-terminal" => { + if (run === undefined || context.live) { + return "attach"; + } + + if (run.status === "complete" || run.status === "error") { + return context.resuming ? "replay-terminal" : "reclaim"; + } + + // `running` with no live producer: the instance died mid-generation. + return context.resuming ? "interrupted" : "reclaim"; +}; + /** * One socket attached to a durable stream run. The producer fans every chunk * out to each attached sink and calls exactly one terminal (`complete`/`fail`); @@ -2006,24 +2049,15 @@ abstract class ShardDO { // // Only the depth bookkeeping stays here: it is `ShardDO` state, and the // nested-transaction error above reads it. - // The mutation body runs in a deterministic scope: `Date.now()` frozen, - // `Math.random()` seeded, `fetch` refused. Sound here and nowhere else — - // this is inside the single-writer gate, so no other dispatch can observe - // the swapped globals. See `withDeterministicScope`. - const now = Date.now(); - const seed = `${this.runner.shardKey ?? ""}:${this.currentRequestMutationId ?? ""}:${String(now)}`; - - return this.runner.runInTransaction(async () => - withDeterministicScope({ now, seed }, async () => { - this.transactionDepth = 1; + return this.runner.runInTransaction(async () => { + this.transactionDepth = 1; - try { - return await handler(); - } finally { - this.transactionDepth = 0; - } - }), - ); + try { + return await handler(); + } finally { + this.transactionDepth = 0; + } + }); } /** @@ -4374,7 +4408,10 @@ abstract class ShardDO { envelope.id, envelope.query.functionPath, decodeWire(envelope.query.args ?? {}) as Record, - typeof envelope.sinceChunk === "number" ? envelope.sinceChunk : 0, + // Client input: a non-integer or negative watermark would seed + // `delivered` past real chunks and silently truncate the stream, + // so it is normalised here rather than trusted. + Number.isInteger(envelope.sinceChunk) && (envelope.sinceChunk as number) > 0 ? (envelope.sinceChunk as number) : 0, ).catch(() => { /* socket already gone; nothing to report */ }); @@ -7674,20 +7711,38 @@ abstract class ShardDO { functionPath: string, args: Record, registration: { durable: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable }, - sinceSeq: number, + sinceChunk: number, ): Promise { const sql = this.sql as SqlExec; - const runKey = `${functionPath}:${stableWireKey(args)}`; + const ttlMs = registration.durable.ttlMs ?? DURABLE_STREAM_DEFAULT_TTL_MS; + // The socket's VERIFIED identity is part of the run key, not just the + // arguments. Sharing a transcript is the feature, but it must never be + // shared across identities: two users asking the same question with the + // same args are two runs, or the second silently reads the first's + // answer — and would skip the procedure's `.use(rls(...))` chain, since + // an attach to an existing run never drives the handler. + const owner = this.readAttachment(ws).userId ?? ""; + const runKey = `${owner}\u0000${functionPath}:${stableWireKey(args)}`; const cancellers = socketMap(this.streamCancellers, ws); const controller = new AbortController(); cancellers.set(id, controller); ws.send(JSON.stringify({ id, type: "ack" })); + const detach = (): void => { + cancellers.delete(id); + + // Drop the now-empty per-socket canceller map so a socket that churns + // through many short-lived streams doesn't accumulate empty `Map`s. + if (cancellers.size === 0) { + this.streamCancellers.delete(ws); + } + }; + // Everything below runs without awaiting: a DO handles one event at a // time, so replay-then-attach cannot interleave with the producer and // there is no window in which a chunk is neither replayed nor delivered. - let delivered = sinceSeq; + let delivered = 0; const sendChunk = (chunk: { data: unknown; seq: number }): boolean => { if (chunk.seq <= delivered) { @@ -7699,36 +7754,52 @@ abstract class ShardDO { return trySendFrame(ws, JSON.stringify({ data: chunk.data, id, seq: chunk.seq, type: "chunk" })); }; - for (const chunk of readStreamChunks(sql, runKey, sinceSeq)) { - if (controller.signal.aborted || !sendChunk({ data: JSON.parse(chunk.dataJson) as unknown, seq: chunk.seq })) { - cancellers.delete(id); - - return; + const run = readStreamRun(sql, runKey); + // A client that already holds part of this transcript is RESUMING; one at + // zero is asking for an answer. The distinction decides every branch + // below, because a finished run is a transcript of one execution — not a + // cached response to hand the next caller who asks the same thing. + const resuming = sinceChunk > 0; + const replay = (): boolean => { + for (const chunk of readStreamChunks(sql, runKey, sinceChunk)) { + if (controller.signal.aborted || !sendChunk({ data: JSON.parse(chunk.dataJson) as unknown, seq: chunk.seq })) { + detach(); + + return false; + } } - } - const run = readStreamRun(sql, runKey); + return true; + }; - if (run?.status === "complete") { - trySendFrame(ws, JSON.stringify({ id, type: "complete" })); - cancellers.delete(id); + const live = this.durableStreamRuns.get(runKey); + const decision = decideDurableAttach(run, { live: live !== undefined, resuming }); - return; - } + if (decision === "replay-terminal") { + if (!replay()) { + return; + } - if (run?.status === "error") { trySendFrame( ws, - JSON.stringify({ error: { code: run.errorCode ?? "INTERNAL_SERVER_ERROR", message: run.error ?? "stream failed" }, id, type: "error" }), + run?.status === "complete" + ? JSON.stringify({ id, type: "complete" }) + : JSON.stringify({ + error: { code: run?.errorCode ?? "INTERNAL_SERVER_ERROR", message: run?.error ?? "stream failed" }, + id, + type: "error", + }), ); - cancellers.delete(id); + detach(); return; } - const live = this.durableStreamRuns.get(runKey); + if (decision === "interrupted") { + if (!replay()) { + return; + } - if (run !== undefined && live === undefined) { trySendFrame( ws, JSON.stringify({ @@ -7737,20 +7808,24 @@ abstract class ShardDO { type: "error", }), ); - cancellers.delete(id); + detach(); return; } + if (decision === "reclaim") { + deleteStreamRun(sql, runKey); + } + const sink: DurableStreamSink = { chunk: sendChunk, complete: () => { trySendFrame(ws, JSON.stringify({ id, type: "complete" })); - cancellers.delete(id); + detach(); }, fail: (failure) => { trySendFrame(ws, JSON.stringify({ error: failure, id, type: "error" })); - cancellers.delete(id); + detach(); }, }; @@ -7762,22 +7837,29 @@ abstract class ShardDO { }); if (live) { + // Replay the prefix this attach missed, THEN subscribe. No await + // between the two, so the producer cannot slip a chunk into the gap. + if (!replay()) { + return; + } + live.sinks.add(sink); return; } - this.trimDurableStreams(sql, registration.durable.ttlMs); - claimStreamRun(sql, runKey, Date.now()); + this.trimDurableStreams(sql); + claimStreamRun(sql, runKey, Date.now(), ttlMs); const state = { sinks: new Set([sink]) }; this.durableStreamRuns.set(runKey, state); - // Detached from the socket, and handed to `waitUntil` where the host - // exposes one so the run isn't cut short the moment the last socket + // Detached from the socket and handed to `waitUntil` where the host + // exposes one, so the run isn't cut short the moment the last socket // closes — a durable stream that dies with its opener is just an - // ephemeral stream with extra writes. + // ephemeral stream with extra writes. Awaited as well so this method's + // own rejection path stays attached (the caller catches and discards). const producing = this.produceDurableStream(runKey, registration.iterator, state); this.shardHost.waitUntil?.(producing); @@ -7809,14 +7891,33 @@ abstract class ShardDO { const data = encodeWire(chunk); seq += 1; + + if (seq > MAX_DURABLE_STREAM_CHUNKS) { + // A runaway generator would otherwise grow the shard's SQLite + // without bound and take every other function on it down with + // the storage limit. Fail loudly at a ceiling no real answer + // reaches instead of degrading the whole shard. + throw new LunoraError( + "STREAM_TOO_LONG", + `durable stream exceeded ${String(MAX_DURABLE_STREAM_CHUNKS)} chunks; yield larger chunks or drop \`durable\``, + { status: 507 }, + ); + } + appendStreamChunk(sql, runKey, seq, JSON.stringify(data)); for (const sink of state.sinks) { - sink.chunk({ data, seq }); + // A sink whose socket refused the frame is gone: drop it + // rather than re-attempting it on every remaining chunk. The + // transcript is durable, so it loses nothing — the client + // resumes from its last `seq`. + if (!sink.chunk({ data, seq })) { + state.sinks.delete(sink); + } } } - finishStreamRun(sql, runKey, "complete"); + finishStreamRun(sql, runKey, "complete", seq); for (const sink of state.sinks) { sink.complete(); @@ -7833,7 +7934,7 @@ abstract class ShardDO { console.error("[@lunora/do] unhandled durable stream error:", error); } - finishStreamRun(sql, runKey, "error", { code: body.code, message: body.message }); + finishStreamRun(sql, runKey, "error", seq, { code: body.code, message: body.message }); for (const sink of state.sinks) { sink.fail({ code: body.code, message: body.message }); @@ -7843,8 +7944,12 @@ abstract class ShardDO { } } - /** Throttled TTL sweep for durable-stream transcripts, amortized onto a run start. */ - private trimDurableStreams(sql: SqlExec, ttlMs?: number): void { + /** + * Throttled TTL sweep for durable-stream transcripts, amortized onto a run + * start. Each run carries its own `ttl_ms`, so the sweep never applies the + * triggering procedure's retention to somebody else's transcript. + */ + private trimDurableStreams(sql: SqlExec): void { const now = Date.now(); if (now - this.lastDurableStreamTrimAt <= DURABLE_STREAM_GC_INTERVAL_MS) { @@ -7854,7 +7959,7 @@ abstract class ShardDO { this.lastDurableStreamTrimAt = now; try { - trimStreamRuns(sql, now - (ttlMs ?? DURABLE_STREAM_DEFAULT_TTL_MS)); + trimStreamRuns(sql, now); } catch { // Best-effort GC (missing table on a pre-migration shard / stub sql // handle) must never fail the stream that triggered it. diff --git a/packages/errors/src/catalog.ts b/packages/errors/src/catalog.ts index 77d3c6cd87..bfbe405de1 100644 --- a/packages/errors/src/catalog.ts +++ b/packages/errors/src/catalog.ts @@ -327,12 +327,12 @@ export const ERROR_CATALOG = { */ EXPIRED: { status: 404, title: "Session expired" }, NESTED_TRANSACTION: { internal: true, status: 500, title: "Nested transaction" }, - NETWORK_IN_MUTATION: { status: 500, title: "Network call inside a mutation" }, OUT_OF_ORDER: { status: 409, title: "Out-of-order mutation" }, SHAPE_GLOBAL_TOO_LARGE: { status: 413, title: "Global shape too large" }, SHAPE_NOT_FOUND: { status: 404, title: "Shape not found" }, SQL_UNAVAILABLE: { internal: true, status: 500, title: "SQL storage unavailable" }, STREAM_INTERRUPTED: { status: 503, title: "Durable stream interrupted" }, + STREAM_TOO_LONG: { status: 507, title: "Durable stream exceeded its chunk ceiling" }, TOKEN_EXPIRED: { status: 401, title: "Authentication token expired" }, TOO_MANY_STREAMS: { status: 429, title: "Too many streams" }, UNKNOWN_ADMIN_OP: { status: 404, title: "Unknown admin operation" }, diff --git a/packages/platform/src/capabilities.ts b/packages/platform/src/capabilities.ts index bf08472d90..5aaf6bb714 100644 --- a/packages/platform/src/capabilities.ts +++ b/packages/platform/src/capabilities.ts @@ -208,8 +208,8 @@ export const NODE_CAPABILITIES: PlatformCapabilities = { note: "Socket registry with attachments/tags persisted to SQLite, so subscription state survives a process restart; nothing is ever actually evicted from memory, so this is durability without hibernation's memory saving", }, durableStreams: { - level: "emulated", - note: "Same shape as the Cloudflare host — chunks persisted per run in the shard's SQLite, producer detached from the socket. One process, so a restart mid-run ends it as STREAM_INTERRUPTED", + level: "unsupported", + note: "The transcript store is host-neutral (@lunora/shard-engine), but the attach/produce state machine lives in @lunora/do and nothing in this host mounts it — a durable stream declared here would silently behave as an ephemeral one", }, localSql: { level: "native", note: "better-sqlite3 (synchronous, embedded)" }, shardAlarms: { diff --git a/packages/ratelimit/__tests__/token-budget.test.ts b/packages/ratelimit/__tests__/token-budget.test.ts index b9ecdb6ca7..0460071ce0 100644 --- a/packages/ratelimit/__tests__/token-budget.test.ts +++ b/packages/ratelimit/__tests__/token-budget.test.ts @@ -35,6 +35,19 @@ describe("tokenBudget", () => { await expect(shared.getValue("tokens", { key: "user-b" })).resolves.toMatchObject({ value: 1000 }); }); + it("charges an oversized call the whole bucket instead of throwing", async () => { + expect.assertions(2); + + const shared = limiter(); + const budget = tokenBudget(shared, "tokens"); + + // A single long-context call past the per-window capacity is ordinary for + // an LLM. Refusing to record it would let the one call the budget exists + // to catch escape it completely. + await expect(budget.record("user-x", 1500)).resolves.toMatchObject({ ok: true }); + await expect(budget.check("user-x")).resolves.toMatchObject({ ok: false }); + }); + it("keeps budgets separate per key", async () => { expect.assertions(1); diff --git a/packages/ratelimit/src/token-budget.ts b/packages/ratelimit/src/token-budget.ts index 7ffd95e164..65dc651ef0 100644 --- a/packages/ratelimit/src/token-budget.ts +++ b/packages/ratelimit/src/token-budget.ts @@ -82,7 +82,15 @@ const tokenBudget = (limiter: RateLimiter, name: Na return limiter.check(name, { key }); } - return limiter.limit(name, { count: Math.ceil(tokens), key, reserve: true }); + // Clamp to the bucket's capacity. The limiter refuses a count larger + // than capacity outright, and for a token budget that is exactly + // backwards: a single long-context call over a per-window budget is + // the ordinary case, and throwing would charge it NOTHING — the one + // call the budget exists to catch would escape it entirely. Charging + // the whole bucket is the strongest true statement available. + const { config } = await limiter.getValue(name, { key }); + + return limiter.limit(name, { count: Math.min(Math.ceil(tokens), config.capacity ?? config.rate), key, reserve: true }); }, }; }; diff --git a/packages/server/src/types.ts b/packages/server/src/types.ts index 6ed6398634..b3b5ae39dd 100644 --- a/packages/server/src/types.ts +++ b/packages/server/src/types.ts @@ -679,12 +679,16 @@ interface RegisteredStream
{ /** * Durability settings for a `.stream()` procedure. * - * A run is identified by the function path plus its arguments, so two clients - * calling the same stream with the same arguments observe one producer and one - * transcript. That identity is the feature: it is what makes a reload resume - * rather than re-generate, and what lets a second tab watch the same answer. - * Give a stream that must NOT be shared a caller-unique argument (a thread id, - * a message id) — the argument set is the run key. + * A run is identified by the socket's verified identity plus the function path + * and arguments, so two clients of the SAME user calling the same stream with + * the same arguments observe one producer and one transcript. That identity is + * the feature: it is what makes a reload resume rather than re-generate, and + * what lets a second tab watch the same answer. A different identity always gets + * its own run. + * + * Sharing applies to a run still in flight. Once a run finishes, a later caller + * asking the same question gets a fresh one — a transcript is the record of one + * execution, not a cached response. */ interface DurableStreamOptions { /** diff --git a/packages/shard-engine/__tests__/deterministic-scope.test.ts b/packages/shard-engine/__tests__/deterministic-scope.test.ts deleted file mode 100644 index dc114186d0..0000000000 --- a/packages/shard-engine/__tests__/deterministic-scope.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { mulberry32, withDeterministicScope } from "../src/deterministic-scope"; - -describe("withDeterministicScope", () => { - it("freezes the clock and seeds randomness for the duration of the body", async () => { - expect.assertions(3); - - const run = async (seed: string) => - withDeterministicScope({ now: 1_700_000_000_000, seed }, async () => { - const first = Date.now(); - - await Promise.resolve(); - - // eslint-disable-next-line sonarjs/pseudo-random -- asserting the SCOPE's seeded PRNG is exactly the point; nothing security-sensitive derives from it - return { first, random: Math.random(), second: Date.now() }; - }); - - const seen = await run("shard:mut-1"); - - // The classic bug this closes: one handler, two clock reads, two answers. - expect([seen.first, seen.second]).toStrictEqual([1_700_000_000_000, 1_700_000_000_000]); - // Seeded, so the sequence is a function of the seed, not host entropy... - await expect(run("shard:mut-1")).resolves.toMatchObject({ random: seen.random }); - // ...and a different mutation gets a different one. - await expect(run("shard:mut-2")).resolves.not.toMatchObject({ random: seen.random }); - }); - - it("refuses network I/O inside the scope", async () => { - expect.assertions(1); - - await expect( - withDeterministicScope({ now: 1, seed: "s" }, async () => { - await globalThis.fetch("https://example.com"); - }), - ).rejects.toThrow("fetch is not available inside a mutation"); - }); - - it("restores the ambient sources, including when the body throws", async () => { - expect.assertions(3); - - const realNow = Date.now; - const realRandom = Math.random; - const realFetch = globalThis.fetch; - - await expect( - withDeterministicScope({ now: 1, seed: "s" }, () => { - throw new Error("boom"); - }), - ).rejects.toThrow("boom"); - - expect(Date.now).toBe(realNow); - expect([Math.random, globalThis.fetch]).toStrictEqual([realRandom, realFetch]); - }); - - it("gives two different seeds two different sequences", () => { - expect.assertions(1); - - expect(mulberry32(1)()).not.toBe(mulberry32(2)()); - }); -}); diff --git a/packages/shard-engine/src/deterministic-scope.ts b/packages/shard-engine/src/deterministic-scope.ts deleted file mode 100644 index 1e5b36c882..0000000000 --- a/packages/shard-engine/src/deterministic-scope.ts +++ /dev/null @@ -1,118 +0,0 @@ -/** - * The deterministic scope a mutation body runs inside. - * - * A mutation is not an ordinary function: it may be re-executed (an OCC retry, a - * replayed dispatch), its result is cached by idempotency key, and its writes - * feed a changelog that subscriptions re-derive from. Ambient nondeterminism - * breaks all three in ways that only show up in production — `Date.now()` read - * twice in one handler yields two different "now"s, `Math.random()` makes a - * replay write different rows, and a `fetch` in the write path makes an atomic - * transaction depend on someone else's uptime. - * - * The advisor flags these statically (`nondeterministic_query_mutation`), but a - * lint only catches the call sites it can see — not the ones inside a dependency. - * So the runtime closes it too, by replacing the ambient sources for the duration - * of the mutation rather than merely forbidding them: - * - * `Date.now()` returns the instant the mutation began, so a handler reading the - * clock twice sees one value. `Math.random()` becomes a seeded PRNG, so the - * sequence is a function of the seed instead of the host's entropy. `fetch` - * throws — network I/O belongs in an action, because a transaction that awaits a - * third party cannot be atomic. - * - * **Scope of the guarantee.** Values are stable WITHIN one execution — the point - * is that a handler reading the clock twice sees one instant. They are not - * identical across two separate executions of the same mutation (a retry gets a - * fresh `now`, hence a fresh seed), which matches what the platform can honestly - * promise: an idempotency key, not a replayed clock, is what makes a retry safe. - * - * **Why only mutations.** Patching a global is only sound while nothing else can - * observe it. A mutation holds the DO's single-writer gate, so nothing else runs - * during it. Queries interleave (a deferred subscription re-run lands between two - * awaits of another query), so freezing the clock for one would hand its instant - * to the other — a worse lie than the one being fixed. Queries keep the static - * lint and the `ctx.now`/`ctx.fetch` surfaces instead. - */ -import { LunoraError } from "@lunora/errors"; - -/** Mixing increment of the published mulberry32 PRNG. */ -const MULBERRY_INCREMENT = 0x6d_2b_79_f5; - -/** 2^32 — divisor that maps the PRNG's 32-bit output into `[0, 1)`. */ -const UINT32_RANGE = 4_294_967_296; - -/* eslint-disable no-bitwise -- FNV-1a and mulberry32 ARE bit manipulation; both - are fixed published algorithms whose constants and shifts define the - distribution, so expressing them otherwise would only obscure them. Mirrors - the same disable in `shard-ring.ts`. */ - -/** FNV-1a (32-bit) over a string, so a seed derives deterministically from the dispatch identity. */ -const seedFrom = (text: string): number => { - let hash = 0x81_1c_9d_c5; - - for (const character of text) { - hash ^= character.codePointAt(0) ?? 0; - hash = Math.imul(hash, 0x01_00_01_93); - } - - return hash >>> 0; -}; - -/** A small, well-distributed 32-bit PRNG (mulberry32) — enough for a seeded `Math.random`. */ -const mulberry32 = (seed: number): (() => number) => { - let state = seed >>> 0; - - return () => { - state = (state + MULBERRY_INCREMENT) >>> 0; - - let next = state; - - next = Math.imul(next ^ (next >>> 15), next | 1); - next ^= next + Math.imul(next ^ (next >>> 7), next | 61); - - return ((next ^ (next >>> 14)) >>> 0) / UINT32_RANGE; - }; -}; - -/* eslint-enable no-bitwise */ - -/** The ambient sources this scope replaces, captured so they can be restored exactly. */ -interface AmbientSources { - fetch: typeof globalThis.fetch; - now: () => number; - random: () => number; -} - -/** - * Run `body` with `Date.now`, `Math.random`, and `fetch` replaced by their - * deterministic counterparts, restoring the originals afterwards — including when - * `body` throws. - * - * `now` is the dispatch instant (the same one `ctx.now` exposes) and `seed` should - * identify the dispatch (function path + instant + any idempotency key), so two - * different mutations never share a random sequence. - */ -const withDeterministicScope = async (options: { now: number; seed: string }, body: () => Promise | T): Promise => { - const original: AmbientSources = { fetch: globalThis.fetch, now: Date.now, random: Math.random }; - const random = mulberry32(seedFrom(options.seed)); - - Date.now = () => options.now; - Math.random = random; - globalThis.fetch = () => { - throw new LunoraError( - "NETWORK_IN_MUTATION", - "fetch is not available inside a mutation: a transaction that awaits a third party cannot be atomic. Move the call to an action (or schedule one).", - { status: 500 }, - ); - }; - - try { - return await body(); - } finally { - Date.now = original.now; - Math.random = original.random; - globalThis.fetch = original.fetch; - } -}; - -export { mulberry32, seedFrom, withDeterministicScope }; diff --git a/packages/shard-engine/src/durable-stream.ts b/packages/shard-engine/src/durable-stream.ts index 89982df832..b0d5bf2634 100644 --- a/packages/shard-engine/src/durable-stream.ts +++ b/packages/shard-engine/src/durable-stream.ts @@ -75,7 +75,8 @@ const migrateDurableStreams = (sql: SqlExec): void => { last_seq INTEGER NOT NULL DEFAULT 0, error_code TEXT, error TEXT, - started_at REAL NOT NULL + started_at REAL NOT NULL, + ttl_ms REAL NOT NULL )`, ); @@ -122,28 +123,46 @@ const readStreamRun = (sql: SqlExec, runKey: string): DurableStreamRun | undefin * attach can interleave. `RETURNING` would fold them into one statement, but * nothing else in the engine relies on it against workerd's SQLite yet. */ -const claimStreamRun = (sql: SqlExec, runKey: string, startedAt: number): boolean => { +const claimStreamRun = (sql: SqlExec, runKey: string, startedAt: number, ttlMs: number): boolean => { if (readStreamRun(sql, runKey) !== undefined) { return false; } runDrizzle( sql, - dsql`INSERT INTO ${dsql.identifier(STREAM_RUNS_TABLE)} (run_key, status, last_seq, started_at) VALUES (${runKey}, ${"running"}, 0, ${startedAt})`, + dsql`INSERT INTO ${dsql.identifier(STREAM_RUNS_TABLE)} (run_key, status, last_seq, started_at, ttl_ms) + VALUES (${runKey}, ${"running"}, 0, ${startedAt}, ${ttlMs})`, ); return true; }; /** - * Append one chunk and advance the run's `last_seq`. Called before the chunk - * reaches any socket, so a client that reconnects can always ask for everything - * after the last `seq` it saw without a gap. + * Drop a run and its chunks outright — the reclaim path. + * + * A run is not a cache of an answer: it is the transcript of one execution. When + * that execution can no longer be attached to (its producer died with the + * instance) or a caller is asking for a NEW one rather than resuming, the row has + * to go, or its key stays wedged until the TTL expires. + */ +const deleteStreamRun = (sql: SqlExec, runKey: string): void => { + runDrizzle(sql, dsql`DELETE FROM ${dsql.identifier(STREAM_CHUNKS_TABLE)} WHERE run_key = ${runKey}`); + runDrizzle(sql, dsql`DELETE FROM ${dsql.identifier(STREAM_RUNS_TABLE)} WHERE run_key = ${runKey}`); +}; + +/** + * Append one chunk. Called before the chunk reaches any socket, so a client that + * reconnects can always ask for everything after the last `seq` it saw without a + * gap. + * + * Deliberately one statement: the run row's `last_seq` is written once at the + * terminal instead of per chunk. A token-at-a-time generation is thousands of + * chunks, and nothing reads `last_seq` mid-run — an attach replays from + * `__stream_chunks` itself — so the second write was pure cost on the hottest + * path this feature has. */ const appendStreamChunk = (sql: SqlExec, runKey: string, seq: number, dataJson: string): void => { runDrizzle(sql, dsql`INSERT OR IGNORE INTO ${dsql.identifier(STREAM_CHUNKS_TABLE)} (run_key, seq, data_json) VALUES (${runKey}, ${seq}, ${dataJson})`); - - runDrizzle(sql, dsql`UPDATE ${dsql.identifier(STREAM_RUNS_TABLE)} SET last_seq = ${seq} WHERE run_key = ${runKey}`); }; /** @@ -162,7 +181,7 @@ const readStreamChunks = (sql: SqlExec, runKey: string, sinceSeq: number): Durab }); /** Mark a run finished. `errorCode`/`error` are written only for the `error` status. */ -const finishStreamRun = (sql: SqlExec, runKey: string, status: "complete" | "error", failure?: { code: string; message: string }): void => { +const finishStreamRun = (sql: SqlExec, runKey: string, status: "complete" | "error", lastSeq: number, failure?: { code: string; message: string }): void => { /* eslint-disable unicorn/no-null -- SQL NULL is the value being written: a run that completes must clear its error columns, and `undefined` is not bindable */ const errorCode = failure?.code ?? null; const errorMessage = failure?.message ?? null; @@ -171,30 +190,33 @@ const finishStreamRun = (sql: SqlExec, runKey: string, status: "complete" | "err runDrizzle( sql, dsql`UPDATE ${dsql.identifier(STREAM_RUNS_TABLE)} - SET status = ${status}, error_code = ${errorCode}, error = ${errorMessage} + SET status = ${status}, last_seq = ${lastSeq}, error_code = ${errorCode}, error = ${errorMessage} WHERE run_key = ${runKey}`, ); }; /** - * Drop runs (and their chunks) that started before `olderThanTs`. Runs are a - * cache of a transcript, not application data — the TTL is what keeps a chatty - * shard's SQLite from growing without bound. + * Drop every run (and its chunks) whose OWN retention window has elapsed. + * + * Each run stores the `ttlMs` of the procedure that created it, so the comparison + * is per row. A shard mixing a 24h chat transcript with a 60s progress stream + * must not lose the former because the latter happened to trigger the sweep. */ -const trimStreamRuns = (sql: SqlExec, olderThanTs: number): void => { +const trimStreamRuns = (sql: SqlExec, now: number): void => { runDrizzle( sql, dsql`DELETE FROM ${dsql.identifier(STREAM_CHUNKS_TABLE)} WHERE run_key IN ( - SELECT run_key FROM ${dsql.identifier(STREAM_RUNS_TABLE)} WHERE started_at < ${olderThanTs} + SELECT run_key FROM ${dsql.identifier(STREAM_RUNS_TABLE)} WHERE started_at + ttl_ms < ${now} )`, ); - runDrizzle(sql, dsql`DELETE FROM ${dsql.identifier(STREAM_RUNS_TABLE)} WHERE started_at < ${olderThanTs}`); + runDrizzle(sql, dsql`DELETE FROM ${dsql.identifier(STREAM_RUNS_TABLE)} WHERE started_at + ttl_ms < ${now}`); }; export { appendStreamChunk, claimStreamRun, + deleteStreamRun, finishStreamRun, migrateDurableStreams, readStreamChunks, diff --git a/packages/shard-engine/src/index.ts b/packages/shard-engine/src/index.ts index 894709eac7..b870fa5f55 100644 --- a/packages/shard-engine/src/index.ts +++ b/packages/shard-engine/src/index.ts @@ -85,7 +85,6 @@ export type { export { DATA_MIGRATION_STATE_TABLE, readMigrationStatus, runDataMigration } from "./data-migration"; export type { DependencyTracker } from "./dependency-tracker"; export { createDependencyTracker, depKey, SCAN_DEP, tableFromDepKey } from "./dependency-tracker"; -export { mulberry32, seedFrom, withDeterministicScope } from "./deterministic-scope"; export { runDrizzle, runSql } from "./do-exec"; export { AGG_COUNT, @@ -107,16 +106,15 @@ export { } from "./do-sql"; export type { RenderedSql, SqlEngine } from "./drizzle"; export { param, renderSql, sqliteInList, unionAll } from "./drizzle"; -export type { DurableStreamChunk, DurableStreamRun, DurableStreamStatus } from "./durable-stream"; +export type { DurableStreamRun } from "./durable-stream"; export { appendStreamChunk, claimStreamRun, + deleteStreamRun, finishStreamRun, migrateDurableStreams, readStreamChunks, readStreamRun, - STREAM_CHUNKS_TABLE, - STREAM_RUNS_TABLE, trimStreamRuns, } from "./durable-stream"; export type { ExternalSourceDiffResult } from "./external-source-diff"; diff --git a/packages/studio/src/features/evals/evals-panel.tsx b/packages/studio/src/features/evals/evals-panel.tsx index cc65d72a02..cbb60449cd 100644 --- a/packages/studio/src/features/evals/evals-panel.tsx +++ b/packages/studio/src/features/evals/evals-panel.tsx @@ -6,6 +6,7 @@ import { Badge } from "../../components/ui/badge"; import { Card, CardContent } from "../../components/ui/card"; import { EmptyState } from "../../components/ui/empty-state"; import { useAdminQuery } from "../../hooks/use-admin-query"; +import useOpenTrace from "../../hooks/use-open-trace"; import { useShardKey } from "../../hooks/use-shard-key"; import { useT } from "../../i18n/i18n-context"; import type { MetricHistoryResult, TracesResult, TraceSummary } from "../../lib/admin"; @@ -93,8 +94,6 @@ const EvalCardView = ({ card, onOpenTrace }: EvalCardViewProps): ReactElement => interface EvalsPanelProps { readonly initialShardKey?: string; - /** Open a trace by id — the drill-down from a graded run to the generation. */ - readonly onOpenTrace?: (traceId: string) => void; } /** @@ -108,9 +107,12 @@ interface EvalsPanelProps { * `recordEvaluation`; the card says so plainly rather than drawing a trend the * data can't back. */ -const EvalsPanel = ({ initialShardKey, onOpenTrace }: EvalsPanelProps): ReactElement => { +const EvalsPanel = ({ initialShardKey }: EvalsPanelProps): ReactElement => { const t = useT(); const { queryShardKey, setShardKey, shardKey } = useShardKey(initialShardKey); + // The drill-down from a score to the generation that earned it — the same + // hand-off the Metrics panel's exemplars use. + const openTrace = useOpenTrace(queryShardKey); const { data, error } = useAdminQuery(ADMIN_FUNCTIONS.getTraces, {}, { live: true, shardKey: queryShardKey }); // The trend read's own failure is ignored: a worker predating the RPC, or an @@ -134,7 +136,7 @@ const EvalsPanel = ({ initialShardKey, onOpenTrace }: EvalsPanelProps): ReactEle ) : (
{cards.map((card) => ( - + ))}
)} diff --git a/plans/334-mutation-determinism.md b/plans/334-mutation-determinism.md new file mode 100644 index 0000000000..a97665b8df --- /dev/null +++ b/plans/334-mutation-determinism.md @@ -0,0 +1,71 @@ +# Plan 334 — Runtime determinism for query/mutation bodies + +- **Category**: correctness (silent nondeterminism in a replayable path) +- **Status**: DESIGN — a global-swap implementation was built, proven unsound under concurrency, and reverted +- **Related**: `nondeterministic_query_mutation` advisor lint, `ctx.now` + +## The gap + +A mutation may be re-executed and its result is cached by idempotency key, so +ambient nondeterminism inside one is a real defect class: `Date.now()` read twice +in one handler yields two instants, `Math.random()` makes a replay write +different rows, and a `fetch` in the write path makes an atomic transaction +depend on someone else's uptime. + +Today this is caught by a static advisor lint plus the `ctx.now` surface. A lint +only sees the call sites it can read — not the ones inside a dependency. + +## What was tried, and why it was reverted + +`withDeterministicScope` captured `Date.now` / `Math.random` / `globalThis.fetch`, +swapped in deterministic replacements for the duration of a mutation, and +restored them in `finally`. It was wired into `ShardDO.runInTransaction`. + +The soundness argument was "a mutation holds the DO's single-writer gate, so +nothing else runs during it". That gate is **per Durable Object**; the globals are +**per isolate**, and workerd hosts many DOs of the same class in one isolate. This +repo already documents that premise, in a comment written to fix the same class of +bug: + +> `packages/observability/src/query-metrics.ts` — "Keyed by the storage handle +> rather than a module-level scalar: workerd hosts several Durable Object +> instances of the same class in one isolate…" + +Two interleaved scopes corrupt the save/restore stack: + +``` +A enter: saves real → patches +B enter: saves A's stub → patches +A exit : restores real → B loses its guarantee mid-mutation +B exit : restores A's stub → globalThis is permanently patched +``` + +Reproduced in a scratch test: after two interleaved scopes `Date.now` is still the +frozen stub and `globalThis.fetch` still throws — for every shard, query, +subscription, alarm, and telemetry flush in that isolate, until it is recycled. On +`@lunora/platform-node` it is worse: `runSerialized` is per shard host while +`globalThis` is process-wide, so any two concurrent shard mutations trip it. + +A refcount would stop the permanent poisoning but still cannot give the +overlapping mutation its guarantee, because there is only one set of globals. + +## What a sound version looks like + +Per-context injection, not global mutation: + +1. **`ctx.now` already exists** and is frozen per dispatch. The remaining gap is + code that calls ambient `Date.now()` — which only a lint or a bundler-level + transform can see. +2. **`ctx.random`** as a seeded PRNG on the mutation context, so a handler that + wants replay-stable randomness has a supported way to get it. +3. **`ctx.fetch` refused in query/mutation.** This one IS sound per-context and is + the Convex-equivalent rule (network I/O belongs in an action). It needs a + codegen change: the emitted `buildCtx` already knows the dispatch mode, so it + can install a throwing `fetch` for the non-action modes without touching + `globalThis`. Note it is a breaking change for any mutation using `ctx.fetch`. +4. An `AsyncLocalStorage`-backed scope would cover ambient calls soundly, but + `node:async_hooks` availability on workerd (and its cost per dispatch) needs + measuring before that is promised. + +Ordering: (3) is small, sound, and independently useful; (2) is additive; (1) and +(4) are the open research. diff --git a/plans/README.md b/plans/README.md index a3a823afe0..81e5af82be 100644 --- a/plans/README.md +++ b/plans/README.md @@ -1609,25 +1609,25 @@ Deliberately skipped: the uncommitted `durable-stream.ts` work in `packages/shard-engine` and the in-flight diffs in `packages/server/src` — another session is mid-feature there. -| Plan | Title | Category | Pkg | Pri | Effort | Risk | Status | -| ---- | ------------------------------------------------------------- | -------------- | ----------------------- | --- | ------ | ---- | ------------------------------------------------------------ | -| 316 | Scope the client mutation watermark by identity | bug | client/db | P1 | S | LOW | TODO — [316](316-client-watermark-identity-scope.md) | -| 317 | Route every `.dev.vars` write through the atomic 0600 writer | security | cli | P1 | S | LOW | TODO — [317](317-dev-vars-atomic-write.md) | -| 318 | Stop the Vite dev loop emitting from a stale TS program | bug | vite/codegen | P1 | S | LOW | TODO — [318](318-vite-stale-codegen-project.md) | -| 319 | Bound the dispatch call; classify deterministic 4xx | bug | dispatch/workflow/queue | P1 | S–M | MED | ✅ **DONE (WS1+WS3+workflow half) — APPROVED, awaiting merge. Queue half BLOCKED by its own STOP, correctly.** Worktree `.claude/worktrees/agent-aaba9f993d37eac1d`, branch `advisor/319-dispatch-timeout-and-retry-class`, commits `bf4a23863` + `970244c04`. Reviewer re-ran both suites: dispatch 27/27, workflow 81/81. The safety invariant holds — `DETERMINISTIC_DISPATCH_STATUSES = {400,403,404,422}` with `408`/`429` excluded in the constant's own comment, and the abort error is minted at **503**, deliberately outside that set, so a timeout stays retryable. **Queue half stopped for the sanctioned reason:** `dispatchQueueBatch` unconditionally rethrows the handler's error (`if (threw) { throw handlerError; }`), `deadLettered` is post-hoc metadata rather than a mechanism, and an existing test pins the rethrow as the deliberate contract — so ack-without-retry would be a redelivery-semantics change, not a bug fix. **§9 Q1 remains unconfirmed**: this implements the docblock's contract; if the maintainer prefers today's retry-everything behaviour, drop commit `970244c04` and edit the docblock — `bf4a23863` (the timeout) stands either way. **Follow-up it surfaced:** `@lunora/queue` re-exports `RunFunctionOptions` from dispatch so it picked up `timeoutMs` automatically, but `@lunora/workflow` declares its own copy in `workflow/src/types.ts`, so `timeoutMs` is invisible on a workflow step's typed `ctx.run` even though the runtime honours it — [319](319-dispatch-timeout-and-retry-class.md) | -| 320 | Cover the Stripe/Polar money-mutating methods | tests | payment | P1 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a43c3ad10c64a298f`, branch `advisor/320-stripe-polar-money-tests`, commit `d9314a192`. 16 new tests (8 per provider), 195/195 pass; reviewer re-ran coverage independently: `stripe.ts` branches 43.39% → **65.09%**, `polar.ts` 33.67% → **60.2%**, aggregate 68.8%, floor raised 60 → 68. `packages/payment/src` untouched (verified). **It found three latent defects on the money path and correctly pinned current behaviour rather than fixing it** — each needs its own plan: (1) **Stripe `updateSubscription` with an empty `items.data` sends `id: undefined`**, which Stripe treats as *creating a new subscription item* rather than updating one — a silent billing split, not a crash, which is why the plan's "if it throws" STOP never fired (`providers/stripe.ts:406`); (2) **Polar's `refundPayment` never returns `partially_refunded`** — it has no `compareMoney` boundary check at all and always reports `refunded` (`providers/polar.ts:289-315`); (3) **Polar's `updateSubscription` silently drops `quantity`** — a quantity-only patch produces an empty `subscriptionUpdate: {}` no-op (`providers/polar.ts:338-342`). Also recommended: `resumeSubscription` coverage for Creem/Dodo (both at 0) — [320](320-stripe-polar-money-tests.md) | -| 321 | Restore the coverage floor on six packages; fix the generator | tests/dx | repo | P2 | S | LOW | TODO — [321](321-coverage-floor-gaps.md) | -| 322 | Make the randomized test seed actually randomize | tests | repo | P2 | S→M | MED | TODO — [322](322-vitest-shuffle.md) | -| 323 | Run `dist:check` on pull requests | dx/ci | ci | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a7678ba2e1d9b517e`, branch `advisor/323-dist-check-pr-gate`, commit `9c81e74d6`, one file. **The gate was proven to fail before it was wired:** `dist:check` against a development build exits **1** with **180 files** flagged (`react/jsx-dev-runtime` and `jsxDEV` markers in `@lunora/react` and `@lunora/studio` output); against a clean production build it exits **0** ("1282 shipped files across 54 packages carry no development-build markers"). So the alpha.31 defect class does reach a dev build today and currently escapes every PR job. **Accepted deviation, and it is the difference between working and decorative:** the executor also added `dist-production` to `lint-required-check`'s `needs` — reviewer verified the aggregate now lists it. Without that the job would report and never block, and this repo's branch ruleset requires only the aggregate checks. WS3 was deliberately **not** done: `CLAUDE.md:42`'s claim that both gates have CI jobs became true once the job landed, so editing it would be churn. **Unverified by construction:** the job has never been observed running in CI, and no `api-surface`-vs-`dist-production` duration comparison exists — local timings were taken under wildly different load and are not comparable — [323](323-dist-check-pr-gate.md) | -| 324 | Test the shared client mutation path; floor `@lunora/client` | tests | client | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a3e993f1a89ad1ba9`, branch `advisor/324-client-mutation-path-tests`, commit `0979c777c`. Reviewer re-ran: 38 files, 657 passed + 1 expected-fail. `mutation-runner.ts` and `snapshot-precondition.ts` both at 100% branches; floor pinned at 84/75/76/84 with `src/sw/**` excluded (§9 Q1 — a real ~11%-stmt gap, excluded with its reason in the config so later sw work doesn't inherit a floor it must fight). **Both proofs were actually performed, not asserted:** the ref-count test reds when `inFlight` is replaced with a boolean, and bumping `branches` to 99 fails the run with the threshold error. **Trap for anyone re-measuring:** the printed coverage table silently omits rows for both new files — read `coverage/lcov.info` (`BRH==BRF`), not the table. §9 Q2: `src/auth/index.ts` (0% across 95 lines of client auth surface) deserves its own plan — [324](324-client-mutation-path-tests.md) | -| 325 | Test the native-signup email gate | tests/security | auth | P2 | S | LOW | TODO — [325](325-email-gate-tests.md) | -| 326 | Make the shape poke baseline survive hibernation | perf | do/shard-engine | P2 | M | MED | TODO — [326](326-shape-poke-cursor-durability.md) | -| 327 | Manifest hygiene: unused dep, prod-vs-dev dep, catalogs, hono | deps | repo | P3 | S | LOW | TODO — [327](327-manifest-hygiene.md) | -| 328 | Stop recording an attacker-chosen IP in the auth audit trail | security | auth | P3 | S | LOW | TODO — [328](328-audit-log-ip-fallback.md) | -| 329 | Publish the Node host's capability matrix | docs | platform-node | P3 | S | LOW | TODO — [329](329-platform-node-docs.md) | -| 330 | Let Angular produce the preloaded query it can consume | direction | angular | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-ae58d3596d59bc8a0`, branch `advisor/330-angular-preload-query`, commit `30ea1ffe4`. `packages/angular/src/server.ts` is a pure re-export of `@lunora/client/ssr` (verified against that module's live surface, not Solid's copy), `./server` added to the exports map, `dist/server.{mjs,d.ts}` emitted, 108 tests pass, key-order gate clean, `api:check` unchanged (angular is untiered). Reviewer re-ran the criteria independently. **Deviation, accepted:** the export-parity test asserts against `@lunora/client/ssr` rather than `@lunora/solid/server` — strictly stronger (it catches Solid drifting too) and avoids a first-of-its-kind adapter-to-adapter dependency edge. **`templates/analog/` was documented, not wired** — its demo still uses a hand-rolled `LunoraService` over the vanilla client, and swapping it is a separate migration. Two follow-ups opened: an API-snapshot tier decision for `@lunora/angular` now that it publishes a second entry point, and `angular.mdx`'s "Deliberately small surface" callout, which undersells an adapter that already exports `paginatedQuery`/`presence`/`subscription`/`rateLimit`/`stream`/`agent*`/`flag*` — [330](330-angular-preload-query.md) | -| 331 | Resolve the `/worker` subpath contradiction | direction | vue/svelte | P3 | S–M | LOW | **PHASE 0 DONE — awaiting a maintainer decision before Phase 1A.** Evidence in §9, reviewer-verified against the live tree: the subpath has **zero** real importers (the only hit repo-wide is its own docblock example), and `withFrameworkWorker` itself is reached only by codegen's `buildFrameworkWorker` → `@lunora/runtime`, never through a framework `/worker` subpath — the Astro and SvelteKit templates both bypass it, leaving `templates/sveltekit`'s prose stale. `cb6df0762` shows the same defect was already caught and half-fixed once. Recommendation: **Option A** (delete both re-exports, promote `withFrameworkWorker` as the documented BYO-framework entry). Phase 1A removes a published surface, so it is not an executor's call — [331](331-worker-subpath-decision.md) | -| 332 | Spike: what would a payment conformance suite assert? | direction | payment | P3 | S | LOW | TODO (spike) — [332](332-payment-conformance-spike.md) | +| Plan | Title | Category | Pkg | Pri | Effort | Risk | Status | +| ---- | ------------------------------------------------------------- | -------------- | ----------------------- | --- | ------ | ---- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 316 | Scope the client mutation watermark by identity | bug | client/db | P1 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a68bfc5859c3a2d80`, branch `advisor/316-client-watermark-identity-scope`, commit `5de113e65`, four files. Reviewer re-ran: client 650 pass + 1 expected-fail, db 87/87, `api:check` clean. The regression test does the real thing — ack 5 under identity A, switch to B → **0**, switch back to A → **5**, on a real `LunoraClient`. **The plan's WS1 as literally written was incomplete, and the mandated stash-and-prove is what caught it:** composite-keying alone breaks the plan's own contract §3.2, because a same-token _subject resolve_ relabels the fingerprint (hash → `subj:…`) and makes the cached watermark invisible under the new key — reintroducing the identical wedge on every routine subject resolution. Fixed with `restampWatermarks(from, to)` called from `setAuthToken`'s existing same-token branch, mirroring the established `restampQueuedIdentity` sibling. The executor nearly reported done with 2 of 3 tests passing before adding the third. **Better than the plan on one detail:** the composite separator is `�`, not the `�` the plan suggested — a NUL would have made the file binary to `grep`, which has bitten this repo before; `file` confirms it is still UTF-8 text. `debug()`'s projection strips the prefix and filters to the current identity, so no other identity's watermark can leak into it. §9: no LRU needed (one entry per visited identity×bucket, same shape as the file's existing unbounded-but-small maps); the durable `OutboxSink` replay path already has its own `currentIdentity()` guard — no gap — [316](316-client-watermark-identity-scope.md) | +| 317 | Route every `.dev.vars` write through the atomic 0600 writer | security | cli | P1 | S | LOW | TODO — [317](317-dev-vars-atomic-write.md) | +| 318 | Stop the Vite dev loop emitting from a stale TS program | bug | vite/codegen | P1 | S | LOW | TODO — [318](318-vite-stale-codegen-project.md) | +| 319 | Bound the dispatch call; classify deterministic 4xx | bug | dispatch/workflow/queue | P1 | S–M | MED | ✅ **DONE (WS1+WS3+workflow half) — APPROVED, awaiting merge. Queue half BLOCKED by its own STOP, correctly.** Worktree `.claude/worktrees/agent-aaba9f993d37eac1d`, branch `advisor/319-dispatch-timeout-and-retry-class`, commits `bf4a23863` + `970244c04`. Reviewer re-ran both suites: dispatch 27/27, workflow 81/81. The safety invariant holds — `DETERMINISTIC_DISPATCH_STATUSES = {400,403,404,422}` with `408`/`429` excluded in the constant's own comment, and the abort error is minted at **503**, deliberately outside that set, so a timeout stays retryable. **Queue half stopped for the sanctioned reason:** `dispatchQueueBatch` unconditionally rethrows the handler's error (`if (threw) { throw handlerError; }`), `deadLettered` is post-hoc metadata rather than a mechanism, and an existing test pins the rethrow as the deliberate contract — so ack-without-retry would be a redelivery-semantics change, not a bug fix. **§9 Q1 remains unconfirmed**: this implements the docblock's contract; if the maintainer prefers today's retry-everything behaviour, drop commit `970244c04` and edit the docblock — `bf4a23863` (the timeout) stands either way. **Follow-up it surfaced:** `@lunora/queue` re-exports `RunFunctionOptions` from dispatch so it picked up `timeoutMs` automatically, but `@lunora/workflow` declares its own copy in `workflow/src/types.ts`, so `timeoutMs` is invisible on a workflow step's typed `ctx.run` even though the runtime honours it — [319](319-dispatch-timeout-and-retry-class.md) | +| 320 | Cover the Stripe/Polar money-mutating methods | tests | payment | P1 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a43c3ad10c64a298f`, branch `advisor/320-stripe-polar-money-tests`, commit `d9314a192`. 16 new tests (8 per provider), 195/195 pass; reviewer re-ran coverage independently: `stripe.ts` branches 43.39% → **65.09%**, `polar.ts` 33.67% → **60.2%**, aggregate 68.8%, floor raised 60 → 68. `packages/payment/src` untouched (verified). **It found three latent defects on the money path and correctly pinned current behaviour rather than fixing it** — each needs its own plan: (1) **Stripe `updateSubscription` with an empty `items.data` sends `id: undefined`**, which Stripe treats as _creating a new subscription item_ rather than updating one — a silent billing split, not a crash, which is why the plan's "if it throws" STOP never fired (`providers/stripe.ts:406`); (2) **Polar's `refundPayment` never returns `partially_refunded`** — it has no `compareMoney` boundary check at all and always reports `refunded` (`providers/polar.ts:289-315`); (3) **Polar's `updateSubscription` silently drops `quantity`** — a quantity-only patch produces an empty `subscriptionUpdate: {}` no-op (`providers/polar.ts:338-342`). Also recommended: `resumeSubscription` coverage for Creem/Dodo (both at 0) — [320](320-stripe-polar-money-tests.md) | +| 321 | Restore the coverage floor on six packages; fix the generator | tests/dx | repo | P2 | S | LOW | TODO — [321](321-coverage-floor-gaps.md) | +| 322 | Make the randomized test seed actually randomize | tests | repo | P2 | S→M | MED | 🟡 **WS1+WS2 DONE — APPROVED as far as it goes; WS3/WS4 still open.** Worktree `.claude/worktrees/agent-a7d4a9d9a94d95128`, branch `advisor/322-vitest-shuffle`, commit `6aa02e52b`, four lines in one file. Shuffle is **default-off** behind `LUNORA_SHUFFLE=1` — reviewer confirmed a plain run is unchanged. Both WS1 proofs hold: two flagged runs of `@lunora/values` give different file orders, and a pinned seed reproduces byte-identically. **The reproducibility check needed `--no-file-parallelism` to be meaningful at all** — with parallel files the reporter prints _completion_ order, not schedule order, so the same seed looks different every run; that belongs in the plan's method, not just this row. **Sweep: 37 of 38 clean, 0 confirmed order-coupling.** `@lunora/cli` failed twice with _different_ failure sets under _different_ seeds and then passed 90/90 on a serial replay of either seed — the signature of genuine flakiness (real subprocesses/fs), not order-coupling, so it is recorded unconfirmed. **Two caveats that bound what this proves.** (a) **My call, and it weakened the evidence:** I authorized one run per package instead of the plan's three, trading depth for breadth while the machine was busy. So WS3's gate ("WS2 complete for every package", three runs each) is **not** satisfied — read this as "no order-coupling in a single-run sweep", not a clearance to flip the default. (b) **`auth`'s PASS is meaningless**: its config never calls the helper, so `LUNORA_SHUFFLE` is a no-op there and the run was unshuffled — which is exactly what WS4 exists to fix. Also worth keeping: the executor's first sweep produced 114 instant "failures" because `timeout` is not a binary on macOS; it read a raw log before trusting the dataset and discarded all of it — [322](322-vitest-shuffle.md) | +| 323 | Run `dist:check` on pull requests | dx/ci | ci | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a7678ba2e1d9b517e`, branch `advisor/323-dist-check-pr-gate`, commit `9c81e74d6`, one file. **The gate was proven to fail before it was wired:** `dist:check` against a development build exits **1** with **180 files** flagged (`react/jsx-dev-runtime` and `jsxDEV` markers in `@lunora/react` and `@lunora/studio` output); against a clean production build it exits **0** ("1282 shipped files across 54 packages carry no development-build markers"). So the alpha.31 defect class does reach a dev build today and currently escapes every PR job. **Accepted deviation, and it is the difference between working and decorative:** the executor also added `dist-production` to `lint-required-check`'s `needs` — reviewer verified the aggregate now lists it. Without that the job would report and never block, and this repo's branch ruleset requires only the aggregate checks. WS3 was deliberately **not** done: `CLAUDE.md:42`'s claim that both gates have CI jobs became true once the job landed, so editing it would be churn. **Unverified by construction:** the job has never been observed running in CI, and no `api-surface`-vs-`dist-production` duration comparison exists — local timings were taken under wildly different load and are not comparable — [323](323-dist-check-pr-gate.md) | +| 324 | Test the shared client mutation path; floor `@lunora/client` | tests | client | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a3e993f1a89ad1ba9`, branch `advisor/324-client-mutation-path-tests`, commit `0979c777c`. Reviewer re-ran: 38 files, 657 passed + 1 expected-fail. `mutation-runner.ts` and `snapshot-precondition.ts` both at 100% branches; floor pinned at 84/75/76/84 with `src/sw/**` excluded (§9 Q1 — a real ~11%-stmt gap, excluded with its reason in the config so later sw work doesn't inherit a floor it must fight). **Both proofs were actually performed, not asserted:** the ref-count test reds when `inFlight` is replaced with a boolean, and bumping `branches` to 99 fails the run with the threshold error. **Trap for anyone re-measuring:** the printed coverage table silently omits rows for both new files — read `coverage/lcov.info` (`BRH==BRF`), not the table. §9 Q2: `src/auth/index.ts` (0% across 95 lines of client auth surface) deserves its own plan — [324](324-client-mutation-path-tests.md) | +| 325 | Test the native-signup email gate | tests/security | auth | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-ac7422e364e41726b`, branch `advisor/325-email-gate-tests`, commit `45a934f7d`, one new 208-line spec. `email-gate.ts` went **0/0/0/0 → 100%** statements/branches/functions/lines; reviewer re-ran the suite: 368/368, `packages/auth/src` untouched. **Both load-bearing proofs were performed:** widening the `typeof email === "string"` check to truthiness reds the passthrough tests, and inverting `withEmailGate`'s gate/existing-hook order reds the ordering tests — each reverted and the empty src diff re-confirmed. No defect found: the gate does run before the caller's hook, and the non-string-email passthrough is correctly guarded. **§9.1** — `onClassify` does **not** fire on rejection (`config.onClassify?.()` sits after the try/catch), matching its docblock. **§9.2** — the `422`/`429` arms of `statusString` are **unreachable today**: every `LunoraError` `assertEmailAllowed` can throw (`VALIDATION_ERROR`, `EMAIL_DOMAIN_BLOCKED`, `EMAIL_UNDELIVERABLE`) carries status 400. Left in place, reasoning recorded inline, since `EmailGateConfig` is user-facing config that could grow a code at another status — a deletion candidate for a later pass, not this one. **Reusable trap:** `vi.mock(import(...))` with a plain top-level `vi.fn()` throws a TDZ `ReferenceError` here because the SUT imports `email-guard` statically, unlike the sibling `email-guard.test.ts` whose mock target is loaded via a runtime dynamic `import()`; `vi.hoisted()` is the fix, and the test file documents it — [325](325-email-gate-tests.md) | +| 326 | Make the shape poke baseline survive hibernation | perf | do/shard-engine | P2 | M | MED | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a411b85e08f3a9548`, branch `advisor/326-shape-poke-cursor-durability`, commits `010e681c0` + `1dc4007fa`. Reviewer re-ran: shard-engine 1121 pass + 1 todo, do 536 pass; measurement scaffolding deleted, never committed. **WS0 measured by counting `readShapeCdcPage` calls, not wall-clock, and the gate did not fire:** warm is always **1** page read; cold is **⌈retained/1000⌉+1** — 0 / 1,000 / 10,000 / 50,000 retained rows → 1 / 2 / 11 / 51 reads. Linear and unbounded, so the durable table was justified rather than assumed. The write costs ~0.04 ms/upsert, structurally identical to the `__global_shape_snapshot` write already in production. **The bigger finding is underneath it: `trimCdcChanges` has ZERO production call sites** (`shard-do.ts` calls it only from tests — grep-verified), so `__cdc_log` retention is effectively unbounded and _that_ is what makes the cold-memo cost unbounded. `CDC_RESUME_SCAN_LIMIT = 10,000` bounds a different read path. **Worth its own plan.** Safety invariant verified by reviewer: the chain degrades **downward** — cache → stored → attachment `sinceSeq` → `0` — so it can never sit above the watermark. The §8 epoch STOP was checked and correctly ruled out: the cursor never leaves the DO's own SQLite, and `bumpCdcEpoch`'s only caller is the PITR restore, which reverts the whole database atomically, so cursor and `readCdcCursor()` always move together. **Scope deviation, accepted:** it edited `packages/shard-engine/src/ctx-db-migrations.ts` (5 lines, CDC-gated) — omitted from the plan's Scope list but explicitly required by WS1's own text. ⚠️ **Merge note: that same file is modified in the working tree by the in-flight `durable-stream` work — expect a conflict** — [326](326-shape-poke-cursor-durability.md) | +| 327 | Manifest hygiene: unused dep, prod-vs-dev dep, catalogs, hono | deps | repo | P3 | S | LOW | TODO — [327](327-manifest-hygiene.md) | +| 328 | Stop recording an attacker-chosen IP in the auth audit trail | security | auth | P3 | S | LOW | TODO — [328](328-audit-log-ip-fallback.md) | +| 329 | Publish the Node host's capability matrix | docs | platform-node | P3 | S | LOW | TODO — [329](329-platform-node-docs.md) | +| 330 | Let Angular produce the preloaded query it can consume | direction | angular | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-ae58d3596d59bc8a0`, branch `advisor/330-angular-preload-query`, commit `30ea1ffe4`. `packages/angular/src/server.ts` is a pure re-export of `@lunora/client/ssr` (verified against that module's live surface, not Solid's copy), `./server` added to the exports map, `dist/server.{mjs,d.ts}` emitted, 108 tests pass, key-order gate clean, `api:check` unchanged (angular is untiered). Reviewer re-ran the criteria independently. **Deviation, accepted:** the export-parity test asserts against `@lunora/client/ssr` rather than `@lunora/solid/server` — strictly stronger (it catches Solid drifting too) and avoids a first-of-its-kind adapter-to-adapter dependency edge. **`templates/analog/` was documented, not wired** — its demo still uses a hand-rolled `LunoraService` over the vanilla client, and swapping it is a separate migration. Two follow-ups opened: an API-snapshot tier decision for `@lunora/angular` now that it publishes a second entry point, and `angular.mdx`'s "Deliberately small surface" callout, which undersells an adapter that already exports `paginatedQuery`/`presence`/`subscription`/`rateLimit`/`stream`/`agent*`/`flag*` — [330](330-angular-preload-query.md) | +| 331 | Resolve the `/worker` subpath contradiction | direction | vue/svelte | P3 | S–M | LOW | **PHASE 0 DONE — awaiting a maintainer decision before Phase 1A.** Evidence in §9, reviewer-verified against the live tree: the subpath has **zero** real importers (the only hit repo-wide is its own docblock example), and `withFrameworkWorker` itself is reached only by codegen's `buildFrameworkWorker` → `@lunora/runtime`, never through a framework `/worker` subpath — the Astro and SvelteKit templates both bypass it, leaving `templates/sveltekit`'s prose stale. `cb6df0762` shows the same defect was already caught and half-fixed once. Recommendation: **Option A** (delete both re-exports, promote `withFrameworkWorker` as the documented BYO-framework entry). Phase 1A removes a published surface, so it is not an executor's call — [331](331-worker-subpath-decision.md) | +| 332 | Spike: what would a payment conformance suite assert? | direction | payment | P3 | S | LOW | ✅ **SPIKE DONE — recommendation: `describe.each`, NOT a published TCK.** Worktree `.claude/worktrees/agent-a581fd0a0e430b202`, one 102-line PoC file; `packages/payment/src` untouched. **It corrected two premises of its own plan, both reviewer-verified.** (1) There are **5** real `PaymentAdapter` implementations, not 6 — `autumn-features.ts` has no `identifier`, no `capabilities`, no `parseWebhook` and is never registered; it is an Autumn-only facade. (2) The plan guessed `portal` was the behaviourally-assertable capability flag. **Wrong — `usageMetering` is the only one with runtime teeth**: `create-payment.ts:452` gates `track()` on it, while `portal` and `merchantOfRecord` are read by **no** runtime code anywhere in `src/` (grep-confirmed). `merchantOfRecord`'s 5/5 correlation with a `notSupported()` `capturePayment` is pure convention nothing enforces; `portal` has no negative-case fixture at all since every shipped provider sets it `true`. **5 shared assertions found** (clears the ≥5 bar, so "do nothing" is also wrong); the PoC — invalid webhook signature → `WEBHOOK_SIGNATURE_INVALID` across all 5 — **passes 5/5**, reviewer re-ran it. **Sizing:** `describe.each` S (~0.5–1 day, ~20 min/assertion measured); published subpath M (1–2 days + a permanent api-snapshot surface) for **zero** current audience — unlike `@lunora/platform`, the payment docs contain no BYO-adapter guidance and no third-party implementation is known. **Highest-value next assertions are not the capability matrix** but the `updateSubscription`/`refundPayment` behaviours plan 320 found broken on 2 of 5 providers; it independently re-verified all three defects from source. Open questions 2 and 3 answered: autumn counts once, and `not-supported.ts` is a shared throw-helper, not a sixth adapter — [332](332-payment-conformance-spike.md) | ### Dependency notes From ab0028599c8ab134356244df1d875656db7829dd Mon Sep 17 00:00:00 2001 From: Daniel Bannert Date: Tue, 11 Aug 2026 11:39:00 +0200 Subject: [PATCH 03/10] refactor: fold the review's structural findings back in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The correctness fixes shipped in the previous commit; these are the structural ones both reviewers raised. The durable-stream state machine moves to @lunora/shard-engine. It touches SQLite through SqlExec, the transcript store next door, and one optional waitUntil — nothing Cloudflare-specific — so it had no business being 250 lines in the repo's largest file. `DurableStreamRunner` owns the run registry and the producer; `ShardDO` keeps only the socket adapter (run key, sink, cancel wiring). The split it had before ran along a persistence/orchestration seam that matched nothing: the store was host-neutral and the logic driving it was not, which is exactly how the Node capability entry came to describe a feature that host cannot reach. One wire vocabulary for both stream paths. `streamFrames(ws, id)` builds the four frames in one place; the ephemeral and durable paths had grown their own copies, and had already drifted on canceller cleanup. The error frame's shape especially wants one home, since the durable path persists what it sends and replays it to every later attach. `EnsureThreadOutcome` replaces a five-field bag of optional booleans whose three states were mutually exclusive. The union is exported, so the loop stops re-declaring the shape inline as a cast, `awaitDequeue` loses its `queued: boolean | undefined` parameter and its internal early-out, and `priorInstanceId` is non-optional exactly where it is read. One in-memory agent-runtime double instead of three. `loop-harness` already exported one and two suites shadowed it with near-copies, so adding the run-completion mutation meant patching the same handler three times — and missing a fourth copy in @lunora/testing entirely, which is how a shipped harness broke. The same goes for the `ctx.db` double two suites had byte-identical copies of. Net: 630 fewer lines of test scaffolding, and one place to change when the runtime grows a function. Also: the RAG sync action is a typed reference rather than `unknown`, so passing the wrong thing is a compile error instead of a silent no-op. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018hGhKLpXkFj3aWUYhZmWQv --- api-snapshots/agent.api.md | 4 + api-snapshots/shard-engine.api.md | 63 +++ packages/agent/__tests__/agent-loop.test.ts | 198 +--------- .../agent/__tests__/agent-state-loop.test.ts | 28 +- packages/agent/__tests__/component.test.ts | 18 +- packages/agent/__tests__/loop-harness.ts | 177 ++++++++- packages/agent/__tests__/run-queue.test.ts | 111 +----- packages/agent/__tests__/skill.test.ts | 159 +------- packages/agent/src/agent-loop.ts | 16 +- packages/agent/src/component.ts | 34 +- packages/agent/src/index.ts | 1 + packages/agent/src/types.ts | 14 + packages/cli/skills/README.md | 11 +- packages/cli/skills/lunora/SKILL.md | 18 + packages/do/src/shard-do.ts | 371 +++--------------- .../shard-engine/src/durable-stream-runner.ts | 287 ++++++++++++++ packages/shard-engine/src/index.ts | 2 + packages/testing/src/agent-harness.ts | 4 +- plans/README.md | 4 +- 19 files changed, 672 insertions(+), 848 deletions(-) create mode 100644 packages/shard-engine/src/durable-stream-runner.ts diff --git a/api-snapshots/agent.api.md b/api-snapshots/agent.api.md index a311ed3a6c..e27b0f7a6b 100644 --- a/api-snapshots/agent.api.md +++ b/api-snapshots/agent.api.md @@ -267,6 +267,10 @@ _Tagged `@experimental` — signature not tracked; churn here does not fail the _Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ +### `EnsureThreadOutcome` (type) + +_Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ + ### `FsToolInput` (type) _Tagged `@experimental` — signature not tracked; churn here does not fail the gate._ diff --git a/api-snapshots/shard-engine.api.md b/api-snapshots/shard-engine.api.md index 5b475823ca..442f40fb58 100644 --- a/api-snapshots/shard-engine.api.md +++ b/api-snapshots/shard-engine.api.md @@ -661,6 +661,24 @@ interface DeployInfo { } ``` +### `DurableAttachDecision` (type) + +```ts +type DurableAttachDecision = "attach" | "interrupted" | "reclaim" | "replay-terminal"; +``` + +### `DurableStreamAttach` (interface) + +```ts +interface DurableStreamAttach { + readonly iterator: (signal: AbortSignal) => AsyncIterable; + readonly runKey: string; + readonly sinceChunk: number; + readonly sink: DurableStreamSink; + readonly ttlMs?: number; +} +``` + ### `DurableStreamRun` (interface) ```ts @@ -673,6 +691,36 @@ interface DurableStreamRun { } ``` +### `DurableStreamRunner` (class) + +```ts +class DurableStreamRunner { + constructor(dependencies: { + sql: () => SqlExec; + waitUntil?: (promise: Promise) => void; + }); + attach(request: DurableStreamAttach): Promise; + detach(runKey: string, sink: DurableStreamSink): void; + isLive(runKey: string): boolean; +} +``` + +### `DurableStreamSink` (interface) + +```ts +interface DurableStreamSink { + readonly chunk: (chunk: { + data: unknown; + seq: number; + }) => boolean; + readonly complete: () => void; + readonly fail: (failure: { + code: string; + message: string; + }) => void; +} +``` + ### `ExportRow` (interface) ```ts @@ -1139,6 +1187,12 @@ const MAIL_RETENTION = 500; const MAIL_TABLE = "__lunora_mail"; ``` +### `MAX_DURABLE_STREAM_CHUNKS` (const) + +```ts +const MAX_DURABLE_STREAM_CHUNKS = 50000; +``` + ### `MAX_PAGE_SIZE` (const) ```ts @@ -3225,6 +3279,15 @@ const createShardCtxDb: (options: CtxDbOptions) => DatabaseWriterLike; const createSystemReader: (options?: SystemReaderOptions) => SystemDatabaseReader; ``` +### `decideDurableAttach` (const) + +```ts +const decideDurableAttach: (run: DurableStreamRun | undefined, context: { + live: boolean; + resuming: boolean; +}) => DurableAttachDecision; +``` + ### `decodeCursor` (const) ```ts diff --git a/packages/agent/__tests__/agent-loop.test.ts b/packages/agent/__tests__/agent-loop.test.ts index f5e83a8835..7aa3b80fd8 100644 --- a/packages/agent/__tests__/agent-loop.test.ts +++ b/packages/agent/__tests__/agent-loop.test.ts @@ -8,7 +8,6 @@ import type { AgentCompact, AgentDefinition, AgentEpisodeExtract, - AgentFunctionReference, AgentGenerate, AgentGenerateResult, AgentGraphExtract, @@ -20,6 +19,7 @@ import type { AgentTokenSink, AgentToolContext, } from "../src/types"; +import { memoryRuntime } from "./loop-harness"; const IN_FLIGHT_PATTERN = /already has a run in flight/u; @@ -111,208 +111,12 @@ const flushMicrotasks = async (): Promise => { }); }; -interface StoredMessage { - content: string; - messageKey: string; - role: "assistant" | "system" | "tool" | "user"; - seq: number; - status?: "approved" | "awaiting_approval" | "rejected"; - stepName?: string; - threadKey: string; - toolCallId?: string; - toolCalls?: { id: string; input: unknown; name: string }[]; - toolName?: string; -} - -interface StoredThread { - agent: string; - error?: string; - instanceId?: string; - key: string; - messageCount: number; - status: string; - title?: string; - usage?: { inputTokens?: number; outputTokens?: number; totalTokens?: number }; -} - /** * In-memory double of the agent runtime functions (`agents:*`), dispatched by * `__lunoraRef` exactly like the real `/_lunora/scheduler/dispatch` runner — * with the same idempotency semantics the component's mutations implement * (keyed appends, get-or-create threads, counter-allocated seq). */ -const memoryRuntime = (options?: { - /** Extra `path → handler` dispatch entries (e.g. an agentic `read` action). */ - handlers?: Record) => unknown>; - /** The memory action's `result` — a fixed value, or a fn of the dispatch args (per-query results). */ - memory?: { path: string; result: Record | ((args?: Record) => unknown) }; -}): { - dispatches: { args: Record | undefined; path: string }[]; - messages: Map; - run: AgentRunFunction; - threads: Map; -} => { - const threads = new Map(); - const messages = new Map(); - const dispatches: { args: Record | undefined; path: string }[] = []; - - const ensureThread = (args?: Record): unknown => { - const key = args?.["key"] as string; - const instanceId = args?.["instanceId"] as string | undefined; - const existing = threads.get(key); - - if (existing) { - // Mirror the real component's concurrency guard: a running thread - // owned by a DIFFERENT instance is a genuine second run. - const isConcurrentRun = - existing.status === "running" && existing.instanceId !== undefined && instanceId !== undefined && existing.instanceId !== instanceId; - - if (isConcurrentRun) { - const policy = (args?.["onConcurrentRun"] as string | undefined) ?? "reject"; - const priorInstanceId = existing.instanceId ?? ""; - - if (policy !== "replace") { - throw new Error(`@lunora/agent: thread "${key}" already has a run in flight (instance "${priorInstanceId}") — onConcurrentRun="${policy}"`); - } - - existing.status = "running"; - existing.instanceId = instanceId; - delete existing.error; - - return { created: false, priorInstanceId, replaced: true }; - } - - existing.status = "running"; - delete existing.error; - - if (instanceId !== undefined) { - existing.instanceId = instanceId; - } - - return { created: false }; - } - - threads.set(key, { - agent: args?.["agent"] as string, - instanceId, - key, - messageCount: 0, - status: "running", - title: args?.["title"] as string | undefined, - }); - - return { created: true }; - }; - - const appendMessage = (args?: Record): unknown => { - const threadKey = args?.["threadKey"] as string; - const messageKey = args?.["messageKey"] as string; - const id = `${threadKey}:${messageKey}`; - const existing = messages.get(id); - - if (existing) { - return { seq: existing.seq }; - } - - const thread = threads.get(threadKey); - - if (!thread) { - throw new Error(`unknown thread ${threadKey}`); - } - - const seq = thread.messageCount; - - thread.messageCount += 1; - messages.set(id, { ...(args as unknown as StoredMessage), seq }); - - return { seq }; - }; - - const listMessages = (args?: Record): unknown => { - const key = args?.["key"] as string; - - return [...messages.values()].filter((message) => message.threadKey === key).toSorted((a, b) => a.seq - b.seq); - }; - - const patchThread = (args?: Record): unknown => { - const thread = threads.get(args?.["key"] as string); - - if (thread) { - if (args?.["status"] !== undefined) { - thread.status = args["status"] as string; - } - - if (args?.["error"] !== undefined) { - thread.error = args["error"] as string; - } - - if (args?.["usage"] !== undefined) { - thread.usage = args["usage"] as StoredThread["usage"]; - } - } - - return undefined; - }; - - /** - * The completion mutation: terminal patch + queue handoff. This double keeps - * no queue (see `run-queue.test.ts` for those semantics), so it ends the run - * exactly as the real one does with an empty queue — including the ownership - * guard that makes a replayed completion a no-op. - */ - const completeRun = (args?: Record): unknown => { - const thread = threads.get(args?.["key"] as string); - - if (!args || !thread || thread.instanceId !== args["instanceId"]) { - return {}; - } - - thread.status = args["status"] as string; - - if (args["error"] !== undefined) { - thread.error = args["error"] as string; - } - - if (args["usage"] !== undefined) { - thread.usage = args["usage"] as StoredThread["usage"]; - } - - return {}; - }; - - const memoryHandler = (args?: Record): unknown => { - const result = options?.memory?.result; - - return typeof result === "function" ? result(args) : result; - }; - - const handlers = new Map) => unknown>([ - [DEFAULT_AGENT_FUNCTION_PATHS.appendMessage, appendMessage], - [DEFAULT_AGENT_FUNCTION_PATHS.completeRun, completeRun], - [DEFAULT_AGENT_FUNCTION_PATHS.ensureThread, ensureThread], - [DEFAULT_AGENT_FUNCTION_PATHS.listMessages, listMessages], - [DEFAULT_AGENT_FUNCTION_PATHS.patchThread, patchThread], - ...(options?.memory ? ([[options.memory.path, memoryHandler]] as const) : []), - ...Object.entries(options?.handlers ?? {}), - ]); - - const run: AgentRunFunction = async (reference: AgentFunctionReference, args?: Record) => { - const path = reference["__lunoraRef"]; - - dispatches.push({ args, path }); - - const handler = handlers.get(path); - - if (!handler) { - throw new Error(`unexpected dispatch: ${path}`); - } - - return handler(args); - }; - - return { dispatches, messages, run, threads }; -}; - /** A scripted LLM: pops one decision per turn, records what it was shown. */ const scriptedGenerate = (script: AgentGenerateResult[]): AgentGenerate & { seen: ReadonlyArray[] } => { const seen: ReadonlyArray[] = []; diff --git a/packages/agent/__tests__/agent-state-loop.test.ts b/packages/agent/__tests__/agent-state-loop.test.ts index e1940e82e6..79d7fb5c48 100644 --- a/packages/agent/__tests__/agent-state-loop.test.ts +++ b/packages/agent/__tests__/agent-state-loop.test.ts @@ -5,28 +5,12 @@ import { agentComponent } from "../src/component"; import { defineAgent, defineAgentTool } from "../src/define-agent"; import { DEFAULT_AGENT_FUNCTION_PATHS } from "../src/paths"; import type { AgentFunctionReference, AgentRunFunction } from "../src/types"; -import { DurableStepJournal, finalTurn, scriptedGenerate, toolTurn } from "./loop-harness"; +import { DurableStepJournal, finalTurn, makeIndexQuery, scriptedGenerate, toolTurn } from "./loop-harness"; interface FakeRow extends Record { _id: string; } -/** Collect the `.eq(...)` conditions a `withIndex` callback declares. */ -const collectConditions = (build: (q: unknown) => unknown): [string, unknown][] => { - const conditions: [string, unknown][] = []; - const builder = { - eq: (field: string, value: unknown) => { - conditions.push([field, value]); - - return builder; - }, - }; - - build(builder); - - return conditions; -}; - const RETRY_MAX_ATTEMPTS = 4; /** @@ -74,16 +58,6 @@ class RetryingStepJournal { } } -const makeIndexQuery = (candidates: FakeRow[], build: (q: unknown) => unknown): { collect: () => Promise; first: () => Promise } => { - const conditions = collectConditions(build); - const matches = (): FakeRow[] => candidates.filter((row) => conditions.every(([field, value]) => row[field] === value)); - - return { - collect: async () => matches(), - first: async () => matches()[0] ?? null, - }; -}; - /** * A `run` that dispatches the loop's `agents:*` refs to the REAL component * functions over an in-memory `ctx.db`, under a verified identity — so a loop diff --git a/packages/agent/__tests__/component.test.ts b/packages/agent/__tests__/component.test.ts index 75819ddd11..6200b57bf9 100644 --- a/packages/agent/__tests__/component.test.ts +++ b/packages/agent/__tests__/component.test.ts @@ -210,8 +210,8 @@ describe(agentComponent, () => { const first = await callMutation(functions.agentEnsureThread, ctx, { agent: "support", key: "t-1", title: "Hello" }); const second = await callMutation(functions.agentEnsureThread, ctx, { agent: "support", key: "t-1" }); - expect(first).toStrictEqual({ created: true }); - expect(second).toStrictEqual({ created: false }); + expect(first).toStrictEqual({ outcome: "created" }); + expect(second).toStrictEqual({ outcome: "continued" }); const appended = await callMutation(functions.agentAppendMessage, ctx, { content: "hi", messageKey: "wf-1:user", role: "user", threadKey: "t-1" }); const replayed = await callMutation(functions.agentAppendMessage, ctx, { content: "hi", messageKey: "wf-1:user", role: "user", threadKey: "t-1" }); @@ -317,7 +317,7 @@ describe("thread ownership", () => { // Same owner (or no owner) continues; a different owner is refused. await expect(callMutation(functions.agentEnsureThread, ctx, { agent: "support", key: "t-1", owner: "user-a" })).resolves.toStrictEqual({ - created: false, + outcome: "continued", }); await expect(callMutation(functions.agentEnsureThread, ctx, { agent: "support", key: "t-1", owner: "user-b" })).rejects.toThrow(ANOTHER_OWNER_PATTERN); }); @@ -1077,7 +1077,7 @@ describe("concurrency guard", () => { const result = await callMutation(functions.agentEnsureThread, ctx, { agent: "support", instanceId: "wf-new", key: "t-1", onConcurrentRun: "replace" }); - expect(result).toStrictEqual({ created: false, priorInstanceId: "wf-old", replaced: true }); + expect(result).toStrictEqual({ outcome: "replaced", priorInstanceId: "wf-old" }); expect(rows.get("agent_threads")?.[0]?.["instanceId"]).toBe("wf-new"); expect(rows.get("agent_threads")?.[0]?.["status"]).toBe("running"); }); @@ -1090,9 +1090,7 @@ describe("concurrency guard", () => { await expect( callMutation(functions.agentEnsureThread, ctx, { agent: "support", instanceId: "wf-a", key: "t-1", onConcurrentRun: "reject" }), - ).resolves.toStrictEqual({ - created: false, - }); + ).resolves.toStrictEqual({ outcome: "continued" }); expect(rows.get("agent_threads")?.[0]?.["status"]).toBe("running"); expect(rows.get("agent_threads")?.[0]?.["instanceId"]).toBe("wf-a"); @@ -1107,7 +1105,7 @@ describe("concurrency guard", () => { await callMutation(functions.agentEnsureThread, ctx, { agent: "support", key: "t-1" }); await expect(callMutation(functions.agentEnsureThread, ctx, { agent: "support", instanceId: "wf-b", key: "t-1" })).resolves.toStrictEqual({ - created: false, + outcome: "continued", }); }); @@ -1143,7 +1141,7 @@ describe("concurrency guard", () => { const result = await callMutation(functions.agentEnsureThread, ctx, { agent: "support", key: "t-1", onConcurrentRun: "replace" }); - expect(result).toStrictEqual({ created: false, priorInstanceId: "wf-old", replaced: true }); + expect(result).toStrictEqual({ outcome: "replaced", priorInstanceId: "wf-old" }); // No new instance id was supplied to record — the column is left as-is // rather than explicitly cleared (which the validators would reject). expect(rows.get("agent_threads")?.[0]?.["instanceId"]).toBe("wf-old"); @@ -1158,7 +1156,7 @@ describe("concurrency guard", () => { const result = await callMutation(functions.agentEnsureThread, ctx, { agent: "support", instanceId: "wf-new", key: "t-1", onConcurrentRun: "replace" }); - expect(result).toStrictEqual({ created: false, priorInstanceId: "wf-old", replaced: true }); + expect(result).toStrictEqual({ outcome: "replaced", priorInstanceId: "wf-old" }); expect(rows.get("agent_threads")?.[0]?.["instanceId"]).toBe("wf-new"); expect(rows.get("agent_threads")?.[0]?.["status"]).toBe("running"); }); diff --git a/packages/agent/__tests__/loop-harness.ts b/packages/agent/__tests__/loop-harness.ts index 99c8d2811c..ea732d6fcb 100644 --- a/packages/agent/__tests__/loop-harness.ts +++ b/packages/agent/__tests__/loop-harness.ts @@ -2,18 +2,20 @@ import type { runAgentLoop } from "../src/agent-loop"; import { DEFAULT_AGENT_FUNCTION_PATHS } from "../src/paths"; import type { AgentDefinition, AgentFunctionReference, AgentGenerate, AgentGenerateResult, AgentRunFunction } from "../src/types"; -interface StoredMessage { +export interface StoredMessage { content: string; messageKey: string; role: "assistant" | "system" | "tool" | "user"; seq: number; + status?: "approved" | "awaiting_approval" | "rejected"; + stepName?: string; threadKey: string; toolCallId?: string; toolCalls?: { id: string; input: unknown; name: string }[]; toolName?: string; } -interface StoredThread { +export interface StoredThread { agent: string; error?: string; instanceId?: string; @@ -21,6 +23,7 @@ interface StoredThread { messageCount: number; status: string; title?: string; + usage?: { inputTokens?: number; outputTokens?: number; totalTokens?: number }; } /** @@ -61,10 +64,19 @@ export class DurableStepJournal { /** * In-memory double of the agent runtime functions (`agents:*`), dispatched by * `__lunoraRef` with the same idempotency semantics the component implements - * (keyed appends, get-or-create threads, counter-allocated seq). Trimmed copy - * of the one in `agent-loop.test.ts`. + * (keyed appends, get-or-create threads, counter-allocated seq). + * + * ONE copy for every suite that drives the loop — `agent-loop`, `skill`, and any + * future one. Three near-identical copies used to exist, so adding a single + * runtime function (the run-completion mutation) meant patching the same handler + * three times and missing it in a fourth place entirely. */ -export const memoryRuntime = (): { +export const memoryRuntime = (options?: { + /** Extra `path → handler` dispatch entries (e.g. an agentic `read` action). */ + handlers?: Record) => unknown>; + /** The memory action's `result` — a fixed value, or a fn of the dispatch args (per-query results). */ + memory?: { path: string; result: Record | ((args?: Record) => unknown) }; +}): { dispatches: { args: Record | undefined; path: string }[]; messages: Map; run: AgentRunFunction; @@ -80,6 +92,26 @@ export const memoryRuntime = (): { const existing = threads.get(key); if (existing) { + // Mirror the real component's concurrency guard: a running thread + // owned by a DIFFERENT instance is a genuine second run. + const isConcurrentRun = + existing.status === "running" && existing.instanceId !== undefined && instanceId !== undefined && existing.instanceId !== instanceId; + + if (isConcurrentRun) { + const policy = (args?.["onConcurrentRun"] as string | undefined) ?? "reject"; + const priorInstanceId = existing.instanceId ?? ""; + + if (policy !== "replace") { + throw new Error(`@lunora/agent: thread "${key}" already has a run in flight (instance "${priorInstanceId}") — onConcurrentRun="${policy}"`); + } + + existing.status = "running"; + existing.instanceId = instanceId; + delete existing.error; + + return { outcome: "replaced", priorInstanceId }; + } + existing.status = "running"; delete existing.error; @@ -87,7 +119,7 @@ export const memoryRuntime = (): { existing.instanceId = instanceId; } - return { created: false }; + return { outcome: "continued" }; } threads.set(key, { @@ -99,7 +131,7 @@ export const memoryRuntime = (): { title: args?.["title"] as string | undefined, }); - return { created: true }; + return { outcome: "created" }; }; const appendMessage = (args?: Record): unknown => { @@ -143,17 +175,20 @@ export const memoryRuntime = (): { if (args?.["error"] !== undefined) { thread.error = args["error"] as string; } + + if (args?.["usage"] !== undefined) { + thread.usage = args["usage"] as StoredThread["usage"]; + } } return undefined; }; /** - * The completion mutation: terminal patch plus the queue handoff. The double - * keeps no queue (the queue's own semantics are covered in - * `run-queue.test.ts`), so it only has to end the run the way the real one - * does for an empty queue — including the ownership guard, which is what - * makes a replayed completion a no-op. + * The completion mutation: terminal patch + queue handoff. This double keeps + * no queue (see `run-queue.test.ts` for those semantics), so it ends the run + * exactly as the real one does with an empty queue — including the ownership + * guard that makes a replayed completion a no-op. */ const completeRun = (args?: Record): unknown => { const thread = threads.get(args?.["key"] as string); @@ -168,15 +203,27 @@ export const memoryRuntime = (): { thread.error = args["error"] as string; } + if (args["usage"] !== undefined) { + thread.usage = args["usage"] as StoredThread["usage"]; + } + return {}; }; + const memoryHandler = (args?: Record): unknown => { + const result = options?.memory?.result; + + return typeof result === "function" ? result(args) : result; + }; + const handlers = new Map) => unknown>([ [DEFAULT_AGENT_FUNCTION_PATHS.appendMessage, appendMessage], [DEFAULT_AGENT_FUNCTION_PATHS.completeRun, completeRun], [DEFAULT_AGENT_FUNCTION_PATHS.ensureThread, ensureThread], [DEFAULT_AGENT_FUNCTION_PATHS.listMessages, listMessages], [DEFAULT_AGENT_FUNCTION_PATHS.patchThread, patchThread], + ...(options?.memory ? ([[options.memory.path, memoryHandler]] as const) : []), + ...Object.entries(options?.handlers ?? {}), ]); const run: AgentRunFunction = async (reference: AgentFunctionReference, args?: Record) => { @@ -236,3 +283,109 @@ export const loopDefaults = (agent: AgentDefinition, overrides?: Partial { + _id: string; +} + +/** Collect the `.eq(...)` conditions a `withIndex` callback declares. */ +export const collectConditions = (build: (q: unknown) => unknown): [string, unknown][] => { + const conditions: [string, unknown][] = []; + const builder = { + eq: (field: string, value: unknown) => { + conditions.push([field, value]); + + return builder; + }, + }; + + build(builder); + + return conditions; +}; + +/** Filter by the `.eq(...)` conditions; insertion order stands in for index order. */ +export const makeIndexQuery = ( + candidates: FakeRow[], + build: (q: unknown) => unknown, +): { collect: () => Promise; first: () => Promise } => { + const conditions = collectConditions(build); + const matches = (): FakeRow[] => candidates.filter((row) => conditions.every(([field, value]) => row[field] === value)); + + return { + collect: async () => matches(), + first: async () => matches()[0] ?? null, + }; +}; + +/** + * An in-memory `ctx.db`. `withIndex` filters by the declared equalities and — as + * the real index read does — returns rows in insertion order, which for the + * `(threadKey, position)` index is position order. + */ +export const fakeDatabase = (): { database: Record; rows: Map } => { + const rows = new Map(); + let nextId = 0; + + const tableRows = (table: string): FakeRow[] => { + const existing = rows.get(table); + + if (existing) { + return existing; + } + + const created: FakeRow[] = []; + + rows.set(table, created); + + return created; + }; + + const database = { + delete: async (id: string) => { + for (const [table, tableContent] of rows) { + rows.set( + table, + tableContent.filter((row) => row["_id"] !== id), + ); + } + }, + insert: async (table: string, document: Record) => { + const id = `id-${String(nextId)}`; + + nextId += 1; + tableRows(table).push({ ...document, _id: id }); + + return id; + }, + patch: async (id: string, patch: Record) => { + for (const tableContent of rows.values()) { + const row = tableContent.find((candidate) => candidate["_id"] === id); + + if (row) { + for (const [key, value] of Object.entries(patch)) { + if (value === undefined) { + Reflect.deleteProperty(row, key); + } else { + row[key] = value; + } + } + } + } + }, + query: (table: string) => { + return { + withIndex: (_name: string, build: (q: unknown) => unknown) => makeIndexQuery(tableRows(table), build), + }; + }, + }; + + return { database, rows }; +}; diff --git a/packages/agent/__tests__/run-queue.test.ts b/packages/agent/__tests__/run-queue.test.ts index 27ec3fd3cd..a44d3a60bc 100644 --- a/packages/agent/__tests__/run-queue.test.ts +++ b/packages/agent/__tests__/run-queue.test.ts @@ -9,102 +9,9 @@ import { describe, expect, it } from "vitest"; import { agentComponent } from "../src/component"; - -interface FakeRow extends Record { - _id: string; -} - -/** Collect the `.eq(...)` conditions a `withIndex` callback declares. */ -const collectConditions = (build: (q: unknown) => unknown): [string, unknown][] => { - const conditions: [string, unknown][] = []; - const builder = { - eq: (field: string, value: unknown) => { - conditions.push([field, value]); - - return builder; - }, - }; - - build(builder); - - return conditions; -}; - -/** Filter by the `.eq(...)` conditions; insertion order stands in for index order. */ -const makeIndexQuery = (candidates: FakeRow[], build: (q: unknown) => unknown): { collect: () => Promise; first: () => Promise } => { - const conditions = collectConditions(build); - const matches = (): FakeRow[] => candidates.filter((row) => conditions.every(([field, value]) => row[field] === value)); - - return { - collect: async () => matches(), - first: async () => matches()[0] ?? null, - }; -}; - -/** - * An in-memory `ctx.db`. `withIndex` filters by the declared equalities and — as - * the real index read does — returns rows in insertion order, which for the - * `(threadKey, position)` index is position order. - */ -const fakeDatabase = (): { database: Record; rows: Map } => { - const rows = new Map(); - let nextId = 0; - - const tableRows = (table: string): FakeRow[] => { - const existing = rows.get(table); - - if (existing) { - return existing; - } - - const created: FakeRow[] = []; - - rows.set(table, created); - - return created; - }; - - const database = { - delete: async (id: string) => { - for (const [table, tableContent] of rows) { - rows.set( - table, - tableContent.filter((row) => row["_id"] !== id), - ); - } - }, - insert: async (table: string, document: Record) => { - const id = `id-${String(nextId)}`; - - nextId += 1; - tableRows(table).push({ ...document, _id: id }); - - return id; - }, - patch: async (id: string, patch: Record) => { - for (const tableContent of rows.values()) { - const row = tableContent.find((candidate) => candidate["_id"] === id); - - if (row) { - for (const [key, value] of Object.entries(patch)) { - if (value === undefined) { - Reflect.deleteProperty(row, key); - } else { - row[key] = value; - } - } - } - } - }, - query: (table: string) => { - return { - withIndex: (_name: string, build: (q: unknown) => unknown) => makeIndexQuery(tableRows(table), build), - }; - }, - }; - - return { database, rows }; -}; +import type { EnsureThreadOutcome } from "../src/types"; +import type { FakeRow } from "./loop-harness"; +import { fakeDatabase } from "./loop-harness"; const setup = () => { const { database, rows } = fakeDatabase(); @@ -121,7 +28,7 @@ const setup = () => { key: "thread-1", onConcurrentRun: "queue", ...(instanceId === undefined ? {} : { instanceId }), - } as never)) as { created: boolean; position?: number; queued?: boolean }, + } as never)) as EnsureThreadOutcome, thread: (): FakeRow | undefined => (rows.get("agent_threads") ?? [])[0], }; }; @@ -132,9 +39,9 @@ describe("onConcurrentRun: queue", () => { const { complete, queue, start, thread } = setup(); - await expect(start("wf-a")).resolves.toStrictEqual({ created: true }); - await expect(start("wf-b")).resolves.toStrictEqual({ created: false, position: 0, queued: true }); - await expect(start("wf-c")).resolves.toStrictEqual({ created: false, position: 1, queued: true }); + await expect(start("wf-a")).resolves.toStrictEqual({ outcome: "created" }); + await expect(start("wf-b")).resolves.toStrictEqual({ outcome: "queued", position: 0 }); + await expect(start("wf-c")).resolves.toStrictEqual({ outcome: "queued", position: 1 }); // A parked run must not take the thread from the one in flight. expect(thread()?.["instanceId"]).toBe("wf-a"); @@ -167,7 +74,7 @@ describe("onConcurrentRun: queue", () => { await start("wf-b"); // A workflow replay re-runs the bootstrap for real (it is outside step.do). - await expect(start("wf-b")).resolves.toStrictEqual({ created: false, position: 0, queued: true }); + await expect(start("wf-b")).resolves.toStrictEqual({ outcome: "queued", position: 0 }); expect(queue()).toHaveLength(1); }); @@ -261,7 +168,7 @@ describe("onConcurrentRun: queue", () => { Object.assign(thread() ?? {}, { updatedAt: Date.now() - 14 * 60 * 60 * 1000 }); // A new run takes it rather than CONFLICTing against a corpse forever. - await expect(start("wf-z")).resolves.toStrictEqual({ created: false }); + await expect(start("wf-z")).resolves.toStrictEqual({ outcome: "continued" }); expect(thread()?.["instanceId"]).toBe("wf-z"); }); diff --git a/packages/agent/__tests__/skill.test.ts b/packages/agent/__tests__/skill.test.ts index 086e43c107..98e7ec3267 100644 --- a/packages/agent/__tests__/skill.test.ts +++ b/packages/agent/__tests__/skill.test.ts @@ -4,7 +4,8 @@ import { runAgentLoop } from "../src/agent-loop"; import { defineAgent, defineAgentTool } from "../src/define-agent"; import { DEFAULT_AGENT_FUNCTION_PATHS } from "../src/paths"; import { defineSkill, isSkillDefinition } from "../src/skill"; -import type { AgentDefinition, AgentFunctionReference, AgentGenerate, AgentGenerateResult, AgentInstructionsContext, AgentRunFunction } from "../src/types"; +import type { AgentDefinition, AgentGenerate, AgentGenerateResult, AgentInstructionsContext } from "../src/types"; +import { memoryRuntime } from "./loop-harness"; const NAME_PATTERN = /identifier/u; const COLLIDES_PATTERN = /collides/u; @@ -46,152 +47,12 @@ class DurableStepJournal { } } -interface StoredMessage { - content: string; - messageKey: string; - role: "assistant" | "system" | "tool" | "user"; - seq: number; - threadKey: string; - toolCallId?: string; - toolCalls?: { id: string; input: unknown; name: string }[]; - toolName?: string; -} - -interface StoredThread { - agent: string; - error?: string; - instanceId?: string; - key: string; - messageCount: number; - status: string; -} - /** - * In-memory double of the agent runtime functions (`agents:*`) plus an - * arbitrary set of memory-retrieval actions, dispatched by `__lunoraRef` with - * the same idempotency semantics the component implements. + * The shared runtime double, adapted to this suite's shape: each skill-knowledge + * source is an extra dispatch handler returning a fixed result. */ -const memoryRuntime = ( - memories: ReadonlyArray<{ path: string; result: unknown }> = [], -): { - dispatches: { args: Record | undefined; path: string }[]; - messages: Map; - run: AgentRunFunction; - threads: Map; -} => { - const threads = new Map(); - const messages = new Map(); - const dispatches: { args: Record | undefined; path: string }[] = []; - - const ensureThread = (args?: Record): unknown => { - const key = args?.["key"] as string; - const instanceId = args?.["instanceId"] as string | undefined; - const existing = threads.get(key); - - if (existing) { - existing.status = "running"; - delete existing.error; - - if (instanceId !== undefined) { - existing.instanceId = instanceId; - } - - return { created: false }; - } - - threads.set(key, { agent: args?.["agent"] as string, instanceId, key, messageCount: 0, status: "running" }); - - return { created: true }; - }; - - const appendMessage = (args?: Record): unknown => { - const threadKey = args?.["threadKey"] as string; - const messageKey = args?.["messageKey"] as string; - const id = `${threadKey}:${messageKey}`; - const existing = messages.get(id); - - if (existing) { - return { seq: existing.seq }; - } - - const thread = threads.get(threadKey); - - if (!thread) { - throw new Error(`unknown thread ${threadKey}`); - } - - const seq = thread.messageCount; - - thread.messageCount += 1; - messages.set(id, { ...(args as unknown as StoredMessage), seq }); - - return { seq }; - }; - - const listMessages = (args?: Record): unknown => { - const key = args?.["key"] as string; - - return [...messages.values()].filter((message) => message.threadKey === key).toSorted((a, b) => a.seq - b.seq); - }; - - const patchThread = (args?: Record): unknown => { - const thread = threads.get(args?.["key"] as string); - - if (thread) { - if (args?.["status"] !== undefined) { - thread.status = args["status"] as string; - } - - if (args?.["error"] !== undefined) { - thread.error = args["error"] as string; - } - } - - return undefined; - }; - - /** Terminal patch + (empty-queue) handoff — see `run-queue.test.ts` for the queue itself. */ - const completeRun = (args?: Record): unknown => { - const thread = threads.get(args?.["key"] as string); - - if (!args || !thread || thread.instanceId !== args["instanceId"]) { - return {}; - } - - thread.status = args["status"] as string; - - if (args["error"] !== undefined) { - thread.error = args["error"] as string; - } - - return {}; - }; - - const handlers = new Map) => unknown>([ - [DEFAULT_AGENT_FUNCTION_PATHS.appendMessage, appendMessage], - [DEFAULT_AGENT_FUNCTION_PATHS.completeRun, completeRun], - [DEFAULT_AGENT_FUNCTION_PATHS.ensureThread, ensureThread], - [DEFAULT_AGENT_FUNCTION_PATHS.listMessages, listMessages], - [DEFAULT_AGENT_FUNCTION_PATHS.patchThread, patchThread], - ...memories.map((memory) => [memory.path, (): unknown => memory.result] as const), - ]); - - const run: AgentRunFunction = async (reference: AgentFunctionReference, args?: Record) => { - const path = reference["__lunoraRef"]; - - dispatches.push({ args, path }); - - const handler = handlers.get(path); - - if (!handler) { - throw new Error(`unexpected dispatch: ${path}`); - } - - return handler(args); - }; - - return { dispatches, messages, run, threads }; -}; +const skillRuntime = (memories: ReadonlyArray<{ path: string; result: unknown }> = []) => + memoryRuntime({ handlers: Object.fromEntries(memories.map((memory) => [memory.path, (): unknown => memory.result])) }); /** A scripted LLM: pops one decision per turn, recording what it was shown. */ const scriptedGenerate = (script: AgentGenerateResult[]): AgentGenerate & { seen: ReadonlyArray[] } => { @@ -228,7 +89,7 @@ const loopDefaults = (agent: AgentDefinition, overrides?: Partial { skills: [billing], }); - const runtime = memoryRuntime([ + const runtime = skillRuntime([ { path: "rag:general", result: { context: "General runtime facts." } }, { path: "rag:billing", result: { context: "Invoices are billed monthly." } }, ]); @@ -414,7 +275,7 @@ describe("skill knowledge retrieval in the loop", () => { it("keeps the exact `memory:retrieve` step for a plain memory agent (back-compat, no skills)", async () => { const agent = defineAgent({ memory: { source: "rag:general", topK: 3 }, model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast" }); - const runtime = memoryRuntime([{ path: "rag:general", result: { context: "Just the default source." } }]); + const runtime = skillRuntime([{ path: "rag:general", result: { context: "Just the default source." } }]); const journal = new DurableStepJournal(); const generate = scriptedGenerate([finalTurn("answered")]); @@ -431,7 +292,7 @@ describe("skill knowledge retrieval in the loop", () => { it("runs no memory step for an agent with neither memory nor skill knowledge", async () => { const agent = defineAgent({ model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast" }); - const runtime = memoryRuntime(); + const runtime = skillRuntime(); const journal = new DurableStepJournal(); await runAgentLoop(loopDefaults(agent, { generate: scriptedGenerate([finalTurn("hi")]), run: runtime.run, step: journal })); diff --git a/packages/agent/src/agent-loop.ts b/packages/agent/src/agent-loop.ts index 6291ad6e08..a5df8c94e5 100644 --- a/packages/agent/src/agent-loop.ts +++ b/packages/agent/src/agent-loop.ts @@ -32,6 +32,7 @@ import type { AgentUsage, AgentWorkflowBindingLike, AnyAgentTool, + EnsureThreadOutcome, } from "./types"; /** @@ -791,11 +792,7 @@ const DEQUEUE_TIMEOUT = "12 hours"; * different one. A workflow replay memoizes the resolved wait, so a resumed run * does not park twice. */ -const awaitDequeue = async (step: AgentStepLike, threadKey: string, instanceId: string, queued: boolean | undefined): Promise => { - if (queued !== true) { - return; - } - +const awaitDequeue = async (step: AgentStepLike, threadKey: string, instanceId: string): Promise => { await step.waitForEvent<{ threadKey: string }>(`dequeue:${instanceId}`, { timeout: DEQUEUE_TIMEOUT, type: `agent-dequeue:${threadKey}:${instanceId}`, @@ -1083,11 +1080,11 @@ const runAgentLoop = async (options: AgentLoopOptions): Promise ...(agent.onConcurrentRun === undefined ? {} : { onConcurrentRun: agent.onConcurrentRun }), ...(params.owner === undefined ? {} : { owner: params.owner }), ...(params.title === undefined ? {} : { title: params.title }), - })) as { created: boolean; position?: number; priorInstanceId?: string; queued?: boolean; replaced?: boolean } | undefined; + })) as EnsureThreadOutcome | undefined; // Replace policy: the thread has been taken over — terminate the run it was // taken from so it cannot resume and race on the shared seq counter. - if (bootstrap?.replaced && bootstrap.priorInstanceId !== undefined) { + if (bootstrap?.outcome === "replaced") { await terminatePriorInstance(env, exportName, bootstrap.priorInstanceId); } @@ -1139,7 +1136,10 @@ const runAgentLoop = async (options: AgentLoopOptions): Promise // escaped without running `finishRun` — leaving this run's queue row // behind forever. Five of those exhaust the depth cap and every later // start on the thread is refused. - await awaitDequeue(step, params.threadKey, instanceId, bootstrap?.queued); + if (bootstrap?.outcome === "queued") { + await awaitDequeue(step, params.threadKey, instanceId); + } + await persistUserTurn(); const { final, stoppedByCondition, turnsRun } = await runTurns(turnContext, maxTurns, stopConditions, usageBox); diff --git a/packages/agent/src/component.ts b/packages/agent/src/component.ts index 575ad104ba..06729947be 100644 --- a/packages/agent/src/component.ts +++ b/packages/agent/src/component.ts @@ -7,6 +7,7 @@ import type { AgentRegisteredFunction } from "./component-shared"; import { AGENT_EXTENSION_KEY, asInternal, definedColumns } from "./component-shared"; import { episodeTables, episodicComponent } from "./episodic-component"; import { graphComponent, graphTables } from "./graph-component"; +import type { EnsureThreadOutcome } from "./types"; /** Bare table names — auto-prefixed with the extension key at merge time. */ const THREADS_BARE_TABLE = "threads"; @@ -184,23 +185,6 @@ const agentExtension: SchemaExtension = defineSchemaExtension(AGENT_EXTENSION_KE // generated server inside a package), same as the presence component. const { mutation, query } = initLunora.dataModel().create(); -/** - * What `agentEnsureThread` reports back to the loop: the thread was created, - * continued, taken over from a prior instance (`replaced`), or — under - * `onConcurrentRun: "queue"` — parked behind the run in flight (`queued`), in - * which case the loop hibernates until the queue hands it the thread. - * @experimental - */ -interface EnsureThreadResult { - created: boolean; - /** FIFO position among the runs parked on this thread. Present only with `queued`. */ - position?: number; - /** The instance the thread was taken over FROM. Present only with `replaced`. */ - priorInstanceId?: string; - queued?: boolean; - replaced?: boolean; -} - /** * The minimal writer surface the queue helpers need. `agentEnsureThread` and * `agentCompleteRun` are ordinary mutations, so this is just `ctx.db` — named @@ -223,7 +207,7 @@ const enqueueRun = async ( database: QueueDatabase, args: { agent: string; instanceId: string | undefined; key: string; priorInstanceId: string }, now: number, -): Promise => { +): Promise => { if (args.instanceId === undefined) { // The id-less dispatch paths (inbound email / inbound channel) can't be // told apart from each other later, so they can't be parked and woken by @@ -242,7 +226,7 @@ const enqueueRun = async ( .first(); if (alreadyQueued) { - return { created: false, position: alreadyQueued["position"] as number, queued: true }; + return { outcome: "queued", position: alreadyQueued["position"] as number }; } const queued = await database @@ -261,7 +245,7 @@ const enqueueRun = async ( await database.insert(RUN_QUEUE_TABLE, { agent: args.agent, enqueuedAt: now, instanceId, position, threadKey: args.key }); - return { created: false, position, queued: true }; + return { outcome: "queued", position }; }; /** @@ -282,7 +266,7 @@ const applyConcurrencyPolicy = async ( priorInstanceId: string; }, now: number, -): Promise => { +): Promise => { if (args.policy === "queue") { return enqueueRun(database, { agent: args.agent, instanceId: args.instanceId, key: args.key, priorInstanceId: args.priorInstanceId }, now); } @@ -306,7 +290,7 @@ const applyConcurrencyPolicy = async ( ...(args.instanceId === undefined ? {} : { instanceId: args.instanceId }), }); - return { created: false, priorInstanceId: args.priorInstanceId, replaced: true }; + return { outcome: "replaced", priorInstanceId: args.priorInstanceId }; }; /** @@ -360,7 +344,7 @@ export const agentComponent = (): AgentComponent => { owner: v.optional(v.string()), title: v.optional(v.string()), }) - .mutation(async ({ args, ctx: context }): Promise => { + .mutation(async ({ args, ctx: context }): Promise => { const now = Date.now(); const existing = await context.db .query(THREADS_TABLE) @@ -433,7 +417,7 @@ export const agentComponent = (): AgentComponent => { ...(args.instanceId === undefined ? {} : { instanceId: args.instanceId }), }); - return { created: false }; + return { outcome: "continued" }; } await context.db.insert(THREADS_TABLE, { @@ -446,7 +430,7 @@ export const agentComponent = (): AgentComponent => { ...definedColumns({ instanceId: args.instanceId, owner: args.owner, state: args.initialState, title: args.title }), }); - return { created: true }; + return { outcome: "created" }; }); const agentAppendMessage = mutation diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 802fb5be53..618735e6ba 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -70,6 +70,7 @@ export type { AgentWorkflowBindingLike, AgentWorkflowInstanceLike, AnyAgentTool, + EnsureThreadOutcome, SkillConfig, SkillDefinition, } from "./types"; diff --git a/packages/agent/src/types.ts b/packages/agent/src/types.ts index 20f52ea6a4..3574e642e6 100644 --- a/packages/agent/src/types.ts +++ b/packages/agent/src/types.ts @@ -891,6 +891,20 @@ export interface AgentDefinition extends AgentConfig { memorySources?: ReadonlyArray; } +/** + * What `agentEnsureThread` reports back to the loop. + * + * A discriminated union rather than a bag of optional booleans: the four + * outcomes are mutually exclusive, and the data each carries only exists for its + * own case. `queued` has a position, `replaced` has the instance it took the + * thread from, and the other two have nothing — encoding that as five + * independent optional fields made every reader re-derive which combination was + * legal. + * @experimental + */ +export type EnsureThreadOutcome = + { outcome: "continued" | "created" } | { outcome: "queued"; position: number } | { outcome: "replaced"; priorInstanceId: string }; + /** * Params of one agent run (the compiled workflow's payload). * @experimental diff --git a/packages/cli/skills/README.md b/packages/cli/skills/README.md index 208bb83728..40d56efd3a 100644 --- a/packages/cli/skills/README.md +++ b/packages/cli/skills/README.md @@ -26,15 +26,18 @@ own directory: | `lunora-deploy` | Deploying to Cloudflare — wrangler bindings, secrets, the gate. | | `lunora-performance-audit` | `lunora insights`, scans, indexes, OCC conflicts, sharding/`.global()`. | -These are mirrored into `.agents/skills/` and `.claude/skills/` (via symlinks) -so agents working inside this repo pick them up directly. The source of truth -lives here so the published `@lunora/cli` tarball carries them. +These are mirrored (via symlinks) into `.agents/skills/` and `.claude/skills/`, +so agents working inside this repo pick them up directly, and into +`plugins/lunora/skills/`, the payload of the Claude Code / Codex plugin. The +source of truth lives here so the published `@lunora/cli` tarball carries them — +`lunora rules install` copies this directory into a consumer project. -**Adding a skill?** Create it here, then mirror it through both hops: +**Adding a skill?** Create it here, then mirror it through all three hops: ```bash ln -s ../../packages/cli/skills/ .agents/skills/ ln -s ../../.agents/skills/ .claude/skills/ +ln -s ../../../packages/cli/skills/ plugins/lunora/skills/ ``` `scripts/check-skill-mirrors.js` (run on every `pnpm install`) fails if a mirror diff --git a/packages/cli/skills/lunora/SKILL.md b/packages/cli/skills/lunora/SKILL.md index 6134fc9e13..9c59311ebc 100644 --- a/packages/cli/skills/lunora/SKILL.md +++ b/packages/cli/skills/lunora/SKILL.md @@ -63,6 +63,24 @@ After codegen is green, use the most specific Lunora skill for the task: - Investigating performance, scan, or write-conflict issues: `lunora-performance-audit` +**Reviewing or auditing existing `lunora/` code** has no skill of its own, +because two commands do the work mechanically — run them before reading +anything: + +```bash +lunora verify # wrangler config + codegen dry-run + tsc --noEmit +lunora advisor # ~95 static + runtime lints, scored per procedure +``` + +`lunora advisor` is the security/quality review: RLS coverage +(`rls-procedures`, `procedure-protections`), ownership and identity checks +(`owner-field-writes`, `normalize-id-authorization`, `identity-claim-reads`, +`fail-open-guards`), input validators, unindexed reads, non-deterministic +calls in queries, SQL interpolation, and leaked secrets. Use +`--entry #` to inspect one procedure and `--min-score` / +`--baseline` to gate CI. Only after it is clean is a by-hand pass over +`lunora-functions` worth the tokens. + If one of those clearly matches the user's goal, switch to it instead of staying in this skill. diff --git a/packages/do/src/shard-do.ts b/packages/do/src/shard-do.ts index fd3055d5c9..3dc621bcd0 100644 --- a/packages/do/src/shard-do.ts +++ b/packages/do/src/shard-do.ts @@ -77,7 +77,7 @@ import type { ColumnMeta, CreateWorkflowInstanceResult, DependencyTracker, - DurableStreamRun, + DurableStreamSink, ExportRow, FanoutMetricsResult, FlagsResult, @@ -129,14 +129,12 @@ import { ADMIN_FUNCTIONS, advanceClientWatermark, appendAuditEntry, - appendStreamChunk, armRestore, awaitWsDrain, buildPokeFrames, buildSettings, bumpCdcEpoch, CDC_LOG_TABLE, - claimStreamRun, clearCapturedMail, clearQueueMessages, ConflictError, @@ -149,12 +147,11 @@ import { DEFAULT_MAX_RELAYS, deleteGlobalShapeSnapshot, deleteGlobalShapeSnapshotsForConnection, - deleteStreamRun, diffGlobalMembership, + DurableStreamRunner, ensureAuditTable, facetColumn, findStorageReferences, - finishStreamRun, FLAGS_FUNCTION_PREFIX, gateReplicaDispatch, handleReplicaControl, @@ -184,8 +181,6 @@ import { readMigrationStatus, readQueueMessageById, readQueueMessages, - readStreamChunks, - readStreamRun, readTablePage, recordCapturedMail, recordChangedKeys, @@ -208,7 +203,6 @@ import { summarizeSubscriptions, TransactionHeadroomTracker, trimIdempotent, - trimStreamRuns, trySendFrame, writeGlobalShapeSnapshot, writeIdempotent, @@ -734,71 +728,30 @@ const IDEMPOTENCY_RETENTION_MS = 86_400_000; const IDEMPOTENCY_GC_INTERVAL_MS = 3_600_000; /** - * How long a durable stream's transcript is kept when the procedure does not - * set its own `ttlMs`. A day covers "I reloaded", "I reopened the tab this - * evening", and a second viewer arriving late, without keeping every - * transcript a chatty shard ever produced. - */ -const DURABLE_STREAM_DEFAULT_TTL_MS = 86_400_000; - -/** Minimum spacing between durable-stream TTL sweeps, amortized onto a stream start. */ -const DURABLE_STREAM_GC_INTERVAL_MS = 3_600_000; - -/** - * Hard ceiling on the chunks one durable run may persist. - * - * A token-at-a-time generation is thousands of chunks, which is fine; a runaway - * generator is unbounded, which is not — the transcript lives in the shard's - * SQLite alongside application data, so "grow until the storage limit" takes the - * whole shard down. 50k covers any real answer with room to spare. - */ -const MAX_DURABLE_STREAM_CHUNKS = 50_000; - -/** - * What an attach should do about the run row it found. + * The stream wire vocabulary, bound to one socket + stream id. * - * Pulled out of `attachDurableStream` because the rule is the interesting part - * and it is pure: whether a stored run may serve this caller turns entirely on - * whether the caller is RESUMING one it already holds part of, or asking fresh. - * - * `replay-terminal` is a finished run whose watcher came back: hand back the tail - * plus the recorded outcome. `interrupted` is a producer that died mid-run while - * this caller holds a prefix — the tail cannot be spliced on and re-generating - * would duplicate it, so say so rather than pretend. `reclaim` is a stored run - * this caller has no claim on (finished, or dead with nobody resuming): drop it - * and produce, because a transcript is the record of one execution rather than a - * cached response, and an eviction must not wedge the key until its TTL expires. - * `attach` joins the live producer, or starts one when there is no run. + * Both stream paths — the ephemeral per-socket iterator and the durable + * runner's sink — speak the same four frames. Building them in one place keeps + * the two from drifting (they already had, on canceller cleanup), and keeps the + * shape of an `error` frame in a single spot, which matters because the durable + * path PERSISTS the message it sends and replays it to every later attach. */ -const decideDurableAttach = ( - run: DurableStreamRun | undefined, - context: { live: boolean; resuming: boolean }, -): "attach" | "interrupted" | "reclaim" | "replay-terminal" => { - if (run === undefined || context.live) { - return "attach"; - } - - if (run.status === "complete" || run.status === "error") { - return context.resuming ? "replay-terminal" : "reclaim"; - } - - // `running` with no live producer: the instance died mid-generation. - return context.resuming ? "interrupted" : "reclaim"; -}; - -/** - * One socket attached to a durable stream run. The producer fans every chunk - * out to each attached sink and calls exactly one terminal (`complete`/`fail`); - * a sink that has detached is simply no longer in the set. - */ -interface DurableStreamSink { - /** Deliver one chunk. Returns `false` when the socket refused it (already closed). */ - readonly chunk: (chunk: { data: unknown; seq: number }) => boolean; - /** The run finished successfully — send `complete` and release the socket's stream slot. */ - readonly complete: () => void; - /** The run failed — send the (already redacted) error and release the socket's stream slot. */ - readonly fail: (failure: { code: string; message: string }) => void; -} +const streamFrames = ( + ws: ShardSocketLike, + id: string, +): { + ack: () => void; + chunk: (data: unknown, seq?: number) => boolean; + complete: () => boolean; + fail: (failure: { code: string; message: string }) => boolean; +} => {return { + ack: () => { + ws.send(JSON.stringify({ id, type: "ack" })); + }, + chunk: (data, seq) => trySendFrame(ws, JSON.stringify(seq === undefined ? { data, id, type: "chunk" } : { data, id, seq, type: "chunk" })), + complete: () => trySendFrame(ws, JSON.stringify({ id, type: "complete" })), + fail: (failure) => trySendFrame(ws, JSON.stringify({ error: failure, id, type: "error" })), +}}; /** * Reserved shard name for the fallback Durable Object that hosts every @@ -1341,10 +1294,12 @@ abstract class ShardDO { * says `running` was interrupted by an eviction — see * {@link ShardDO.attachDurableStream}. */ - private readonly durableStreamRuns = new Map }>(); - /** Throttle for the durable-stream TTL sweep — at most one pass per {@link DURABLE_STREAM_GC_INTERVAL_MS}. */ - private lastDurableStreamTrimAt = 0; + /** + * Durable-stream runs for this shard. The engine owns the state machine and + * the producer; this class only adapts sockets onto it. + */ + private readonly durableStreams = new DurableStreamRunner({ sql: () => this.sql as SqlExec, waitUntil: (promise) => this.shardHost.waitUntil?.(promise) }); /** * Lifetime request counters surfaced by the `__lunora_admin__:getMetrics` @@ -7682,28 +7637,19 @@ abstract class ShardDO { } /** - * Attach a socket to a **durable** stream run, starting the run if this is - * the first attach. - * - * The run is keyed by `(functionPath, args)`, which is what makes the - * feature work: a reload sends the same start frame with the `seq` it last - * saw, lands on the same run, replays the gap out of SQLite, and continues - * live. A second tab asking the same question watches the same transcript - * instead of paying for a second generation. + * Attach a socket to a **durable** stream run. * - * Four states, all reachable on a first frame. **No run:** claim it and - * start the producer, detached from this socket. **Running with a live - * producer:** replay the gap, then attach a sink. **Finished - * (`complete`/`error`):** replay the transcript and send the recorded - * terminal frame, with no producer. **Running with no live producer:** the - * DO was evicted mid-run — replay what survived, then fail with - * `STREAM_INTERRUPTED` rather than silently re-running the handler, which - * would bill a model call twice and duplicate the tail. + * Everything about the run's lifecycle — who may attach to a stored + * transcript, when a dead run is reclaimed, driving the producer — lives in + * `@lunora/shard-engine`'s {@link DurableStreamRunner}, because none of it is + * Cloudflare-specific. This method is the socket adapter: it derives the run + * key, turns the WebSocket into a sink, and wires the client's cancel. * - * **Known ceiling:** fan-out does not apply socket backpressure the way the - * ephemeral path does. A sink whose `send` throws is dropped, and the client - * resumes from its last `seq` on reconnect — the transcript is durable, so - * dropping a slow socket loses nothing. + * The run key folds in the socket's **verified identity**. Sharing a live run + * is the feature — a second tab of the same user watches one generation + * instead of paying for two — but an attach never drives the handler, so a + * run shared across identities would also skip the procedure's RLS + * middleware. Identity-scoping the key is what keeps that impossible. */ private async attachDurableStream( ws: ShardSocketLike, @@ -7713,21 +7659,14 @@ abstract class ShardDO { registration: { durable: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable }, sinceChunk: number, ): Promise { - const sql = this.sql as SqlExec; - const ttlMs = registration.durable.ttlMs ?? DURABLE_STREAM_DEFAULT_TTL_MS; - // The socket's VERIFIED identity is part of the run key, not just the - // arguments. Sharing a transcript is the feature, but it must never be - // shared across identities: two users asking the same question with the - // same args are two runs, or the second silently reads the first's - // answer — and would skip the procedure's `.use(rls(...))` chain, since - // an attach to an existing run never drives the handler. const owner = this.readAttachment(ws).userId ?? ""; const runKey = `${owner}\u0000${functionPath}:${stableWireKey(args)}`; const cancellers = socketMap(this.streamCancellers, ws); const controller = new AbortController(); + const frames = streamFrames(ws, id); cancellers.set(id, controller); - ws.send(JSON.stringify({ id, type: "ack" })); + frames.ack(); const detach = (): void => { cancellers.delete(id); @@ -7739,231 +7678,43 @@ abstract class ShardDO { } }; - // Everything below runs without awaiting: a DO handles one event at a - // time, so replay-then-attach cannot interleave with the producer and - // there is no window in which a chunk is neither replayed nor delivered. let delivered = 0; - const sendChunk = (chunk: { data: unknown; seq: number }): boolean => { - if (chunk.seq <= delivered) { - return true; - } - - delivered = chunk.seq; - - return trySendFrame(ws, JSON.stringify({ data: chunk.data, id, seq: chunk.seq, type: "chunk" })); - }; - - const run = readStreamRun(sql, runKey); - // A client that already holds part of this transcript is RESUMING; one at - // zero is asking for an answer. The distinction decides every branch - // below, because a finished run is a transcript of one execution — not a - // cached response to hand the next caller who asks the same thing. - const resuming = sinceChunk > 0; - const replay = (): boolean => { - for (const chunk of readStreamChunks(sql, runKey, sinceChunk)) { - if (controller.signal.aborted || !sendChunk({ data: JSON.parse(chunk.dataJson) as unknown, seq: chunk.seq })) { - detach(); - - return false; + const sink: DurableStreamSink = { + chunk: (chunk) => { + if (chunk.seq <= delivered) { + return true; } - } - - return true; - }; - - const live = this.durableStreamRuns.get(runKey); - const decision = decideDurableAttach(run, { live: live !== undefined, resuming }); - - if (decision === "replay-terminal") { - if (!replay()) { - return; - } - - trySendFrame( - ws, - run?.status === "complete" - ? JSON.stringify({ id, type: "complete" }) - : JSON.stringify({ - error: { code: run?.errorCode ?? "INTERNAL_SERVER_ERROR", message: run?.error ?? "stream failed" }, - id, - type: "error", - }), - ); - detach(); - - return; - } - if (decision === "interrupted") { - if (!replay()) { - return; - } + delivered = chunk.seq; - trySendFrame( - ws, - JSON.stringify({ - error: { code: "STREAM_INTERRUPTED", message: "the producer for this durable stream did not survive; start a new run" }, - id, - type: "error", - }), - ); - detach(); - - return; - } - - if (decision === "reclaim") { - deleteStreamRun(sql, runKey); - } - - const sink: DurableStreamSink = { - chunk: sendChunk, + return frames.chunk(chunk.data, chunk.seq); + }, complete: () => { - trySendFrame(ws, JSON.stringify({ id, type: "complete" })); + frames.complete(); detach(); }, fail: (failure) => { - trySendFrame(ws, JSON.stringify({ error: failure, id, type: "error" })); + frames.fail(failure); detach(); }, }; - // A consumer cancelling a durable stream detaches it; the producer keeps + // Cancelling a durable stream detaches this consumer; the producer keeps // going so the transcript completes for whoever attaches next. That is // the whole point of declaring it durable. controller.signal.addEventListener("abort", () => { - this.durableStreamRuns.get(runKey)?.sinks.delete(sink); + this.durableStreams.detach(runKey, sink); + detach(); }); - if (live) { - // Replay the prefix this attach missed, THEN subscribe. No await - // between the two, so the producer cannot slip a chunk into the gap. - if (!replay()) { - return; - } - - live.sinks.add(sink); - - return; - } - - this.trimDurableStreams(sql); - claimStreamRun(sql, runKey, Date.now(), ttlMs); - - const state = { sinks: new Set([sink]) }; - - this.durableStreamRuns.set(runKey, state); - - // Detached from the socket and handed to `waitUntil` where the host - // exposes one, so the run isn't cut short the moment the last socket - // closes — a durable stream that dies with its opener is just an - // ephemeral stream with extra writes. Awaited as well so this method's - // own rejection path stays attached (the caller catches and discards). - const producing = this.produceDurableStream(runKey, registration.iterator, state); - - this.shardHost.waitUntil?.(producing); - - await producing; - } - - /** - * Drive a durable run's producer: persist each chunk under the next `seq`, - * then fan it out to whichever sockets are attached right now (possibly - * none — a run whose viewers all left still finishes and is still there for - * the next attach). - */ - private async produceDurableStream( - runKey: string, - iterator: (signal: AbortSignal) => AsyncIterable, - state: { sinks: Set }, - ): Promise { - const sql = this.sql as SqlExec; - // Not wired to any consumer's cancel: see `attachDurableStream`. The - // controller exists only to satisfy the handler's signature. - const controller = new AbortController(); - // The producer owns the sequence: it is the only writer, and every - // attach reads the persisted transcript rather than this counter. - let seq = 0; - - try { - for await (const chunk of iterator(controller.signal)) { - const data = encodeWire(chunk); - - seq += 1; - - if (seq > MAX_DURABLE_STREAM_CHUNKS) { - // A runaway generator would otherwise grow the shard's SQLite - // without bound and take every other function on it down with - // the storage limit. Fail loudly at a ceiling no real answer - // reaches instead of degrading the whole shard. - throw new LunoraError( - "STREAM_TOO_LONG", - `durable stream exceeded ${String(MAX_DURABLE_STREAM_CHUNKS)} chunks; yield larger chunks or drop \`durable\``, - { status: 507 }, - ); - } - - appendStreamChunk(sql, runKey, seq, JSON.stringify(data)); - - for (const sink of state.sinks) { - // A sink whose socket refused the frame is gone: drop it - // rather than re-attempting it on every remaining chunk. The - // transcript is durable, so it loses nothing — the client - // resumes from its last `seq`. - if (!sink.chunk({ data, seq })) { - state.sinks.delete(sink); - } - } - } - - finishStreamRun(sql, runKey, "complete", seq); - - for (const sink of state.sinks) { - sink.complete(); - } - } catch (error: unknown) { - // Same redaction invariant as the ephemeral path: an intentional - // `LunoraError` keeps its message, anything else is redacted before - // it is persisted — the transcript is replayed to every later - // attach, so a leaked internal message would leak repeatedly. - const { body, redacted } = toErrorBody(error, { fallbackCode: "INTERNAL_SERVER_ERROR", redactedMessage: "internal error" }); - - if (redacted) { - // eslint-disable-next-line no-console -- server-side diagnostic for an internal/unhandled stream error - console.error("[@lunora/do] unhandled durable stream error:", error); - } - - finishStreamRun(sql, runKey, "error", seq, { code: body.code, message: body.message }); - - for (const sink of state.sinks) { - sink.fail({ code: body.code, message: body.message }); - } - } finally { - this.durableStreamRuns.delete(runKey); - } - } - - /** - * Throttled TTL sweep for durable-stream transcripts, amortized onto a run - * start. Each run carries its own `ttl_ms`, so the sweep never applies the - * triggering procedure's retention to somebody else's transcript. - */ - private trimDurableStreams(sql: SqlExec): void { - const now = Date.now(); - - if (now - this.lastDurableStreamTrimAt <= DURABLE_STREAM_GC_INTERVAL_MS) { - return; - } - - this.lastDurableStreamTrimAt = now; - - try { - trimStreamRuns(sql, now); - } catch { - // Best-effort GC (missing table on a pre-migration shard / stub sql - // handle) must never fail the stream that triggered it. - } + await this.durableStreams.attach({ + iterator: registration.iterator, + runKey, + sinceChunk, + sink, + ...(registration.durable.ttlMs === undefined ? {} : { ttlMs: registration.durable.ttlMs }), + }); } /** diff --git a/packages/shard-engine/src/durable-stream-runner.ts b/packages/shard-engine/src/durable-stream-runner.ts new file mode 100644 index 0000000000..bd059838df --- /dev/null +++ b/packages/shard-engine/src/durable-stream-runner.ts @@ -0,0 +1,287 @@ +/** + * The durable-stream state machine: who may attach to a stored run, and who + * drives the producer. + * + * Lives here rather than in a host because nothing about it is host-specific — + * it touches SQLite through {@link SqlExec}, the transcript store next door, and + * one optional `waitUntil`. The host's job is narrower: turn a socket into a + * {@link DurableStreamSink} and hand over a run key. + * + * The rule the whole feature turns on: **a transcript is the record of one + * execution, not a cached response.** A caller resuming a run it already holds + * part of gets the rest; a caller asking the same question fresh gets a fresh + * run. Getting that backwards makes a durable stream a response cache with a + * 24-hour TTL — stale answers for everyone, and a failed run re-served as the + * same error with no way to retry. + */ +import { LunoraError, toErrorBody } from "@lunora/errors"; + +import { encodeWire } from "../../../shared/wire-codec"; +import type { SqlExec } from "./ctx-db"; +import type { DurableStreamRun } from "./durable-stream"; +import { appendStreamChunk, claimStreamRun, deleteStreamRun, finishStreamRun, readStreamChunks, readStreamRun, trimStreamRuns } from "./durable-stream"; + +/** + * Hard ceiling on the chunks one run may persist. + * + * A token-at-a-time generation is thousands of chunks, which is fine; a runaway + * generator is unbounded, which is not — the transcript shares the shard's + * SQLite with application data, so "grow until the storage limit" takes every + * other function on that shard down with it. + */ +const MAX_DURABLE_STREAM_CHUNKS = 50_000; + +/** Default retention for a finished transcript: long enough to survive a reload and a commute. */ +const DEFAULT_TTL_MS = 86_400_000; + +/** Minimum spacing between TTL sweeps, amortized onto a run start. */ +const GC_INTERVAL_MS = 3_600_000; + +/** + * One consumer attached to a run. The producer fans every chunk out to each + * attached sink and calls exactly one terminal; a detached sink is simply no + * longer in the set. + */ +interface DurableStreamSink { + /** Deliver one chunk. `false` = the consumer is gone, and the sink is dropped. */ + readonly chunk: (chunk: { data: unknown; seq: number }) => boolean; + /** The run finished successfully. */ + readonly complete: () => void; + /** The run failed — the message is already redacted. */ + readonly fail: (failure: { code: string; message: string }) => void; +} + +/** What an attach should do about the run row it found — see {@link decideDurableAttach}. */ +type DurableAttachDecision = "attach" | "interrupted" | "reclaim" | "replay-terminal"; + +/** + * Decide how a stored run may serve this caller. + * + * Pure, and the interesting part of the feature: the answer turns entirely on + * whether the caller is RESUMING a transcript it already holds part of, or + * asking fresh. + * + * `replay-terminal` is a finished run whose watcher came back — hand back the + * tail plus the recorded outcome. `interrupted` is a producer that died mid-run + * while this caller holds a prefix; the tail cannot be spliced on and + * re-generating would duplicate it, so say so rather than pretend. `reclaim` is + * a stored run this caller has no claim on (finished, or dead with nobody + * resuming) — drop it and produce, so an eviction cannot wedge the key until its + * TTL expires. `attach` joins the live producer, or starts one. + */ +const decideDurableAttach = (run: DurableStreamRun | undefined, context: { live: boolean; resuming: boolean }): DurableAttachDecision => { + if (run === undefined || context.live) { + return "attach"; + } + + if (run.status === "complete" || run.status === "error") { + return context.resuming ? "replay-terminal" : "reclaim"; + } + + // `running` with no live producer: the instance died mid-generation. + return context.resuming ? "interrupted" : "reclaim"; +}; + +/** One attach request, from the host's point of view. */ +interface DurableStreamAttach { + /** Builds the handler's async iterable; called only when this attach starts the run. */ + readonly iterator: (signal: AbortSignal) => AsyncIterable; + /** Identity of the run — the host folds in the caller's identity, function path, and arguments. */ + readonly runKey: string; + /** Highest chunk `seq` the caller already holds; `0` means "asking fresh". */ + readonly sinceChunk: number; + readonly sink: DurableStreamSink; + /** Retention for a run this attach starts. */ + readonly ttlMs?: number; +} + +/** + * Drives durable runs for one shard. In-memory state tracks who is *currently* + * producing; the transcript itself lives in SQLite, which is what lets an attach + * on a later instance still replay it. + */ +class DurableStreamRunner { + private lastTrimAt = 0; + + private readonly runs = new Map }>(); + + private readonly sql: () => SqlExec; + + private readonly waitUntil: ((promise: Promise) => void) | undefined; + + public constructor(dependencies: { sql: () => SqlExec; waitUntil?: (promise: Promise) => void }) { + this.sql = dependencies.sql; + this.waitUntil = dependencies.waitUntil; + } + + /** + * Attach a consumer, starting the run when this attach is the first. + * + * Nothing awaits between reading the run row and registering the sink — a DO + * handles one event at a time — so there is no window in which a chunk is + * neither replayed nor delivered live. + */ + public async attach(request: DurableStreamAttach): Promise { + const sql = this.sql(); + const { runKey, sinceChunk, sink } = request; + const run = readStreamRun(sql, runKey); + const resuming = sinceChunk > 0; + const live = this.runs.get(runKey); + const decision = decideDurableAttach(run, { live: live !== undefined, resuming }); + const replay = (): boolean => { + for (const chunk of readStreamChunks(sql, runKey, sinceChunk)) { + if (!sink.chunk({ data: JSON.parse(chunk.dataJson) as unknown, seq: chunk.seq })) { + return false; + } + } + + return true; + }; + + if (decision === "replay-terminal") { + if (replay()) { + if (run?.status === "error") { + sink.fail({ code: run.errorCode ?? "INTERNAL_SERVER_ERROR", message: run.error ?? "stream failed" }); + } else { + sink.complete(); + } + } + + return; + } + + if (decision === "interrupted") { + if (replay()) { + sink.fail({ code: "STREAM_INTERRUPTED", message: "the producer for this durable stream did not survive; start a new run" }); + } + + return; + } + + if (live) { + // Replay the prefix this attach missed, THEN subscribe. + if (replay()) { + live.sinks.add(sink); + } + + return; + } + + if (decision === "reclaim") { + deleteStreamRun(sql, runKey); + } + + this.trim(sql); + claimStreamRun(sql, runKey, Date.now(), request.ttlMs ?? DEFAULT_TTL_MS); + + const state = { sinks: new Set([sink]) }; + + this.runs.set(runKey, state); + + // Detached from the socket and handed to `waitUntil` where the host has + // one, so the run isn't cut short the moment the last consumer leaves — + // a durable stream that dies with its opener is an ephemeral stream with + // extra writes. Awaited as well so the caller's error path stays attached. + const producing = this.produce(runKey, request.iterator, state); + + this.waitUntil?.(producing); + + await producing; + } + + /** Detach a consumer without disturbing the producer — cancelling a durable stream leaves the run going. */ + public detach(runKey: string, sink: DurableStreamSink): void { + this.runs.get(runKey)?.sinks.delete(sink); + } + + /** Whether a run is currently producing on this instance. */ + public isLive(runKey: string): boolean { + return this.runs.has(runKey); + } + + /** + * Drive one run: persist each chunk under the next `seq`, then fan it out to + * whichever consumers are attached right now — possibly none, since a run + * whose viewers all left still finishes and is still there for the next + * attach. + */ + private async produce(runKey: string, iterator: (signal: AbortSignal) => AsyncIterable, state: { sinks: Set }): Promise { + const sql = this.sql(); + // Not wired to any consumer's cancel: a consumer leaving must not end the + // run. The controller exists to satisfy the handler's signature. + const controller = new AbortController(); + let seq = 0; + + try { + for await (const chunk of iterator(controller.signal)) { + const data = encodeWire(chunk); + + seq += 1; + + if (seq > MAX_DURABLE_STREAM_CHUNKS) { + throw new LunoraError( + "STREAM_TOO_LONG", + `durable stream exceeded ${String(MAX_DURABLE_STREAM_CHUNKS)} chunks; yield larger chunks or drop \`durable\``, + { status: 507 }, + ); + } + + appendStreamChunk(sql, runKey, seq, JSON.stringify(data)); + + for (const sink of state.sinks) { + // A consumer that refused the frame is gone: drop it rather + // than re-attempting on every remaining chunk. It loses + // nothing — the transcript is durable and it resumes from its + // last `seq`. + if (!sink.chunk({ data, seq })) { + state.sinks.delete(sink); + } + } + } + + finishStreamRun(sql, runKey, "complete", seq); + + for (const sink of state.sinks) { + sink.complete(); + } + } catch (error: unknown) { + // Redact BEFORE persisting: the stored message is replayed to every + // later attach, so a leaked internal detail would leak repeatedly. + const { body, redacted } = toErrorBody(error, { fallbackCode: "INTERNAL_SERVER_ERROR", redactedMessage: "internal error" }); + + if (redacted) { + // eslint-disable-next-line no-console -- server-side diagnostic for an internal/unhandled stream error + console.error("[@lunora/shard-engine] unhandled durable stream error:", error); + } + + finishStreamRun(sql, runKey, "error", seq, { code: body.code, message: body.message }); + + for (const sink of state.sinks) { + sink.fail({ code: body.code, message: body.message }); + } + } finally { + this.runs.delete(runKey); + } + } + + /** Throttled TTL sweep. Each run carries its own retention, so one procedure's sweep never trims another's transcripts. */ + private trim(sql: SqlExec): void { + const now = Date.now(); + + if (now - this.lastTrimAt <= GC_INTERVAL_MS) { + return; + } + + this.lastTrimAt = now; + + try { + trimStreamRuns(sql, now); + } catch { + // Best-effort GC (pre-migration shard, stub sql handle) must never + // fail the stream that triggered it. + } + } +} + +export type { DurableAttachDecision, DurableStreamAttach, DurableStreamSink }; +export { decideDurableAttach, DurableStreamRunner, MAX_DURABLE_STREAM_CHUNKS }; diff --git a/packages/shard-engine/src/index.ts b/packages/shard-engine/src/index.ts index b870fa5f55..04b97ed700 100644 --- a/packages/shard-engine/src/index.ts +++ b/packages/shard-engine/src/index.ts @@ -117,6 +117,8 @@ export { readStreamRun, trimStreamRuns, } from "./durable-stream"; +export type { DurableAttachDecision, DurableStreamAttach, DurableStreamSink } from "./durable-stream-runner"; +export { decideDurableAttach, DurableStreamRunner, MAX_DURABLE_STREAM_CHUNKS } from "./durable-stream-runner"; export type { ExternalSourceDiffResult } from "./external-source-diff"; export { diffExternalSource } from "./external-source-diff"; export { liftSourceId, normalizeSourceDocument, normalizeSourceValue } from "./external-source-lift"; diff --git a/packages/testing/src/agent-harness.ts b/packages/testing/src/agent-harness.ts index 069168815e..e679b0e9e1 100644 --- a/packages/testing/src/agent-harness.ts +++ b/packages/testing/src/agent-harness.ts @@ -185,7 +185,7 @@ const createRuntime = ( existing.instanceId = instanceId; } - return { created: false }; + return { outcome: "continued" }; } threads.set(key, { @@ -198,7 +198,7 @@ const createRuntime = ( title: args?.["title"] as string | undefined, }); - return { created: true }; + return { outcome: "created" }; }; const appendMessage = (args?: Record): unknown => { diff --git a/plans/README.md b/plans/README.md index 81e5af82be..ca73a82c85 100644 --- a/plans/README.md +++ b/plans/README.md @@ -1612,11 +1612,11 @@ session is mid-feature there. | Plan | Title | Category | Pkg | Pri | Effort | Risk | Status | | ---- | ------------------------------------------------------------- | -------------- | ----------------------- | --- | ------ | ---- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | 316 | Scope the client mutation watermark by identity | bug | client/db | P1 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a68bfc5859c3a2d80`, branch `advisor/316-client-watermark-identity-scope`, commit `5de113e65`, four files. Reviewer re-ran: client 650 pass + 1 expected-fail, db 87/87, `api:check` clean. The regression test does the real thing — ack 5 under identity A, switch to B → **0**, switch back to A → **5**, on a real `LunoraClient`. **The plan's WS1 as literally written was incomplete, and the mandated stash-and-prove is what caught it:** composite-keying alone breaks the plan's own contract §3.2, because a same-token _subject resolve_ relabels the fingerprint (hash → `subj:…`) and makes the cached watermark invisible under the new key — reintroducing the identical wedge on every routine subject resolution. Fixed with `restampWatermarks(from, to)` called from `setAuthToken`'s existing same-token branch, mirroring the established `restampQueuedIdentity` sibling. The executor nearly reported done with 2 of 3 tests passing before adding the third. **Better than the plan on one detail:** the composite separator is `�`, not the `�` the plan suggested — a NUL would have made the file binary to `grep`, which has bitten this repo before; `file` confirms it is still UTF-8 text. `debug()`'s projection strips the prefix and filters to the current identity, so no other identity's watermark can leak into it. §9: no LRU needed (one entry per visited identity×bucket, same shape as the file's existing unbounded-but-small maps); the durable `OutboxSink` replay path already has its own `currentIdentity()` guard — no gap — [316](316-client-watermark-identity-scope.md) | -| 317 | Route every `.dev.vars` write through the atomic 0600 writer | security | cli | P1 | S | LOW | TODO — [317](317-dev-vars-atomic-write.md) | +| 317 | Route every `.dev.vars` write through the atomic 0600 writer | security | cli | P1 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a211304207fda7ea9`, branch `advisor/317-dev-vars-atomic-write`, commit `4f813642b`, three files. Reviewer verified: `writeFileSync` is **gone** from `env/handler.ts`, and `registry/apply.ts` retains exactly the two out-of-scope writes (`package.json:155`, `wrangler.jsonc:426`). Env suite 37/37 on a re-run. **The regression test is real and was proven both ways** — `it.skipIf(win32)` asserts `statSync(".dev.vars").mode & 0o777 === 0o600` after `env generate --set` creates the file fresh; reverting WS1 yields **420 (0o644)** against the expected **384 (0o600)**. Two STOP conditions were checked and cleared before starting: `writeDevVariablesFileAtomically` is exported (`config/src/index.ts:111`), `@lunora/cli` already depends on `@lunora/config` (`cli/package.json:73`), and the writer's temp file is a **sibling** (`${path}.tmp-${pid}`), so there is no cross-device `EXDEV` rename risk. **§9 answered — recommend yes:** `lunora doctor` should warn when an existing `.dev.vars` is group- or world-readable. It is one POSIX-only `statSync`, and without it a file created 0644 by a pre-fix CLI stays silently exposed forever, since this plan deliberately ships no in-tool rotation warning. Follow-up target: `runEnvDoctor` in `cli/src/commands/env/handler.ts` — [317](317-dev-vars-atomic-write.md) | | 318 | Stop the Vite dev loop emitting from a stale TS program | bug | vite/codegen | P1 | S | LOW | TODO — [318](318-vite-stale-codegen-project.md) | | 319 | Bound the dispatch call; classify deterministic 4xx | bug | dispatch/workflow/queue | P1 | S–M | MED | ✅ **DONE (WS1+WS3+workflow half) — APPROVED, awaiting merge. Queue half BLOCKED by its own STOP, correctly.** Worktree `.claude/worktrees/agent-aaba9f993d37eac1d`, branch `advisor/319-dispatch-timeout-and-retry-class`, commits `bf4a23863` + `970244c04`. Reviewer re-ran both suites: dispatch 27/27, workflow 81/81. The safety invariant holds — `DETERMINISTIC_DISPATCH_STATUSES = {400,403,404,422}` with `408`/`429` excluded in the constant's own comment, and the abort error is minted at **503**, deliberately outside that set, so a timeout stays retryable. **Queue half stopped for the sanctioned reason:** `dispatchQueueBatch` unconditionally rethrows the handler's error (`if (threw) { throw handlerError; }`), `deadLettered` is post-hoc metadata rather than a mechanism, and an existing test pins the rethrow as the deliberate contract — so ack-without-retry would be a redelivery-semantics change, not a bug fix. **§9 Q1 remains unconfirmed**: this implements the docblock's contract; if the maintainer prefers today's retry-everything behaviour, drop commit `970244c04` and edit the docblock — `bf4a23863` (the timeout) stands either way. **Follow-up it surfaced:** `@lunora/queue` re-exports `RunFunctionOptions` from dispatch so it picked up `timeoutMs` automatically, but `@lunora/workflow` declares its own copy in `workflow/src/types.ts`, so `timeoutMs` is invisible on a workflow step's typed `ctx.run` even though the runtime honours it — [319](319-dispatch-timeout-and-retry-class.md) | | 320 | Cover the Stripe/Polar money-mutating methods | tests | payment | P1 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a43c3ad10c64a298f`, branch `advisor/320-stripe-polar-money-tests`, commit `d9314a192`. 16 new tests (8 per provider), 195/195 pass; reviewer re-ran coverage independently: `stripe.ts` branches 43.39% → **65.09%**, `polar.ts` 33.67% → **60.2%**, aggregate 68.8%, floor raised 60 → 68. `packages/payment/src` untouched (verified). **It found three latent defects on the money path and correctly pinned current behaviour rather than fixing it** — each needs its own plan: (1) **Stripe `updateSubscription` with an empty `items.data` sends `id: undefined`**, which Stripe treats as _creating a new subscription item_ rather than updating one — a silent billing split, not a crash, which is why the plan's "if it throws" STOP never fired (`providers/stripe.ts:406`); (2) **Polar's `refundPayment` never returns `partially_refunded`** — it has no `compareMoney` boundary check at all and always reports `refunded` (`providers/polar.ts:289-315`); (3) **Polar's `updateSubscription` silently drops `quantity`** — a quantity-only patch produces an empty `subscriptionUpdate: {}` no-op (`providers/polar.ts:338-342`). Also recommended: `resumeSubscription` coverage for Creem/Dodo (both at 0) — [320](320-stripe-polar-money-tests.md) | -| 321 | Restore the coverage floor on six packages; fix the generator | tests/dx | repo | P2 | S | LOW | TODO — [321](321-coverage-floor-gaps.md) | +| 321 | Restore the coverage floor on six packages; fix the generator | tests/dx | repo | P2 | S | LOW | 🟡 **WS1/WS2/WS4 DONE — APPROVED. WS3's guard BLOCKED on a maintainer decision.** Worktree `.claude/worktrees/agent-ab50d9ff17fca8a80`, branch `advisor/321-coverage-floor-gaps`, commits `2953ae6fe` + `81ada196f` + `edbc0b6ab`. All six packages now carry measured floors; reviewer confirmed and re-ran `agent` (338 tests, 88.52/80.34/88.82/88.52 against its 88/80/88/88 floor). Live-floor proof performed: `agent` statements → 99 fails with the threshold error, restored to 88 → passes. Template now emits `test:coverage`. **Adding `include: ["src"]` lowered two packages** — `agent` 89.29 → 88.52 statements, `angular` 81.1 → 79.8 — exactly the denominator effect the plan warned about, and the reason floors were pinned from the post-change run. `angular` is below default on all four metrics with a `// ratchet:` comment; `browser` and `search-core` are near-ceiling. **WS3's check stopped on §8's second STOP, correctly: `packages/auth-ui` has no `build`/`build:prod` by design** (`private: true`, internal, ships raw `.ts`/`.css`, no `dist`), so the required set as specified fails against a package this plan never touches. The three fixes — drop those two from the set, allowlist `auth-ui` (the file already models documented exceptions), or add stub scripts — are maintainer calls, not executor calls. **Two of my own plan's premises were wrong and are now corrected in the plan file:** the script does **not** run from `postinstall` (it is wired to `lint:project-json`), and the `grep -L "thresholds"` done-criterion is invalid for helper-routed packages, since the word lives in `tools/get-vitest-config.ts` rather than the call site — [321](321-coverage-floor-gaps.md) | | 322 | Make the randomized test seed actually randomize | tests | repo | P2 | S→M | MED | 🟡 **WS1+WS2 DONE — APPROVED as far as it goes; WS3/WS4 still open.** Worktree `.claude/worktrees/agent-a7d4a9d9a94d95128`, branch `advisor/322-vitest-shuffle`, commit `6aa02e52b`, four lines in one file. Shuffle is **default-off** behind `LUNORA_SHUFFLE=1` — reviewer confirmed a plain run is unchanged. Both WS1 proofs hold: two flagged runs of `@lunora/values` give different file orders, and a pinned seed reproduces byte-identically. **The reproducibility check needed `--no-file-parallelism` to be meaningful at all** — with parallel files the reporter prints _completion_ order, not schedule order, so the same seed looks different every run; that belongs in the plan's method, not just this row. **Sweep: 37 of 38 clean, 0 confirmed order-coupling.** `@lunora/cli` failed twice with _different_ failure sets under _different_ seeds and then passed 90/90 on a serial replay of either seed — the signature of genuine flakiness (real subprocesses/fs), not order-coupling, so it is recorded unconfirmed. **Two caveats that bound what this proves.** (a) **My call, and it weakened the evidence:** I authorized one run per package instead of the plan's three, trading depth for breadth while the machine was busy. So WS3's gate ("WS2 complete for every package", three runs each) is **not** satisfied — read this as "no order-coupling in a single-run sweep", not a clearance to flip the default. (b) **`auth`'s PASS is meaningless**: its config never calls the helper, so `LUNORA_SHUFFLE` is a no-op there and the run was unshuffled — which is exactly what WS4 exists to fix. Also worth keeping: the executor's first sweep produced 114 instant "failures" because `timeout` is not a binary on macOS; it read a raw log before trusting the dataset and discarded all of it — [322](322-vitest-shuffle.md) | | 323 | Run `dist:check` on pull requests | dx/ci | ci | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a7678ba2e1d9b517e`, branch `advisor/323-dist-check-pr-gate`, commit `9c81e74d6`, one file. **The gate was proven to fail before it was wired:** `dist:check` against a development build exits **1** with **180 files** flagged (`react/jsx-dev-runtime` and `jsxDEV` markers in `@lunora/react` and `@lunora/studio` output); against a clean production build it exits **0** ("1282 shipped files across 54 packages carry no development-build markers"). So the alpha.31 defect class does reach a dev build today and currently escapes every PR job. **Accepted deviation, and it is the difference between working and decorative:** the executor also added `dist-production` to `lint-required-check`'s `needs` — reviewer verified the aggregate now lists it. Without that the job would report and never block, and this repo's branch ruleset requires only the aggregate checks. WS3 was deliberately **not** done: `CLAUDE.md:42`'s claim that both gates have CI jobs became true once the job landed, so editing it would be churn. **Unverified by construction:** the job has never been observed running in CI, and no `api-surface`-vs-`dist-production` duration comparison exists — local timings were taken under wildly different load and are not comparable — [323](323-dist-check-pr-gate.md) | | 324 | Test the shared client mutation path; floor `@lunora/client` | tests | client | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a3e993f1a89ad1ba9`, branch `advisor/324-client-mutation-path-tests`, commit `0979c777c`. Reviewer re-ran: 38 files, 657 passed + 1 expected-fail. `mutation-runner.ts` and `snapshot-precondition.ts` both at 100% branches; floor pinned at 84/75/76/84 with `src/sw/**` excluded (§9 Q1 — a real ~11%-stmt gap, excluded with its reason in the config so later sw work doesn't inherit a floor it must fight). **Both proofs were actually performed, not asserted:** the ref-count test reds when `inFlight` is replaced with a boolean, and bumping `branches` to 99 fails the run with the threshold error. **Trap for anyone re-measuring:** the printed coverage table silently omits rows for both new files — read `coverage/lcov.info` (`BRH==BRF`), not the table. §9 Q2: `src/auth/index.ts` (0% across 95 lines of client auth surface) deserves its own plan — [324](324-client-mutation-path-tests.md) | From 75f72fcfc0a92f8af13ec40b2eea735844865931 Mon Sep 17 00:00:00 2001 From: Daniel Bannert Date: Tue, 11 Aug 2026 11:52:32 +0200 Subject: [PATCH 04/10] docs(plans): remove the shipped plan file, keep the record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `plans/README.md` says a completed plan is deleted once shipped and its record lives in the index plus git history. 312 was the only file still present that had actually shipped — verified against live code rather than its own status line, since those have been wrong in both directions before: `isLiveForCompanion` is in `query-args.ts` and wired through `ctx-db-companions.ts` and `ctx-db-backfill.ts`. Its two spun-out follow-ups (314, 315) keep their own files. Everything else in `plans/` is genuinely TODO, a spike whose remaining work is open, or a reference doc gating unbuilt follow-ons — so nothing else qualifies. Two corrections found while checking: the index prose still claimed 312 was "the only one still open" while its own table row two paragraphs earlier said DONE, and the row's spun-out link pointed at 314's number with 315's filename. Also drops the wave-21 rows this branch had swept into its earlier commit from another session's uncommitted work in the same checkout. They stay in the working tree, uncommitted, where they belong. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018hGhKLpXkFj3aWUYhZmWQv --- plans/312-bigint-aggregate-gaps.md | 416 ----------------------------- plans/README.md | 130 +-------- 2 files changed, 10 insertions(+), 536 deletions(-) delete mode 100644 plans/312-bigint-aggregate-gaps.md diff --git a/plans/312-bigint-aggregate-gaps.md b/plans/312-bigint-aggregate-gaps.md deleted file mode 100644 index 70f9ee416e..0000000000 --- a/plans/312-bigint-aggregate-gaps.md +++ /dev/null @@ -1,416 +0,0 @@ -# Plan 312 — Close the two gaps that leave a `v.bigint()` column unaggregatable - -**Baseline:** `e67cd58a9` (2026-08-08) — the head of `fix/265-wire-codec-query-parity` (#365) -**Status:** EXECUTED (2026-08-09) — gap 1 fixed, gap 2 decided and kept loud, and a -**third, silent** defect found on the way. See §10. -**Priority:** P2 · **Effort:** M · **Risk:** MED · **Category:** bug/design - -> **Executor instructions**: both gaps are **loud** today — they throw, they do not -> return wrong answers. That is deliberate and it is the bar to preserve: do not -> close either gap by making an aggregate return an approximate number. If the only -> way to support a case is to round, the honest answer is to keep throwing. - -## 0. Headline finding - -#365 made `v.bigint()` columns exactly comparable by storing them as a fixed-width -order-preserving decimal key. Padded text is not something SQLite can add up, so the -aggregate paths grew two guards. Each is correct in isolation; together they leave -two shapes with **no way to aggregate a bigint column at all**: - -1. **A soft-delete table cannot use the aggregate companion, at any magnitude.** - `ctx-db.ts`'s aggregate reader takes the maintained-companion fast path only when - `definition.aggregateIndexes && !aggOptions.baseWhere && !hasRelation && !aggScope`. - For a soft-delete table `aggScope` is _always_ truthy (it is `softDeleteScope(definition.softDeleteMode, undefined)`, computed unconditionally), so every aggregate falls - to the SQL scan — which now refuses a `v.bigint()` field rather than coerce padded - text into the ~1.5e40 that falls out of it. Net effect: **no aggregate over a - bigint column on a soft-delete table**, even for values well inside 2^53. - -2. **Declaring `aggregateIndex` on a bigint column caps that column's writes at - 2^53.** `aggregate-tally.ts`'s `coerceAggregateNumber` refuses a magnitude past `MAX_SAFE_BIGINT`, because - the companion's `__value__` is a REAL column that cannot hold it exactly. The - refusal fires on the **write**, not the read — so one index declaration makes the - column unusable for large values, which is the opposite of the direction people - reach for `v.bigint()` in. - -Neither is a regression: before #365 the scan answer was garbage (`SUM` read 0) and -the write path threw for a different reason. Both are now correct-but-narrow, and the -combination is the gap. - -## 1. Current state (audit) - -> **Citations are by symbol, not line number.** An earlier draft of this plan -> cited `ctx-db.ts:2451`/`:2467` and `aggregate-tally.ts:62`; the plan-265 -> round-two work inserted `assertReducibleBySql` above them and moved all three -> by ~26 lines within days. Grep for the symbol. - -**Since this plan was written**, the scan-path refusal is no longer inline: it is -`assertReducibleBySql` (`ctx-db.ts`), applied at three entry points — the -aggregate reader plus **both** halves of `groupBy`, which previously had no guard -at all. That closed the "unguarded `groupBy`" hole but did **not** close either -gap below: a soft-delete table still cannot reach the companion, so it still -lands on the scan, which now refuses a bigint field loudly instead of returning -~1.5e40. - -- `packages/shard-engine/src/ctx-db.ts` — `aggScope`, computed unconditionally from `softDeleteMode` in the aggregate reader. -- `packages/shard-engine/src/ctx-db.ts` — the `definition.aggregateIndexes && !baseWhere && !hasRelation && !aggScope` fast-path predicate that `aggScope` disqualifies. -- `packages/shard-engine/src/aggregate-tally.ts` — `MAX_SAFE_BIGINT`, documented - as "the largest magnitude the companion's REAL `__value__` column holds exactly". -- `packages/shard-engine/src/aggregate-tally.ts` — `coerceAggregateNumber`'s write-side refusal, whose - message already points at the two workarounds ("aggregate a narrower column, or read - the rows and reduce them in the handler"). - -## 2. Existing seams (do not reinvent) - -- The **companion table** and its backfill (`ctx-db-companions.ts`, - `ensureBackfilled`) already maintain per-group tallies incrementally and rebuild on - first write after a format change — so a `__value__` representation change does not - need a migration plan of its own. -- **`softDeleteScope`** already expresses "live rows only" as a where-fragment; the - question is whether the companion can be keyed to respect it, not whether the - predicate exists. - -## 3. The behavioural contract to preserve - -- **An aggregate either returns the exact answer or throws.** No rounding, no - silent approximation. This is the rule #365 was built on and the reason it is - landable; a `SUM` that is quietly off by an ULP on a money column is the defect - class this plan family exists to eliminate. -- **The error must stay actionable.** The current message names both workarounds; do - not regress it to a bare type error. -- **Do not widen the write-side refusal into a read-side surprise**, or vice versa. - Whichever side rejects, it should be the one the developer can act on. - -## 4. Design decisions to make (not yet made) - -1. **Gap 1**: can the companion be maintained per soft-delete state — e.g. keyed to - include the live/deleted discriminator so `aggScope` becomes a companion lookup - rather than a disqualifier? If yes this is the better fix, because it also speeds - up every non-bigint aggregate on a soft-delete table, which is a pre-existing - performance cliff nobody has measured. If no, the fallback is to teach the scan - path to aggregate the padded key exactly (decode per row and reduce in JS), which - is correct but O(rows). -2. **Gap 2**: store the companion's `__value__` as something exact for bigints — a - decimal string tallied with `BigInt` arithmetic in JS, or a second column — rather - than REAL. Weigh against: the companion is a hot write path, and every insert - touches it. -3. Whether `defineTable` should **reject `aggregateIndex` on a bigint column** at - schema time while the cap stands. A loud failure at build time beats a loud failure - on the first large write in production, and `@lunora/advisor` is the natural home - for the lint if a hard error is too strong. - -## 5. Workstreams - -| # | Work | Size | -| --- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | -| 1 | Decide (1) and (2) above with a measurement, not an opinion: how many aggregates in `examples/` + `apps/` hit a soft-delete table, and what the companion write cost is today | S | -| 2 | Gap 1 fix per the decision | M | -| 3 | Gap 2 fix per the decision | M | -| 4 | Schema-time guard or advisor lint for `aggregateIndex` on a bigint column, if the cap survives | S | -| 5 | Tests: exact `SUM`/`MIN`/`MAX` over a bigint column above 2^53; the same on a soft-delete table; a soft-delete table's aggregate excluding deleted rows | M | - -## 6. Platform parity - -**Not applicable** — no `ctx.*` surface, binding, or deploy capability changes. The -aggregate surface and its errors are unchanged in shape; only which inputs are -accepted changes. Recorded explicitly per the repo convention. - -## 7. Phasing & ordering - -| Phase | Work | Gate | -| ----- | ------ | ----------------------------------------------------------------------------------------------------------- | -| 0 | WS 1 | A written decision with the measurement behind it | -| 1 | WS 2 | An aggregate over a bigint column on a soft-delete table returns the exact answer and excludes deleted rows | -| 2 | WS 3 | `SUM` over values above 2^53 is exact — assert against a `BigInt`-reduced expected value, not a `Number` | -| 3 | WS 4–5 | Suite green; the refusal, wherever it survives, still names a workaround | - -## 8. Risks & STOP conditions - -- **STOP if the only way to make an aggregate work is to round it.** Keeping the - throw is the correct outcome; record it and close the plan as REJECTED with the - reasoning. That is a legitimate result here. -- **Watch the companion write path.** Both fixes touch what every insert maintains. A - correctness win that halves insert throughput needs the number in the PR body — - `packages/shard-engine` has benches. -- The soft-delete companion keying is the kind of change that looks small and - reaches into backfill, invalidation and the poke protocol. If it does, split it out - rather than growing this plan. - -## 9. Open questions (answer during execution) - -- Is `avg` consistent with `sum`/`max` after these changes? #365's review noted `avg` - returning a value where `sum` threw; confirm the three agree. -- Does `groupBy` over a bigint `by` field share either gap? -- Should `aggregateIndex` on a bigint column simply be **unsupported** rather than - capped — is anyone summing one, or is the real use case counting/ordering by a - snowflake id? - -## 10. Execution record (2026-08-09) - -### WS 1 — the measurement, and what it changed - -**No first-party schema declares `.softDelete()`.** `grep -rn "\.softDelete("` -across `examples/`, `apps/`, `templates/`, `registry/` and every package `src/` -returns nothing outside tests and generated output. So gap 1 had **no in-repo -consumer**, and §4.1's secondary argument — "it also speeds up every non-bigint -aggregate on a soft-delete table, a pre-existing performance cliff nobody has -measured" — has no instance to measure either. It is still true in principle; -it is not evidence. The fix landed because it is small and the correctness half -stands on its own, not because the cliff was observed. - -### WS 2 — gap 1: the companion tallies live rows only (**done**) - -Taken as §4.1's first option, and it turned out cheaper than the plan assumed: -no companion re-keying, no backfill/invalidation/poke reach-through, no -migration. A soft delete is mechanically a patch that stamps the marker column, -so it already arrives at `syncAggregates` as an ordinary `(previous, next)` -pair. Gating **both sides** on liveness decomposes it into "remove the live row, -add nothing" — and a restore into its mirror — with no second code path: - -- `query-args.ts` — new `isLiveForCompanion(document, field)`, placed directly - beside `softDeleteScope`, whose predicate it mirrors record-side. Adjacency is - the point: a difference between the two is a maintained counter that silently - disagrees with the scan it replaces. -- `ctx-db-companions.ts` — `applyAggregateDelta` gates `removes`/`adds` through - it; the lazy `ensureBackfilledIndex` skips non-live rows; `recomputeExtreme` - ANDs the marker into its group predicate. -- `ctx-db-backfill.ts` — the explicit backfill twin takes the marker column so - both seeds agree. - -**No migration is needed**, which the plan flagged as a risk: each ctx-db -instance already rebuilds a companion from source on first touch, so the format -change heals on the next read or write. - -**The readers were then scoped back — see the review round in §11.** The first -version of this change let a soft-delete table take the companion for _every_ -aggregate. That is correct but slower, because reaching the companion means -paying `ensureBackfilledIndex`'s full rebuild, per dispatch. The shipped gate -takes the companion only when the scan cannot answer at all. - -### WS 3 — gap 2: kept loud, deliberately (**decided, not fixed**) - -The cap stands. Lifting it needs **both** an exact tally — a decimal-string -column accumulated with `BigInt` on a read-modify-write, on the path every -insert takes — **and** `AggregateResult` widened from `null | number` to carry -a bigint through every adapter, the client, and the API snapshots. That prices -the whole hot write path for "sum a column of snowflake ids", which is not a -question anyone has asked. §8's STOP condition covers this: keeping the throw -is a legitimate result. - -What shipped instead is the message. The throw fires on the **insert**, so -"cannot be aggregated exactly" read as a puzzle; it now names the -`aggregateIndex` as the thing that capped the column, and the docstring records -why the exact version was rejected so it is not re-litigated. - -### WS 4 — the advisor lint: **not filed** - -§4.3's lint would need `aggregateIndexes` in the advisor IR, which carries no -such kind today (`AdvisorIndex.kind` is `geo | index | rank | search | vector`) -— so it is a feeder change in both the runtime and codegen halves, not a lint. -Weighed against zero in-repo declarations of `aggregateIndex` on a bigint -column, and the now-explicit runtime message, it is not worth the surface. - -### The third gap — silent, and the reason to keep writing tests first - -The soft-delete max test failed on its first run with **`1e+39` where the -answer was `9`** — not from anything this plan changed. `recomputeExtreme` -answers "the row carrying the stored extreme just left, what is the new one?" -with `MIN`/`MAX(json_extract(__doc__, '$.field'))`. For a `v.bigint()` column -that reduces the **order-preserving padded key**, and SQLite coerces that text -to a REAL. So deleting the max row of a bigint `max` index wrote ~1e39 into the -companion and every later read returned it — a **wrong answer, silently**, -which is worse than either gap the plan was written for. - -The reader's `assertReducibleBySql` cannot cover it: this is the write path, and -refusing there would break `delete`. Fixed by reducing that group in JS off the -decoded documents when the field is a projected kind (`reduceExtremeInJs`), -which sees exact values and keeps the same `coerceAggregateNumber` bound as -every other contribution. Regression test on a plain table in -`ctx-db.bigint-bytes.test.ts`, since the defect never needed soft delete. - -### §9's open questions - -- **`avg` vs `sum`/`max`:** consistent. All three route through the same reader, - so all three now take the companion on a soft-delete table and all three hit - `assertReducibleBySql` on the scan. -- **`groupBy` over a bigint `by` field:** shares neither gap. Its fast path is - fixed by the same change, and both halves were already guarded. -- **Should `aggregateIndex` on a bigint be unsupported rather than capped?** No. - Money in minor units is the real use case and sits far inside 2^53; the id - case is `min`/`max`/`count`, which the cap does not reach until the id itself - exceeds 2^53. - -### Not in scope, and why - -`@lunora/sql-store` has the same soft-delete disqualifier in its own reader. -Left alone deliberately: its `.global()` tables store real columns rather than a -JSON document, so there is no projected key there and no bigint gap — only the -performance cliff, which is the half with no measured instance. - -### Gates - -`@lunora/shard-engine` 1047/1047 · `@lunora/do` 524/524 · both `lint:types` and -`lint:eslint` clean · `api:check` green after `api:update` (one added export, -`isReprojectableColumnType`, which the plan-311 follow-up below needed). - -## 11. Review round (2026-08-09) - -Two adversarial reviews over the working tree. Both found real defects; one of -them found the change had introduced one. - -### Introduced, now fixed — the companion is not free to reach - -Letting a soft-delete table take the companion for every aggregate was a -pessimisation, not just a widening. Reaching the companion calls -`ensureBackfilled`, and `ensureBackfilledIndex` is an **unconditional TRUNCATE + -full rebuild** memoised per ctx-db instance — i.e. per dispatch, and again per -subscription re-run, since codegen's `buildCtx` constructs one each time -(`_generated/shard.ts:1064`, `:1113`). So a `count()` on a 100k-row soft-delete -table would have gone from one SQL `COUNT` to decoding 100k documents in JS and -rewriting the companion, **on a read, every request** — and a read-only query -dispatch would have started performing storage writes. - -Shipped instead: the companion is taken on a soft-delete table only when the -field is projected, which is exactly when the scan refuses (`isProjectedField`, -shared with `assertReducibleBySql`). `count()` is never gated on a field, so it -keeps the scan unconditionally; `groupBy` qualifies on either half. Everything -else keeps the plan it always had, and the gap this plan exists to close is -still closed. - -This also retires §4.1's secondary argument for good. It claimed the companion -would "speed up every non-bigint aggregate on a soft-delete table". With the -rebuild cost measured, the opposite is true. - -### Widened — the recompute gate was keyed off the declared type - -Fix B in §10 used `isProjectedKind` (declared `bigint`/`bytes`). But the -projection runs on the **runtime value type** — `sqlComparableProjection` -branches on `typeof value === "bigint"` — so a `v.any()` / `v.union()` / -`v.from()` column holding a bigint is stored as the same padded key, took the -SQL branch, and wrote ~1e39 into the companion. The same silent wrong number the -fix was for, one declaration away. Now gated on `mayHoldProjectedValue`; -over-including is safe here because the branch only reads documents, unlike the -reader's guard, which refuses. Regression test asserts the companion row's -stored `__value__` directly, and fails with `1e+39` when reverted. - -### Found, NOT fixed — the reader's refusal is narrow in the same way - -Surfaced by writing the test above. `assertReducibleBySql` also keys off -`isProjectedKind`, so an `aggregate({ op: "max", field })` over a `v.any()` -column holding bigints, with no matching `aggregateIndex`, is **neither refused -nor answered**: the scan returns the 40-character padded key as the result -(`SUM` returns ~1e39). Reproducible on a plain table; pre-existing, not -introduced here. - -It is not fixed because the obvious fix is wrong. Widening the refusal to -`mayHoldProjectedValue` would refuse every aggregate over a `v.any()` / -`v.union()` / `v.from()` column, including the majority that hold plain numbers -and work correctly today. Doing it properly needs a decision — probe the stored -rows (`typeof(json_extract(…)) = 'text'`) and refuse only when a projected value -is actually present, or route those columns to the companion — and that decision -deserves its own plan rather than being made at the end of this one. - -### Deleted — the plan-311 follow-up was redundant - -The `legacyRows` count added to the `migrationStatus` admin RPC has been -reverted in full, along with its `isReprojectableColumnType` export and API -snapshot entry. The capability already existed: `buildReprojectionMigration`'s -transform returns `undefined` for a row already in the current projection, and -the runner counts `changed` only when a transform returns a document — so -`lunora migrate up __lunora_reproject__ --dry-run` already reports the -exact legacy-row count per shard. - -The reverted version was also worse than the one that existed. Both reviews -independently found that it derived affected columns from codegen's column -metadata without reproducing `reprojectableFields`' `.global()` exclusion, so -`migrate status __lunora_reproject__` would have hit `no such -table` in the DO — an error where the old behaviour returned cleanly, on a -command the docs added in the same change told operators to run. Two further -finds fell out of the same seam: a live admin subscription to that id would -re-run the unindexed COUNT on every write flush, and `__lunora_reproject__constructor` -reached a prototype member and threw. All five defects were in code that did not -need to exist. - -`concepts/migrations.mdx` documents the reserved migration and `--dry-run` -instead — the discovery gap was that the docs had never mentioned it at all. - -## 12. Review round 2 (2026-08-09) - -Both reviews re-run against the three commits. One HIGH, found independently by -the bug reviewer and by a probe written while acting on the quality review — the -same defect from two directions. - -### Introduced and fixed — the liveness gate bypassed the 2^53 storage cap - -`coerceAggregateNumber` is the only enforcement of "a bigint past -`Number.MAX_SAFE_INTEGER` cannot live under an `aggregateIndex`", and it is -reached only through the tally contribution. Gating that contribution on liveness -meant a row that is **dead on arrival** — inserted with the marker already -stamped, or patched while soft-deleted — was never coerced, so the cap silently -stopped applying. Measured on the real engine: - -``` -control (no softDeleteMode): insert 2^70 → THREW -this branch (soft-delete): insert 2^70 with deletedAt set → ACCEPTED - restore() → THREW, permanently -``` - -The escalation is worse than the accepted write. The row is inert only while the -schema still declares the marker: drop `.softDelete()` (an ordinary edit) and -every row reads as live, so the next rebuild folds the out-of-range value and -**every write path throws — including the delete that would remove the row.** -Read-alive, write-dead, with no in-band repair. - -Root cause: the cap is a **storage** constraint enforced from the **tally** -branch, where two conditions now disqualify a record (`isLiveForCompanion` and -`matchesStaticWhere`). Fixed by coercing the post-image whenever it matches the -index's static `where`, independently of liveness — restoring exactly the set the -bound applied to before. Pinned by a test that asserts both the dead-on-arrival -insert and the patch-while-dead path throw. - -### Acted on from the quality review - -- **The three reader gates were three shapes of one rule.** Collapsed onto - `scanRefusesAny(definition, fields)`, derived from the same predicate - `assertReducibleBySql` refuses on. `groupBy` now builds its SQL-field list once - and uses it for **both** the gate and the refusal assertions, so a future - SQL-reducing field cannot be added to one and forgotten in the other. `count`'s - `!countScope` is that predicate over an empty field list. -- **`reduceExtremeInJs` was a second copy of `foldAggregateTally`'s min/max arm.** - Deleted; the recompute now folds through the canonical reducer, so it cannot - disagree with either backfill's seed. -- **A reconstruction artifact.** The soft-delete write path carried a comment - block spliced mid-sentence (`// row to filter) and a`) that also claimed the - aggregate companion is passed `undefined`, which it is not. Reverted to the - pre-branch text with the aggregate case stated separately. -- The eager backfill twin's liveness filter had no coverage (the suite's fixture - seeds after migration, so its loop never ran); tested, and verified to fail - with `19` instead of `10` when the filter is removed. Two stale comments fixed, - the `sql-console` memo cut to the decision plus a pointer, and the 312/313 - README cells cut from ~3,000 characters to the outcome plus a link. - -### Spun out rather than absorbed - -**Plan 315** — the gate added here is a workaround for `ensureBackfilledIndex` -being memoised per ctx-db instance rather than durably, so **every** table with an -`aggregateIndex` rescans on the first companion touch of every dispatch. Making -that marker durable retires `scanRefusesAny`, its note, and the asymmetry between -the three readers. Pre-existing on `alpha`, found while sizing this gate; -backreferenced from the rebuild it would fix. - -### Correction to the docs - -`migrations.mdx` claimed `changed: 0` proves a table is fully re-projected. It -does not: the transform also skips a row it cannot re-project at all — a bigint -wider than the 39-digit key, reachable for a uint256-shaped value — and counts it -processed, not changed. The only signal is a `console.warn`. Qualified. - -### Cleared on re-audit - -Reader-gate symmetry is exact — no input takes the companion that previously got -an answer, and none falls to a scan that now throws where it used to answer. -Companion and scan agree within one request across every op, including empty-group -`null` vs `0`. `reduceExtremeInJs`'s replacement is strictly better than the SQL -reducer for non-projected values (SQL ranked by storage class and could return a -string). No dropped or duplicated hunk from the mid-split reconstruction: all six -raw user-table write sites still pair with `syncAggregates`. Commit ordering -holds — the first stands alone, the second would be red without it. diff --git a/plans/README.md b/plans/README.md index ca73a82c85..82f5e13ebf 100644 --- a/plans/README.md +++ b/plans/README.md @@ -1165,7 +1165,8 @@ Reference design docs gating unbuilt follow-on work: **052** streaming hook · **333** query snapshot coherence (client-only prototype built, measured, and reverted — the real fix needs a `pokeStart`/`pokeEnd`-style batch boundary for query subscriptions; see the plan for why the microtask window was the wrong -rung) · +rung) · **334** mutation determinism (global-swap implementation proven unsound +under concurrency and reverted; a sound version injects per-context) · **137** release-train rehearsal (feeds 135 Phase 3) · **162** crossTabSync relay · **234** node-host findings (spike DONE; the file was re-added by `370994075` and is kept as the host's reference) · **237** admin-auth hooks (one-adapter @@ -1201,6 +1202,11 @@ lost-wake covered by a durable send step plus the waiter's own timeout (the supersedes the run in flight without flushing the queue. Prototype + plan file deleted; record in git history. +**312** (bigint aggregate gaps) DONE & REMOVED: verified against the live code — +`isLiveForCompanion` is in `query-args.ts` and wired through +`ctx-db-companions.ts` and `ctx-db-backfill.ts`. Its two spun-out follow-ups +(**314**, **315**) keep their own files. Record in git history. + Retired since this list was last written: **235** (progressive sharding) and **248** (runtime lints), deleted by `7c3d3f42e`; **311** (the bigint backfill), which shipped inside #365 rather than trailing it; and **265** / **302**, both @@ -1592,129 +1598,13 @@ verify`_ is YAGNI — an agent can run `pnpm test` itself; the half with real value (live function-spec parity against the deployed worker) is a candidate for a later wave. -## Wave 21 — all-package sweep (baseline `70b7451b5`, 2026-08-11) - -First all-package sweep since Wave 18 (`071c6a29c`, ten days earlier). Five read-only -audit agents (core runtime, security surfaces, tooling packages, tests+architecture, -deps/DX/docs/direction), every surviving finding re-read against live code before it -reached a plan — five subagent claims did not survive that pass and are recorded under -"considered and rejected" below. - -The headline is that the two most valuable findings are both **cache-scoping** bugs of -the same family: a client-side map keyed by less than the server keys it (316), and an -in-memory baseline with no durable backing where its own sibling has one (326). Both -were invisible to every existing test because both degrade rather than fail. - -Deliberately skipped: the uncommitted `durable-stream.ts` work in -`packages/shard-engine` and the in-flight diffs in `packages/server/src` — another -session is mid-feature there. - -| Plan | Title | Category | Pkg | Pri | Effort | Risk | Status | -| ---- | ------------------------------------------------------------- | -------------- | ----------------------- | --- | ------ | ---- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 316 | Scope the client mutation watermark by identity | bug | client/db | P1 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a68bfc5859c3a2d80`, branch `advisor/316-client-watermark-identity-scope`, commit `5de113e65`, four files. Reviewer re-ran: client 650 pass + 1 expected-fail, db 87/87, `api:check` clean. The regression test does the real thing — ack 5 under identity A, switch to B → **0**, switch back to A → **5**, on a real `LunoraClient`. **The plan's WS1 as literally written was incomplete, and the mandated stash-and-prove is what caught it:** composite-keying alone breaks the plan's own contract §3.2, because a same-token _subject resolve_ relabels the fingerprint (hash → `subj:…`) and makes the cached watermark invisible under the new key — reintroducing the identical wedge on every routine subject resolution. Fixed with `restampWatermarks(from, to)` called from `setAuthToken`'s existing same-token branch, mirroring the established `restampQueuedIdentity` sibling. The executor nearly reported done with 2 of 3 tests passing before adding the third. **Better than the plan on one detail:** the composite separator is `�`, not the `�` the plan suggested — a NUL would have made the file binary to `grep`, which has bitten this repo before; `file` confirms it is still UTF-8 text. `debug()`'s projection strips the prefix and filters to the current identity, so no other identity's watermark can leak into it. §9: no LRU needed (one entry per visited identity×bucket, same shape as the file's existing unbounded-but-small maps); the durable `OutboxSink` replay path already has its own `currentIdentity()` guard — no gap — [316](316-client-watermark-identity-scope.md) | -| 317 | Route every `.dev.vars` write through the atomic 0600 writer | security | cli | P1 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a211304207fda7ea9`, branch `advisor/317-dev-vars-atomic-write`, commit `4f813642b`, three files. Reviewer verified: `writeFileSync` is **gone** from `env/handler.ts`, and `registry/apply.ts` retains exactly the two out-of-scope writes (`package.json:155`, `wrangler.jsonc:426`). Env suite 37/37 on a re-run. **The regression test is real and was proven both ways** — `it.skipIf(win32)` asserts `statSync(".dev.vars").mode & 0o777 === 0o600` after `env generate --set` creates the file fresh; reverting WS1 yields **420 (0o644)** against the expected **384 (0o600)**. Two STOP conditions were checked and cleared before starting: `writeDevVariablesFileAtomically` is exported (`config/src/index.ts:111`), `@lunora/cli` already depends on `@lunora/config` (`cli/package.json:73`), and the writer's temp file is a **sibling** (`${path}.tmp-${pid}`), so there is no cross-device `EXDEV` rename risk. **§9 answered — recommend yes:** `lunora doctor` should warn when an existing `.dev.vars` is group- or world-readable. It is one POSIX-only `statSync`, and without it a file created 0644 by a pre-fix CLI stays silently exposed forever, since this plan deliberately ships no in-tool rotation warning. Follow-up target: `runEnvDoctor` in `cli/src/commands/env/handler.ts` — [317](317-dev-vars-atomic-write.md) | -| 318 | Stop the Vite dev loop emitting from a stale TS program | bug | vite/codegen | P1 | S | LOW | TODO — [318](318-vite-stale-codegen-project.md) | -| 319 | Bound the dispatch call; classify deterministic 4xx | bug | dispatch/workflow/queue | P1 | S–M | MED | ✅ **DONE (WS1+WS3+workflow half) — APPROVED, awaiting merge. Queue half BLOCKED by its own STOP, correctly.** Worktree `.claude/worktrees/agent-aaba9f993d37eac1d`, branch `advisor/319-dispatch-timeout-and-retry-class`, commits `bf4a23863` + `970244c04`. Reviewer re-ran both suites: dispatch 27/27, workflow 81/81. The safety invariant holds — `DETERMINISTIC_DISPATCH_STATUSES = {400,403,404,422}` with `408`/`429` excluded in the constant's own comment, and the abort error is minted at **503**, deliberately outside that set, so a timeout stays retryable. **Queue half stopped for the sanctioned reason:** `dispatchQueueBatch` unconditionally rethrows the handler's error (`if (threw) { throw handlerError; }`), `deadLettered` is post-hoc metadata rather than a mechanism, and an existing test pins the rethrow as the deliberate contract — so ack-without-retry would be a redelivery-semantics change, not a bug fix. **§9 Q1 remains unconfirmed**: this implements the docblock's contract; if the maintainer prefers today's retry-everything behaviour, drop commit `970244c04` and edit the docblock — `bf4a23863` (the timeout) stands either way. **Follow-up it surfaced:** `@lunora/queue` re-exports `RunFunctionOptions` from dispatch so it picked up `timeoutMs` automatically, but `@lunora/workflow` declares its own copy in `workflow/src/types.ts`, so `timeoutMs` is invisible on a workflow step's typed `ctx.run` even though the runtime honours it — [319](319-dispatch-timeout-and-retry-class.md) | -| 320 | Cover the Stripe/Polar money-mutating methods | tests | payment | P1 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a43c3ad10c64a298f`, branch `advisor/320-stripe-polar-money-tests`, commit `d9314a192`. 16 new tests (8 per provider), 195/195 pass; reviewer re-ran coverage independently: `stripe.ts` branches 43.39% → **65.09%**, `polar.ts` 33.67% → **60.2%**, aggregate 68.8%, floor raised 60 → 68. `packages/payment/src` untouched (verified). **It found three latent defects on the money path and correctly pinned current behaviour rather than fixing it** — each needs its own plan: (1) **Stripe `updateSubscription` with an empty `items.data` sends `id: undefined`**, which Stripe treats as _creating a new subscription item_ rather than updating one — a silent billing split, not a crash, which is why the plan's "if it throws" STOP never fired (`providers/stripe.ts:406`); (2) **Polar's `refundPayment` never returns `partially_refunded`** — it has no `compareMoney` boundary check at all and always reports `refunded` (`providers/polar.ts:289-315`); (3) **Polar's `updateSubscription` silently drops `quantity`** — a quantity-only patch produces an empty `subscriptionUpdate: {}` no-op (`providers/polar.ts:338-342`). Also recommended: `resumeSubscription` coverage for Creem/Dodo (both at 0) — [320](320-stripe-polar-money-tests.md) | -| 321 | Restore the coverage floor on six packages; fix the generator | tests/dx | repo | P2 | S | LOW | 🟡 **WS1/WS2/WS4 DONE — APPROVED. WS3's guard BLOCKED on a maintainer decision.** Worktree `.claude/worktrees/agent-ab50d9ff17fca8a80`, branch `advisor/321-coverage-floor-gaps`, commits `2953ae6fe` + `81ada196f` + `edbc0b6ab`. All six packages now carry measured floors; reviewer confirmed and re-ran `agent` (338 tests, 88.52/80.34/88.82/88.52 against its 88/80/88/88 floor). Live-floor proof performed: `agent` statements → 99 fails with the threshold error, restored to 88 → passes. Template now emits `test:coverage`. **Adding `include: ["src"]` lowered two packages** — `agent` 89.29 → 88.52 statements, `angular` 81.1 → 79.8 — exactly the denominator effect the plan warned about, and the reason floors were pinned from the post-change run. `angular` is below default on all four metrics with a `// ratchet:` comment; `browser` and `search-core` are near-ceiling. **WS3's check stopped on §8's second STOP, correctly: `packages/auth-ui` has no `build`/`build:prod` by design** (`private: true`, internal, ships raw `.ts`/`.css`, no `dist`), so the required set as specified fails against a package this plan never touches. The three fixes — drop those two from the set, allowlist `auth-ui` (the file already models documented exceptions), or add stub scripts — are maintainer calls, not executor calls. **Two of my own plan's premises were wrong and are now corrected in the plan file:** the script does **not** run from `postinstall` (it is wired to `lint:project-json`), and the `grep -L "thresholds"` done-criterion is invalid for helper-routed packages, since the word lives in `tools/get-vitest-config.ts` rather than the call site — [321](321-coverage-floor-gaps.md) | -| 322 | Make the randomized test seed actually randomize | tests | repo | P2 | S→M | MED | 🟡 **WS1+WS2 DONE — APPROVED as far as it goes; WS3/WS4 still open.** Worktree `.claude/worktrees/agent-a7d4a9d9a94d95128`, branch `advisor/322-vitest-shuffle`, commit `6aa02e52b`, four lines in one file. Shuffle is **default-off** behind `LUNORA_SHUFFLE=1` — reviewer confirmed a plain run is unchanged. Both WS1 proofs hold: two flagged runs of `@lunora/values` give different file orders, and a pinned seed reproduces byte-identically. **The reproducibility check needed `--no-file-parallelism` to be meaningful at all** — with parallel files the reporter prints _completion_ order, not schedule order, so the same seed looks different every run; that belongs in the plan's method, not just this row. **Sweep: 37 of 38 clean, 0 confirmed order-coupling.** `@lunora/cli` failed twice with _different_ failure sets under _different_ seeds and then passed 90/90 on a serial replay of either seed — the signature of genuine flakiness (real subprocesses/fs), not order-coupling, so it is recorded unconfirmed. **Two caveats that bound what this proves.** (a) **My call, and it weakened the evidence:** I authorized one run per package instead of the plan's three, trading depth for breadth while the machine was busy. So WS3's gate ("WS2 complete for every package", three runs each) is **not** satisfied — read this as "no order-coupling in a single-run sweep", not a clearance to flip the default. (b) **`auth`'s PASS is meaningless**: its config never calls the helper, so `LUNORA_SHUFFLE` is a no-op there and the run was unshuffled — which is exactly what WS4 exists to fix. Also worth keeping: the executor's first sweep produced 114 instant "failures" because `timeout` is not a binary on macOS; it read a raw log before trusting the dataset and discarded all of it — [322](322-vitest-shuffle.md) | -| 323 | Run `dist:check` on pull requests | dx/ci | ci | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a7678ba2e1d9b517e`, branch `advisor/323-dist-check-pr-gate`, commit `9c81e74d6`, one file. **The gate was proven to fail before it was wired:** `dist:check` against a development build exits **1** with **180 files** flagged (`react/jsx-dev-runtime` and `jsxDEV` markers in `@lunora/react` and `@lunora/studio` output); against a clean production build it exits **0** ("1282 shipped files across 54 packages carry no development-build markers"). So the alpha.31 defect class does reach a dev build today and currently escapes every PR job. **Accepted deviation, and it is the difference between working and decorative:** the executor also added `dist-production` to `lint-required-check`'s `needs` — reviewer verified the aggregate now lists it. Without that the job would report and never block, and this repo's branch ruleset requires only the aggregate checks. WS3 was deliberately **not** done: `CLAUDE.md:42`'s claim that both gates have CI jobs became true once the job landed, so editing it would be churn. **Unverified by construction:** the job has never been observed running in CI, and no `api-surface`-vs-`dist-production` duration comparison exists — local timings were taken under wildly different load and are not comparable — [323](323-dist-check-pr-gate.md) | -| 324 | Test the shared client mutation path; floor `@lunora/client` | tests | client | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a3e993f1a89ad1ba9`, branch `advisor/324-client-mutation-path-tests`, commit `0979c777c`. Reviewer re-ran: 38 files, 657 passed + 1 expected-fail. `mutation-runner.ts` and `snapshot-precondition.ts` both at 100% branches; floor pinned at 84/75/76/84 with `src/sw/**` excluded (§9 Q1 — a real ~11%-stmt gap, excluded with its reason in the config so later sw work doesn't inherit a floor it must fight). **Both proofs were actually performed, not asserted:** the ref-count test reds when `inFlight` is replaced with a boolean, and bumping `branches` to 99 fails the run with the threshold error. **Trap for anyone re-measuring:** the printed coverage table silently omits rows for both new files — read `coverage/lcov.info` (`BRH==BRF`), not the table. §9 Q2: `src/auth/index.ts` (0% across 95 lines of client auth surface) deserves its own plan — [324](324-client-mutation-path-tests.md) | -| 325 | Test the native-signup email gate | tests/security | auth | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-ac7422e364e41726b`, branch `advisor/325-email-gate-tests`, commit `45a934f7d`, one new 208-line spec. `email-gate.ts` went **0/0/0/0 → 100%** statements/branches/functions/lines; reviewer re-ran the suite: 368/368, `packages/auth/src` untouched. **Both load-bearing proofs were performed:** widening the `typeof email === "string"` check to truthiness reds the passthrough tests, and inverting `withEmailGate`'s gate/existing-hook order reds the ordering tests — each reverted and the empty src diff re-confirmed. No defect found: the gate does run before the caller's hook, and the non-string-email passthrough is correctly guarded. **§9.1** — `onClassify` does **not** fire on rejection (`config.onClassify?.()` sits after the try/catch), matching its docblock. **§9.2** — the `422`/`429` arms of `statusString` are **unreachable today**: every `LunoraError` `assertEmailAllowed` can throw (`VALIDATION_ERROR`, `EMAIL_DOMAIN_BLOCKED`, `EMAIL_UNDELIVERABLE`) carries status 400. Left in place, reasoning recorded inline, since `EmailGateConfig` is user-facing config that could grow a code at another status — a deletion candidate for a later pass, not this one. **Reusable trap:** `vi.mock(import(...))` with a plain top-level `vi.fn()` throws a TDZ `ReferenceError` here because the SUT imports `email-guard` statically, unlike the sibling `email-guard.test.ts` whose mock target is loaded via a runtime dynamic `import()`; `vi.hoisted()` is the fix, and the test file documents it — [325](325-email-gate-tests.md) | -| 326 | Make the shape poke baseline survive hibernation | perf | do/shard-engine | P2 | M | MED | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-a411b85e08f3a9548`, branch `advisor/326-shape-poke-cursor-durability`, commits `010e681c0` + `1dc4007fa`. Reviewer re-ran: shard-engine 1121 pass + 1 todo, do 536 pass; measurement scaffolding deleted, never committed. **WS0 measured by counting `readShapeCdcPage` calls, not wall-clock, and the gate did not fire:** warm is always **1** page read; cold is **⌈retained/1000⌉+1** — 0 / 1,000 / 10,000 / 50,000 retained rows → 1 / 2 / 11 / 51 reads. Linear and unbounded, so the durable table was justified rather than assumed. The write costs ~0.04 ms/upsert, structurally identical to the `__global_shape_snapshot` write already in production. **The bigger finding is underneath it: `trimCdcChanges` has ZERO production call sites** (`shard-do.ts` calls it only from tests — grep-verified), so `__cdc_log` retention is effectively unbounded and _that_ is what makes the cold-memo cost unbounded. `CDC_RESUME_SCAN_LIMIT = 10,000` bounds a different read path. **Worth its own plan.** Safety invariant verified by reviewer: the chain degrades **downward** — cache → stored → attachment `sinceSeq` → `0` — so it can never sit above the watermark. The §8 epoch STOP was checked and correctly ruled out: the cursor never leaves the DO's own SQLite, and `bumpCdcEpoch`'s only caller is the PITR restore, which reverts the whole database atomically, so cursor and `readCdcCursor()` always move together. **Scope deviation, accepted:** it edited `packages/shard-engine/src/ctx-db-migrations.ts` (5 lines, CDC-gated) — omitted from the plan's Scope list but explicitly required by WS1's own text. ⚠️ **Merge note: that same file is modified in the working tree by the in-flight `durable-stream` work — expect a conflict** — [326](326-shape-poke-cursor-durability.md) | -| 327 | Manifest hygiene: unused dep, prod-vs-dev dep, catalogs, hono | deps | repo | P3 | S | LOW | TODO — [327](327-manifest-hygiene.md) | -| 328 | Stop recording an attacker-chosen IP in the auth audit trail | security | auth | P3 | S | LOW | TODO — [328](328-audit-log-ip-fallback.md) | -| 329 | Publish the Node host's capability matrix | docs | platform-node | P3 | S | LOW | TODO — [329](329-platform-node-docs.md) | -| 330 | Let Angular produce the preloaded query it can consume | direction | angular | P2 | S | LOW | ✅ **DONE — APPROVED, awaiting merge.** Worktree `.claude/worktrees/agent-ae58d3596d59bc8a0`, branch `advisor/330-angular-preload-query`, commit `30ea1ffe4`. `packages/angular/src/server.ts` is a pure re-export of `@lunora/client/ssr` (verified against that module's live surface, not Solid's copy), `./server` added to the exports map, `dist/server.{mjs,d.ts}` emitted, 108 tests pass, key-order gate clean, `api:check` unchanged (angular is untiered). Reviewer re-ran the criteria independently. **Deviation, accepted:** the export-parity test asserts against `@lunora/client/ssr` rather than `@lunora/solid/server` — strictly stronger (it catches Solid drifting too) and avoids a first-of-its-kind adapter-to-adapter dependency edge. **`templates/analog/` was documented, not wired** — its demo still uses a hand-rolled `LunoraService` over the vanilla client, and swapping it is a separate migration. Two follow-ups opened: an API-snapshot tier decision for `@lunora/angular` now that it publishes a second entry point, and `angular.mdx`'s "Deliberately small surface" callout, which undersells an adapter that already exports `paginatedQuery`/`presence`/`subscription`/`rateLimit`/`stream`/`agent*`/`flag*` — [330](330-angular-preload-query.md) | -| 331 | Resolve the `/worker` subpath contradiction | direction | vue/svelte | P3 | S–M | LOW | **PHASE 0 DONE — awaiting a maintainer decision before Phase 1A.** Evidence in §9, reviewer-verified against the live tree: the subpath has **zero** real importers (the only hit repo-wide is its own docblock example), and `withFrameworkWorker` itself is reached only by codegen's `buildFrameworkWorker` → `@lunora/runtime`, never through a framework `/worker` subpath — the Astro and SvelteKit templates both bypass it, leaving `templates/sveltekit`'s prose stale. `cb6df0762` shows the same defect was already caught and half-fixed once. Recommendation: **Option A** (delete both re-exports, promote `withFrameworkWorker` as the documented BYO-framework entry). Phase 1A removes a published surface, so it is not an executor's call — [331](331-worker-subpath-decision.md) | -| 332 | Spike: what would a payment conformance suite assert? | direction | payment | P3 | S | LOW | ✅ **SPIKE DONE — recommendation: `describe.each`, NOT a published TCK.** Worktree `.claude/worktrees/agent-a581fd0a0e430b202`, one 102-line PoC file; `packages/payment/src` untouched. **It corrected two premises of its own plan, both reviewer-verified.** (1) There are **5** real `PaymentAdapter` implementations, not 6 — `autumn-features.ts` has no `identifier`, no `capabilities`, no `parseWebhook` and is never registered; it is an Autumn-only facade. (2) The plan guessed `portal` was the behaviourally-assertable capability flag. **Wrong — `usageMetering` is the only one with runtime teeth**: `create-payment.ts:452` gates `track()` on it, while `portal` and `merchantOfRecord` are read by **no** runtime code anywhere in `src/` (grep-confirmed). `merchantOfRecord`'s 5/5 correlation with a `notSupported()` `capturePayment` is pure convention nothing enforces; `portal` has no negative-case fixture at all since every shipped provider sets it `true`. **5 shared assertions found** (clears the ≥5 bar, so "do nothing" is also wrong); the PoC — invalid webhook signature → `WEBHOOK_SIGNATURE_INVALID` across all 5 — **passes 5/5**, reviewer re-ran it. **Sizing:** `describe.each` S (~0.5–1 day, ~20 min/assertion measured); published subpath M (1–2 days + a permanent api-snapshot surface) for **zero** current audience — unlike `@lunora/platform`, the payment docs contain no BYO-adapter guidance and no third-party implementation is known. **Highest-value next assertions are not the capability matrix** but the `updateSubscription`/`refundPayment` behaviours plan 320 found broken on 2 of 5 providers; it independently re-verified all three defects from source. Open questions 2 and 3 answered: autumn counts once, and `not-supported.ts` is a shared throw-helper, not a sixth adapter — [332](332-payment-conformance-spike.md) | - -### Dependency notes - -- **332 depends on 320.** The shared-assertion inventory is far easier to read once all - six providers have comparable coverage, and 320's cases are half its input. -- **321 before 324's floor step and before 322.** 321 owns floor policy; 324 adds one - package's thresholds and 322 changes what a green run means. Landing them out of - order produces two conflicting edits to the same configs. -- **326 has a gating measurement (WS0).** If the cold-memo rescan turns out cheap at - realistic CDC retention, the plan's own STOP fires and the one-line - attachment-`sinceSeq` fallback is the whole fix. Do not start WS1 before WS0. -- **331 is a decision, not an implementation.** Its Phase 0 is three greps and a - maintainer call; the two outcomes touch different files. -- 316 and 324 both touch `@lunora/client` but not the same modules (`lunora-client.ts` - vs `mutation-runner.ts` + `vitest.config.ts`). They can run in parallel; 316 first if - they must be serialized. - -### Findings considered and rejected (Wave 21) - -Vetted against live code and dropped — recorded so they aren't re-audited: - -- **"Jurisdiction handling is duplicated across five packages and has already - drifted."** The duplication is real and **deliberate**, and it is guarded: - `packages/runtime/__tests__/jurisdiction-union-consistency.test.ts` pins all five - copies (`runtime`, `scheduler`, `server`, `mail`, `container`) to the canonical - `"eu" | "fedramp" | "us"` literal, with the decoupling rationale in its header - comment. The "widening union" cited as evidence of drift was `ShardJurisdiction` in - `@lunora/platform` — a different type. The residual sliver (five hand-rolled - `namespace.get(idFromName(…))` adapters that bypass `ShardDirectory`) is a real - portability observation, but it is a decoupling trade the repo made knowingly, not a - defect. -- **"The `hono@4.12.34` pin is in the wrong YAML key."** It sits under - `minimumReleaseAgeExclude`, which is exactly the key that waives the release-age - hold; `undici`, `js-yaml` and `dompurify` are there for the same reason. Only the - stale lockfile is real, and that is plan 327 WS4. -- **`applyPlanToCollections` partial-apply** (`packages/db/src/apply-plan.ts:133`). It - is an exported helper, its keyless-insert throw is already tested - (`db/__tests__/apply-plan.test.ts:88`), and the half-applied state requires calling - it outside a TanStack optimistic transaction. A one-line reorder if anyone is in the - file; not a plan. -- **"~14 peer ranges hardcode a catalogued version."** A peer range is a _supported_ - range, deliberately independent of the version we build against — `react ^19.2.7` - already admits the catalogued `19.2.8`, so nothing lies. Narrowing peers to catalog - values would shrink what consumers may install. `@lunora/auth` and `@lunora/vite` - using `catalog:` in peers is a per-case judgement, not a rule to generalise. -- **"No watch mode in 54 of 55 packages."** MED confidence, no evidence the loop is - actually painful; the repo's cross-package loop goes through `vis` anyway. -- **`pg`/`mysql2`/`postgres` `*` peer ranges in `@lunora/hyperdrive`.** Thin — the peers - are already `optional: true`, so a narrower range produces a warning, not a fix. -- **Two transitive high advisories, checked and un-upgradable from this repo:** - `ip-address` 10.2.0 reaches `mcp`/`agent` only via - `@modelcontextprotocol/sdk > express-rate-limit`, and Lunora serves MCP through - `WebStandardStreamableHTTPServerTransport` (`packages/mcp/src/serve-stateless.ts:14`), - never the Express path; `image-size` 2.0.2 reaches `storage` via - `@visulima/storage > @netlify/blobs`, and Lunora instantiates `AwsLightStorage` - (`packages/storage/src/upload-handler.ts:22`), never the Netlify provider. -- **Verified clean under direct read** (no findings): the wire codec round-trip - (`shared/wire-codec.ts`, `sql-projection.ts`, `serialize-sql.ts`, `index-key-codec.ts` - — args stay wire-tagged worker→shard and are only `encodeWire`'d on the - server-initiated path, so no double-encode); OCC/idempotency transaction boundaries - and the batch replay path; read-set/range invalidation (every uncertainty path is - conservative); the client socket lifecycle (superseded-socket guards, heartbeat and - timer cleanup); `session-do.ts`; `resolve-shard.ts`; the whole `/_lunora/admin/*` - plane (every extracted route cluster reaches `assertAdminAuthorized`, including the - clusters with no textual `isAdmin` hit, and `auth-admin-routes.ts:694` gates _before_ - the method check so route topology does not leak); identity-header smuggling on both - the fetch and WS-upgrade paths; constant-time token comparison throughout; payment - webhook verification on both schemes; x402's fail-closed pay-side policy; - `@lunora/storage` signed-URL binding with CR/LF canonical-injection guards; - `@lunora/cloudflare-access` RS256 pinning; CLI path handling (`fileSegment`, - `assertSafeItemName`, registry `from`/`to` traversal rejection); no committed secrets. -- **Not filed, noted only:** `packages/react/src/payment.tsx:6-23` hand-mirrors - `Subscription` with `provider: "polar" | "stripe"` while `payment/src/types.ts:31` - has five providers, and omits `currentPeriodStart`. Real drift, invisible to - `api:check` and `lint:types` because both sides compile in isolation — it motivates - plan 332 and is called out in that plan's out-of-scope list. Worth a two-line fix - next time anyone is in that file. Also: `packages/queue/src/metric-history.ts` writes - a `last_ts` column nothing reads (dead, not a defect), and `verifyDiscord` lacks the - replay-window check `verifySlack` has (thin — Discord signs the timestamp). - ## Filed outside a wave | Plan | Title | Category | Pkg | Pri | Effort | Risk | Status | | ---- | ------------------------------------------------------------------------- | --------------- | ------------------- | --- | ------ | ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | 306 | A pluggable queue-driver package, so a deployment picks its own backend | platform/queues | queue/platform-node | P2 | L | MED | TODO — filed with the Node host (`370994075`), no wave. Headline: **no general-purpose Node job library delivers a batch whose members settle independently** (twelve checked); every one is one-job-per-handler. `@lunora/queue` is message-shaped, so the adapters worth writing are message-**broker** adapters (SQS, Pub/Sub, Service Bus, AMQP, JetStream), not job-library wrappers. Batch assembly, the implicit-ack rule, `attempts` accounting and the dead-letter threshold must live above any adapter — that engine already exists inside `createNodeQueueHost`, and extracting it is most of the first release. See [306-pluggable-queue-drivers.md](306-pluggable-queue-drivers.md) | | 311 | Rewrite the rows `v.bigint()`/`v.bytes()` left unqueryable | data/migration | shard-engine/cli | P1 | M | MED | ✅ **DONE & REMOVED — shipped in #365** (`bb398f461`) — it ships with the fix it completes rather than trailing it, so the two land or revert together. Rows written between `ab0afaf00` (2026-08-03) and #365 hold the bigint tagged in place: they read back correctly but `json_extract` never matches, so `filter`/`withIndex`/`ORDER BY`/`SUM` miss them. #365 heals a row on any `patch`/`replace` and migrates nothing on its own, so the exposure is **the rows nobody rewrites** — for `paymentSessions` that is exactly the settled sessions never touched again. No new machinery was written: `runDataMigration` already owns batching, resumption, the cursor, claim/heartbeat and `dryRun`, so `reprojection-backfill.ts` (~175 lines) contributes only the affected-table derivation, a stored-text detector and an identity transform. The detector is `json_extract(__doc__, '$.f[0]')` — it resolves for a legacy array, `NULL` for a current row whose field is a JSON string — with **bound** paths and a sentinel derived from `encodeWire(0n)[0]` rather than hand-copied. A test pins the STOP condition: **a row whose only tagged value is nested is not matched**, because nested values stay wire-tagged under the current projection and a blob-scanning detector would rewrite every table forever while looking like it worked. Decided during execution: **not automatic on deploy** (every rewrite pokes every subscriber on the row, so an all-legacy shard is a deploy-time thundering herd on a path nobody asked for), and `countLegacyRows` is a cheap completeness check exported for `lunora doctor` but **not wired** — that needs a per-shard fan-out decision. **Follow-up CLOSED (2026-08-09) — it was already done.** `lunora doctor` is entirely local (filesystem + `wrangler.jsonc`, no deployment call), so it cannot know a shard's row counts; and nothing new was needed anyway. `buildReprojectionMigration`'s transform returns `undefined` for a row already in the current projection, and the runner counts `changed` only when a transform returns a document — so `lunora migrate up __lunora_reproject__
--dry-run` **already reports the exact legacy-row count per shard**. A `legacyRows` field was added to the `migrationStatus` admin RPC before that was noticed, and reverted in full: both thermo reviews independently found it omitted `reprojectableFields`' `.global()` exclusion, so `migrate status __lunora_reproject__` would hit `no such table` where the old behaviour returned cleanly — plus a live subscription re-running the unindexed COUNT per write flush, and `__lunora_reproject__constructor` throwing off a prototype member. Five defects in code that did not need to exist. The real gap was documentation: `concepts/migrations.mdx` had never mentioned the reserved migration, and now documents it with `--dry-run`. Plan file removed per the convention above; the record is this row plus `bb398f461` | -| 312 | Close the two gaps that leave a `v.bigint()` column unaggregatable | bug/design | shard-engine | P2 | M | MED | ✅ **DONE (2026-08-09)** — gap 1 fixed (the aggregate companion tallies LIVE rows only, so a `.softDelete()` table can reach it instead of falling to a scan that refuses a projected field); gap 2 kept loud and unchanged. Two adversarial-review rounds found three further defects, two of them silent: `recomputeExtreme` reduced the padded sort KEY in SQL and stored ~1e39 as a group's extreme; the same guard was keyed off the declared type, missing `v.any()` columns; and the liveness gate let a dead-on-arrival row bypass the 2^53 companion bound, so `restore()` then threw forever. All three fixed and pinned. **Found and deliberately NOT fixed:** `assertReducibleBySql` is narrow the same way, so an aggregate over a `v.any()` column holding bigints is neither refused nor answered — needs its own plan. Execution record and both review rounds in §10–§12 of the plan file; read that, not this cell. Spun out: [314](314-durable-companion-backfill-marker.md) | +| 312 | Close the two gaps that leave a `v.bigint()` column unaggregatable | bug/design | shard-engine | P2 | M | MED | ✅ **DONE (2026-08-09)** — gap 1 fixed (the aggregate companion tallies LIVE rows only, so a `.softDelete()` table can reach it instead of falling to a scan that refuses a projected field); gap 2 kept loud and unchanged. Two adversarial-review rounds found three further defects, two of them silent: `recomputeExtreme` reduced the padded sort KEY in SQL and stored ~1e39 as a group's extreme; the same guard was keyed off the declared type, missing `v.any()` columns; and the liveness gate let a dead-on-arrival row bypass the 2^53 companion bound, so `restore()` then threw forever. All three fixed and pinned. **Found and deliberately NOT fixed:** `assertReducibleBySql` is narrow the same way, so an aggregate over a `v.any()` column holding bigints is neither refused nor answered — needs its own plan. Execution record and both review rounds in §10–§12 of the plan file; read that, not this cell. Spun out: [315](315-durable-companion-backfill-marker.md) | | 313 | Put the long-term backup tier on object storage instead of someone's disk | data/durability | cli/storage | P2 | M | MED | ✅ **DONE & REMOVED — shipped in #375 and #389.** `lunora backup` had two tiers and its own docblock named them: `pitr` drives the platform's change log for the last 30 days **in place**, and `create`/`restore` is the portable tier for everything older — but that second one wrote NDJSON through `node:fs` to a local directory, so the durability of the tier that exists for exactly the case the 30-day window cannot cover was whatever disk the CLI ran on. #375 made object storage a destination (a `BackupDestination` seam, `restore --verify`, an admin read route so a bucket restore needs no URL signing) and #389 closed WS4: **the scheduled backup no longer deletes at all**, `lunora backup prune` is the only thing that removes a backup, and `lunora backup retention` previews what it would take. One `selectStaleBackups` feeds all three consumers. **Six review rounds**, and the pattern is the record worth keeping: every defect after the first round was in the _remediation_, and the recurring shape was a claim about a second party's behaviour never checked against the second party — a guard documented as preventing an OOM it could not prevent, an upload documented as checksum-verified above the size where verification stops, remediation text whose suggested call does not compile, and a prune whose confirmation was not binding because the POST carried no selection and the server re-selected. Nine such comments in all | | 314 | Stop shipping an English NLP library in every Worker | perf/deps | observability | P2 | S–M | MED | TODO — every Lunora Worker ships **`compromise`, an English NLP library**, because `@visulima/redact` hard-depends on it (`"compromise": "^14.15.1"`, verified in the installed package) and `observability/src/request-log.ts` imports redact for `redactArgs`, which `database-telemetry.ts` and `context-telemetry.ts` both call — so it is reachable from any app that logs. Wave 20 measured it at ~606 KiB raw, **larger than any first-party package**, against a `templates/standalone` worker of 1684.9 KiB raw. Nobody asked for NLP; they asked for their logs not to leak secrets. The measurement tooling already exists (`bundle-meta.json` carries per-module `bytesInOutput`), and the STOP condition is that redaction must not get weaker — this is a security path, so characterisation tests come first. See [314-compromise-in-every-worker.md](314-compromise-in-every-worker.md) | | 315 | Make the aggregate-companion backfill marker durable | perf | shard-engine | P2 | M | MED | TODO — spun out of 312. `ensureBackfilledIndex` is an unconditional full-table rescan + companion rewrite memoised only in a `Set` on the ctx-db instance, and codegen builds one **per dispatch** — so the mechanism whose purpose is to answer without scanning pays a full scan before it answers, once per request, for every table with an `aggregateIndex`. Pre-existing on `alpha`; found while sizing 312's reader gate. Landing it deletes that gate and its note. The risk is the trade it makes: today's design is wasteful but self-correcting, and a durable marker must invalidate on an index-definition change or the companion answers a stale grouping forever. See [315-durable-companion-backfill-marker.md](315-durable-companion-backfill-marker.md) | @@ -1723,8 +1613,8 @@ Worth reading before picking **306** up: it only pays off if a non-Cloudflare target becomes real, and `@lunora/platform-node` is still experimental with no `lunora dev --target node`. **311 and 312** both fall out of the plan-265 work (#365) — 311 is the data left behind by the defect and now ships inside #365 -itself, while 312 is the price its fix paid and is the only one still open. 312 -needs its §4 decision made with a measurement before anyone writes code. +itself, while 312 was the price its fix paid — since executed (see the DONE & +REMOVED note above); its open follow-ups live in 314 and 315. ## Notes for executors (carried from prior waves) From 43bfff51c0f8e6d6f08990106944aaa839efb907 Mon Sep 17 00:00:00 2001 From: Daniel Bannert Date: Tue, 11 Aug 2026 12:00:45 +0200 Subject: [PATCH 05/10] refactor: name two types the review called out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `StreamOptions` was written inline at both `.stream()` terminals; `QueueDatabase` was three levels of `Parameters` indexing off the builder to arrive at a type `@lunora/server` already exports as `DatabaseWriter`. Both derivations were correct and neither told the next reader what the type is. A hand-written structural stand-in for the writer was the obvious move and the wrong one — it did not match `DatabaseWriter` and broke the build, which is the argument for naming the real type rather than approximating it. Also records the ceiling the audit raised on durable streams: an attach replays a transcript, it does not re-run the handler, so the procedure's middleware chain runs once for the caller that started the run. That is why the run key is identity-scoped, and it belongs next to the option that turns sharing on. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018hGhKLpXkFj3aWUYhZmWQv --- api-snapshots/server.api.md | 8 ++------ packages/agent/src/component.ts | 14 +++++++------- packages/server/src/builder/types.ts | 12 ++++++++++-- packages/server/src/types.ts | 6 ++++++ 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/api-snapshots/server.api.md b/api-snapshots/server.api.md index b61555daa7..e082e1e03f 100644 --- a/api-snapshots/server.api.md +++ b/api-snapshots/server.api.md @@ -822,9 +822,7 @@ interface InternalQueryBuilder; ctx: Context; signal: AbortSignal; - }) => AsyncGenerator | AsyncIterable, options?: { - durable?: boolean | DurableStreamOptions; - }) => RegisteredStream; + }) => AsyncGenerator | AsyncIterable, options?: StreamOptions) => RegisteredStream; use: (middleware: Middleware) => InternalQueryBuilder; } ``` @@ -1431,9 +1429,7 @@ interface QueryBuilder args: InferArgs; ctx: Context; signal: AbortSignal; - }) => AsyncGenerator | AsyncIterable, options?: { - durable?: boolean | DurableStreamOptions; - }) => RegisteredStream; + }) => AsyncGenerator | AsyncIterable, options?: StreamOptions) => RegisteredStream; use: (middleware: Middleware) => QueryBuilder; x402: (config: X402ProcedureConfig) => QueryBuilder; } diff --git a/packages/agent/src/component.ts b/packages/agent/src/component.ts index 06729947be..a5f108d86c 100644 --- a/packages/agent/src/component.ts +++ b/packages/agent/src/component.ts @@ -1,5 +1,5 @@ import { LunoraError } from "@lunora/errors"; -import type { SchemaExtension } from "@lunora/server"; +import type { DatabaseWriter, SchemaExtension } from "@lunora/server"; import { defineSchemaExtension, defineTable, initLunora } from "@lunora/server"; import { v } from "@lunora/values"; @@ -186,12 +186,12 @@ const agentExtension: SchemaExtension = defineSchemaExtension(AGENT_EXTENSION_KE const { mutation, query } = initLunora.dataModel().create(); /** - * The minimal writer surface the queue helpers need. `agentEnsureThread` and - * `agentCompleteRun` are ordinary mutations, so this is just `ctx.db` — named - * so the helpers can be lifted out of the closures without dragging the whole - * generated data-model type along. + * The writer the queue helpers take. Named from `@lunora/server`'s exported + * `DatabaseWriter` rather than derived through three levels of `Parameters` + * indexing off the builder — the derivation was correct and told the next + * reader nothing about what the type actually is. */ -type QueueDatabase = Parameters[0]>[0]["ctx"]["db"]; +type QueueDatabase = DatabaseWriter; /** * Park a run behind the one currently in flight (`onConcurrentRun: "queue"`). @@ -241,7 +241,7 @@ const enqueueRun = async ( ); } - const position = queued.length === 0 ? 0 : Math.max(...queued.map((row) => row["position"] as number)) + 1; + const position = ((queued.at(-1)?.["position"] as number | undefined) ?? -1) + 1; await database.insert(RUN_QUEUE_TABLE, { agent: args.agent, enqueuedAt: now, instanceId, position, threadKey: args.key }); diff --git a/packages/server/src/builder/types.ts b/packages/server/src/builder/types.ts index fcdb71291c..c29f0f1776 100644 --- a/packages/server/src/builder/types.ts +++ b/packages/server/src/builder/types.ts @@ -16,6 +16,14 @@ import type { X402ProcedureConfig, } from "../types"; +/** + * Options for the `.stream()` terminal. `durable: true` is shorthand for + * `durable: {}` — the runtime only ever sees the object form. + */ +export interface StreamOptions { + durable?: boolean | DurableStreamOptions; +} + /** Builder discriminator. Codegen reads this kind. */ export type TerminalKind = FunctionKind; @@ -95,7 +103,7 @@ export interface QueryBuilder( handler: (options: { args: InferArgs; ctx: Context; signal: AbortSignal }) => AsyncGenerator | AsyncIterable, - options?: { durable?: boolean | DurableStreamOptions }, + options?: StreamOptions, ) => RegisteredStream; use: (middleware: Middleware) => QueryBuilder; @@ -213,7 +221,7 @@ export interface InternalQueryBuilder( handler: (options: { args: InferArgs; ctx: Context; signal: AbortSignal }) => AsyncGenerator | AsyncIterable, - options?: { durable?: boolean | DurableStreamOptions }, + options?: StreamOptions, ) => RegisteredStream; use: (middleware: Middleware) => InternalQueryBuilder; } diff --git a/packages/server/src/types.ts b/packages/server/src/types.ts index b3b5ae39dd..33edb5900e 100644 --- a/packages/server/src/types.ts +++ b/packages/server/src/types.ts @@ -689,6 +689,12 @@ interface RegisteredStream { * Sharing applies to a run still in flight. Once a run finishes, a later caller * asking the same question gets a fresh one — a transcript is the record of one * execution, not a cached response. + * + * **What an attach does not do:** it replays a transcript, it does not re-run the + * handler — so the procedure's middleware chain (`.use(rls(...))`, rate limits, + * any custom guard) runs once, for the caller that started the run. That is why + * the run key is identity-scoped: a guard that has already passed for one user + * must never be skipped for another. */ interface DurableStreamOptions { /** From c8b62400471dd4599a49e7ffe5b36e11183211fc Mon Sep 17 00:00:00 2001 From: Daniel Bannert Date: Tue, 11 Aug 2026 12:16:31 +0200 Subject: [PATCH 06/10] fix(codegen): declare `durable` on the emitted dispatch type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The emitted `executeStream` reads `registered.durable`, but the emitted `RegisteredLunoraFunction` — the type of the `LUNORA_FUNCTIONS` dispatch table every app generates — never declared it. So every project's `_generated/shard.ts` failed `tsc`, and the codegen suite stayed green throughout: emitted code is built as a string and asserted on as substrings, never compiled. `apps/playground` and `examples/expo` are where it surfaced, both only under the repo-wide typecheck. This is the failure mode AGENTS.md documents for this file and I walked straight into it. Verified the fix the way the guard asks: by typechecking two real apps' generated output, not the emitter's tests. Also reformats two files a later `eslint --fix` had re-broken after Prettier. `buildReferences` is now a statement body rather than a bare ternary, because the two tools disagreed about parenthesising it and each undid the other on every pass — ending the fight beats picking a winner. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018hGhKLpXkFj3aWUYhZmWQv --- packages/agent/src/reply.ts | 9 +++++++-- .../simple/expected/_generated/functions.ts | 6 ++++++ packages/codegen/src/emit.ts | 6 ++++++ packages/do/src/shard-do.ts | 18 ++++++++++-------- 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/packages/agent/src/reply.ts b/packages/agent/src/reply.ts index 5e4162116e..e40ccfab39 100644 --- a/packages/agent/src/reply.ts +++ b/packages/agent/src/reply.ts @@ -47,8 +47,13 @@ export const captureEmailReplyRef = (email: InboundEmail): EmailReplyRef | undef }; /** RFC 5322 threading: append the replied-to id to any existing `References` chain. */ -export const buildReferences = (replyRef: EmailReplyRef): string => - (replyRef.references === undefined ? replyRef.messageId : `${replyRef.references} ${replyRef.messageId}`); +export const buildReferences = (replyRef: EmailReplyRef): string => { + if (replyRef.references === undefined) { + return replyRef.messageId; + } + + return `${replyRef.references} ${replyRef.messageId}`; +}; /** Subject + body of a reply. */ export interface ReplyBody { diff --git a/packages/codegen/__tests__/fixtures/simple/expected/_generated/functions.ts b/packages/codegen/__tests__/fixtures/simple/expected/_generated/functions.ts index ff9cb8f244..6a4979659d 100644 --- a/packages/codegen/__tests__/fixtures/simple/expected/_generated/functions.ts +++ b/packages/codegen/__tests__/fixtures/simple/expected/_generated/functions.ts @@ -18,6 +18,12 @@ import type { Id } from "./dataModel.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/packages/codegen/src/emit.ts b/packages/codegen/src/emit.ts index ba50c2b148..202da12507 100644 --- a/packages/codegen/src/emit.ts +++ b/packages/codegen/src/emit.ts @@ -2481,6 +2481,12 @@ ${dataModelImport} export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a \`"stream"\` declared \`durable\`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For \`"action" | "mutation" | "query"\` the handler is awaited and its result returned. * For \`"stream"\` the handler returns an \`AsyncIterable\` synchronously and takes an diff --git a/packages/do/src/shard-do.ts b/packages/do/src/shard-do.ts index 3dc621bcd0..96b75b5a2d 100644 --- a/packages/do/src/shard-do.ts +++ b/packages/do/src/shard-do.ts @@ -744,14 +744,16 @@ const streamFrames = ( chunk: (data: unknown, seq?: number) => boolean; complete: () => boolean; fail: (failure: { code: string; message: string }) => boolean; -} => {return { - ack: () => { - ws.send(JSON.stringify({ id, type: "ack" })); - }, - chunk: (data, seq) => trySendFrame(ws, JSON.stringify(seq === undefined ? { data, id, type: "chunk" } : { data, id, seq, type: "chunk" })), - complete: () => trySendFrame(ws, JSON.stringify({ id, type: "complete" })), - fail: (failure) => trySendFrame(ws, JSON.stringify({ error: failure, id, type: "error" })), -}}; +} => { + return { + ack: () => { + ws.send(JSON.stringify({ id, type: "ack" })); + }, + chunk: (data, seq) => trySendFrame(ws, JSON.stringify(seq === undefined ? { data, id, type: "chunk" } : { data, id, seq, type: "chunk" })), + complete: () => trySendFrame(ws, JSON.stringify({ id, type: "complete" })), + fail: (failure) => trySendFrame(ws, JSON.stringify({ error: failure, id, type: "error" })), + }; +}; /** * Reserved shard name for the fallback Durable Object that hosts every From f39c47e58795416b6c8bb3d41b0fef92d0b21890 Mon Sep 17 00:00:00 2001 From: Daniel Bannert Date: Tue, 11 Aug 2026 12:42:30 +0200 Subject: [PATCH 07/10] feat(plugins): package the agent skills as a plugin (#405) Co-authored-by: Claude Opus 5 --- .agents/plugins/marketplace.json | 16 ++ .claude-plugin/marketplace.json | 17 ++ AGENTS.md | 1 + package.json | 2 +- packages/cli/skills/README.md | 3 +- packages/cli/skills/lunora/SKILL.md | 17 +- plugins/lunora/.claude-plugin/plugin.json | 17 ++ plugins/lunora/.codex-plugin/plugin.json | 20 ++ plugins/lunora/README.md | 91 +++++++ plugins/lunora/hooks/hooks.json | 15 ++ plugins/lunora/scripts/verify-turn.mjs | 225 ++++++++++++++++++ plugins/lunora/scripts/verify-turn.test.mjs | 82 +++++++ plugins/lunora/skills/lunora | 1 + plugins/lunora/skills/lunora-create-package | 1 + plugins/lunora/skills/lunora-deploy | 1 + plugins/lunora/skills/lunora-functions | 1 + plugins/lunora/skills/lunora-migration-helper | 1 + .../lunora/skills/lunora-performance-audit | 1 + plugins/lunora/skills/lunora-quickstart | 1 + plugins/lunora/skills/lunora-realtime | 1 + plugins/lunora/skills/lunora-setup-auth | 1 + plugins/lunora/skills/lunora-setup-hyperdrive | 1 + .../skills/lunora-setup-hyperdrive-global | 1 + plugins/lunora/skills/lunora-setup-mail | 1 + plugins/lunora/skills/lunora-setup-scheduler | 1 + plugins/lunora/skills/lunora-setup-storage | 1 + scripts/check-skill-mirrors.js | 74 +++++- 27 files changed, 576 insertions(+), 18 deletions(-) create mode 100644 .agents/plugins/marketplace.json create mode 100644 .claude-plugin/marketplace.json create mode 100644 plugins/lunora/.claude-plugin/plugin.json create mode 100644 plugins/lunora/.codex-plugin/plugin.json create mode 100644 plugins/lunora/README.md create mode 100644 plugins/lunora/hooks/hooks.json create mode 100644 plugins/lunora/scripts/verify-turn.mjs create mode 100644 plugins/lunora/scripts/verify-turn.test.mjs create mode 120000 plugins/lunora/skills/lunora create mode 120000 plugins/lunora/skills/lunora-create-package create mode 120000 plugins/lunora/skills/lunora-deploy create mode 120000 plugins/lunora/skills/lunora-functions create mode 120000 plugins/lunora/skills/lunora-migration-helper create mode 120000 plugins/lunora/skills/lunora-performance-audit create mode 120000 plugins/lunora/skills/lunora-quickstart create mode 120000 plugins/lunora/skills/lunora-realtime create mode 120000 plugins/lunora/skills/lunora-setup-auth create mode 120000 plugins/lunora/skills/lunora-setup-hyperdrive create mode 120000 plugins/lunora/skills/lunora-setup-hyperdrive-global create mode 120000 plugins/lunora/skills/lunora-setup-mail create mode 120000 plugins/lunora/skills/lunora-setup-scheduler create mode 120000 plugins/lunora/skills/lunora-setup-storage diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json new file mode 100644 index 0000000000..24c5fb3a12 --- /dev/null +++ b/.agents/plugins/marketplace.json @@ -0,0 +1,16 @@ +{ + "name": "lunora", + "interface": { + "displayName": "Lunora" + }, + "plugins": [ + { + "name": "lunora", + "source": { + "source": "local", + "path": "./plugins/lunora" + }, + "category": "Engineering" + } + ] +} diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000000..bdfec7f3b0 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,17 @@ +{ + "name": "lunora", + "description": "Plugins for building on Lunora — a type-safe, real-time backend on Cloudflare Workers + Durable Objects.", + "owner": { + "name": "Anolilab", + "email": "d.bannert@anolilab.de", + "url": "https://github.com/anolilab" + }, + "plugins": [ + { + "name": "lunora", + "source": "./plugins/lunora", + "category": "Engineering", + "tags": ["backend", "cloudflare", "realtime", "typescript"] + } + ] +} diff --git a/AGENTS.md b/AGENTS.md index 8c050fa2bb..bd942e04df 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,7 @@ pnpm workspaces are `apps/*`, `packages/*`, `examples/*`, `tests/*`. The rest of | `api-snapshots/` | Committed public-API snapshots, one `.api.md` per package — gated by `pnpm run api:check`. | | `shared/` | Bundler-inlined helpers, **not** a package. See [Top-level `shared/`](#top-level-shared--bundler-inlined-source-not-a-package). | | `protocol/` | Wire-protocol spec + shared fixtures. | +| `plugins/*` | The Claude Code / Codex agent plugin (`plugins/lunora`) — the `packages/cli/skills` payload + the end-of-turn `lunora verify` hook. | | `plans/` | Implementation plans handed to agents; `plans/README.md` is the index. | | `sdks/` | Non-JS client SDKs (`python`, `go`, `ruby`, `rust`, `swift`, `java`, `kotlin`) + `lunora sdk generate` targets. See `sdks/README.md`. | | `patches/` | pnpm patches applied to third-party deps. | diff --git a/package.json b/package.json index e9984ddccb..8dbfd0b18b 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "lint:types": "vis run lint:types && pnpm run lint:types:registry", "lint:types:registry": "pnpm run build:packages && tsc -p registry/tsconfig.json", "multi-semantic-release": "multi-semantic-release", - "postinstall": "node scripts/generate-package-og-images.js && node scripts/generate-labeler-config.js --skip-ci && node scripts/check-cerebro-peer-lockstep.js && node scripts/check-sibling-peer-ranges.js && node scripts/check-skill-mirrors.js && node scripts/check-registry-catalog-ranges.js && node scripts/check-agents-md-packages.js && node scripts/check-roadmap-tiers.js && node scripts/no-nul-bytes.mjs", + "postinstall": "node scripts/generate-package-og-images.js && node scripts/generate-labeler-config.js --skip-ci && node scripts/check-cerebro-peer-lockstep.js && node scripts/check-sibling-peer-ranges.js && node scripts/check-skill-mirrors.js && node scripts/check-registry-catalog-ranges.js && node scripts/check-agents-md-packages.js && node scripts/check-roadmap-tiers.js && node scripts/no-nul-bytes.mjs && node --test --test-reporter=dot plugins/lunora/scripts/verify-turn.test.mjs", "prepare": "vis hook install", "test": "vis run test --query \"project!=lunora-e2e&&project!=lunora-playground\"", "test:affected": "vis affected test --fail-fast --query \"project!=lunora-e2e&&project!=lunora-playground\"", diff --git a/packages/cli/skills/README.md b/packages/cli/skills/README.md index 40d56efd3a..9feffb5600 100644 --- a/packages/cli/skills/README.md +++ b/packages/cli/skills/README.md @@ -43,4 +43,5 @@ ln -s ../../../packages/cli/skills/ plugins/lunora/skills/ `scripts/check-skill-mirrors.js` (run on every `pnpm install`) fails if a mirror is missing, dangling, or has become a real directory — a copied directory keeps resolving while it silently drifts from the source, so it is treated as an -error rather than a warning. +error rather than a warning. Forget a hop and it prints the exact `ln -s` for +the one you missed, so `pnpm install` is the shortest path to the right command. diff --git a/packages/cli/skills/lunora/SKILL.md b/packages/cli/skills/lunora/SKILL.md index 9c59311ebc..70fa8d1482 100644 --- a/packages/cli/skills/lunora/SKILL.md +++ b/packages/cli/skills/lunora/SKILL.md @@ -69,17 +69,16 @@ anything: ```bash lunora verify # wrangler config + codegen dry-run + tsc --noEmit -lunora advisor # ~95 static + runtime lints, scored per procedure +lunora advisor # the static + runtime lint set, scored per procedure ``` -`lunora advisor` is the security/quality review: RLS coverage -(`rls-procedures`, `procedure-protections`), ownership and identity checks -(`owner-field-writes`, `normalize-id-authorization`, `identity-claim-reads`, -`fail-open-guards`), input validators, unindexed reads, non-deterministic -calls in queries, SQL interpolation, and leaked secrets. Use -`--entry #` to inspect one procedure and `--min-score` / -`--baseline` to gate CI. Only after it is clean is a by-hand pass over -`lunora-functions` worth the tokens. +`lunora advisor` is the security/quality review: RLS coverage, ownership and +identity checks, fail-open guards, input validators, unindexed reads, +non-deterministic calls in queries, SQL interpolation, and leaked secrets. It +prints each finding's own lint id and category, so read its output rather than +grepping for rule names. Use `--entry #` to inspect one procedure +and `--min-score` / `--baseline` to gate CI. Only after it is clean is a by-hand +pass over `lunora-functions` worth the tokens. If one of those clearly matches the user's goal, switch to it instead of staying in this skill. diff --git a/plugins/lunora/.claude-plugin/plugin.json b/plugins/lunora/.claude-plugin/plugin.json new file mode 100644 index 0000000000..4d1ee67e5f --- /dev/null +++ b/plugins/lunora/.claude-plugin/plugin.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", + "name": "lunora", + "displayName": "Lunora", + "version": "0.1.0", + "description": "Build type-safe, real-time Lunora backends on Cloudflare Workers + Durable Objects — the first-party agent skills plus an end-of-turn `lunora verify` gate that hands back its own compiler errors. Run `lunora mcp install` for the MCP server.", + "author": { + "name": "Anolilab", + "email": "d.bannert@anolilab.de", + "url": "https://github.com/anolilab" + }, + "homepage": "https://lunora.sh", + "repository": "https://github.com/anolilab/lunora", + "license": "FSL-1.1-Apache-2.0", + "keywords": ["lunora", "cloudflare", "workers", "durable-objects", "realtime", "backend", "typescript", "vite"], + "hooks": "./hooks/hooks.json" +} diff --git a/plugins/lunora/.codex-plugin/plugin.json b/plugins/lunora/.codex-plugin/plugin.json new file mode 100644 index 0000000000..a8d163a249 --- /dev/null +++ b/plugins/lunora/.codex-plugin/plugin.json @@ -0,0 +1,20 @@ +{ + "name": "lunora", + "version": "0.1.0", + "description": "Build type-safe, real-time Lunora backends on Cloudflare Workers + Durable Objects — the first-party agent skills. Run `lunora mcp install` to add the MCP server over your dev deployment (schema, data, logs, advisories, docs search).", + "author": { + "name": "Anolilab", + "email": "d.bannert@anolilab.de", + "url": "https://github.com/anolilab" + }, + "repository": "https://github.com/anolilab/lunora", + "license": "FSL-1.1-Apache-2.0", + "keywords": ["lunora", "cloudflare", "workers", "durable-objects", "realtime", "backend", "typescript", "vite"], + "interface": { + "displayName": "Lunora", + "shortDescription": "Type-safe real-time backends on Cloudflare.", + "developerName": "Anolilab", + "category": "Developer Tools" + }, + "skills": "./skills/" +} diff --git a/plugins/lunora/README.md b/plugins/lunora/README.md new file mode 100644 index 0000000000..bc3b6de05a --- /dev/null +++ b/plugins/lunora/README.md @@ -0,0 +1,91 @@ +# Lunora agent plugin + +The first-party Lunora plugin for Claude Code and Codex. It ships two things: + +| Component | What it is | +| ---------- | --------------------------------------------------------------------------------------------------------------- | +| **Skills** | The 14 first-party skills from `packages/cli/skills/`, symlinked in — `lunora` routes, the rest do the work. | +| **Hook** | A `Stop` gate that re-runs `lunora verify` at turn end and blocks with the compiler's own errors when it fails. | + +## Install + +**Claude Code** + +```bash +/plugin marketplace add anolilab/lunora +/plugin install lunora@lunora +``` + +**Codex** — search for `lunora` in the plugin directory, or add this repo as a +marketplace (`.agents/plugins/marketplace.json`). + +**Neither** — `lunora rules install` copies the same skills into the project's +`.agents/skills/`, which Cursor, Copilot and Codex all read. That path gives you +the skills without the hook. + +## MCP is a separate install, on purpose + +The plugin declares **no** MCP server. `lunora mcp install` already owns that +job, and it makes two decisions a plugin manifest cannot express: it spawns the +_project's_ binary through the project's package manager (a plugin-level `npx` +would fetch whatever is newest on npm and point a mismatched CLI at this +project's `.lunora/dev.json`), and it deliberately refuses to write a +machine-wide `lunora` entry, because the stdio spec carries no `cwd` and one +global entry would serve whichever directory the editor happened to start in. +See the comments on `McpServerPlan` in `packages/cli/src/commands/mcp/install.ts`. + +```bash +lunora mcp install # every MCP client configured on this machine +lunora mcp install --print # show what it would write +``` + +## The `Stop` hook + +`scripts/verify-turn.mjs` is the only new behaviour here. The skills already +say "run `lunora verify` before you call it done"; agents skip it, and the type +error surfaces on the user's next `lunora dev` instead of in the turn that +caused it. The hook makes the rule a mechanism: on turn end it runs the same +command and, on failure, blocks the stop with the output, so the next turn +starts from the real errors. + +It stays quiet unless it has something to say: + +- No `lunora/` directory **next to a wrangler config** → not a Lunora project, + exit. The walk up stops at the enclosing `.git`, so a sibling checkout named + `lunora` can't capture an unrelated repo. +- `git status --porcelain -- lunora` clean → the turn didn't touch the backend, + so skip without paying for a `tsc` run. Non-git projects always verify. +- `stop_hook_active` → Claude Code is already continuing because this hook + blocked. It also caps a hook at 8 consecutive blocks, which is why there is no + counter here to get wrong. +- No `lunora` in the project's (or a parent's) `node_modules/.bin` → an + environment fact, not a type error. + +A verify that times out or cannot be spawned returns a `systemMessage` rather +than a silent `{}`, so "could not check" never reads as "checked, and it +passed". + +```bash +node --test plugins/lunora/scripts/verify-turn.test.mjs # also runs in postinstall +``` + +## Adding a skill + +Skills are **not** authored here. `packages/cli/skills//` is the source of +truth; this directory holds symlinks, as do `.agents/skills/` and +`.claude/skills/`. See `packages/cli/skills/README.md` for the hops — or just +run `pnpm install`, and `scripts/check-skill-mirrors.js` prints the exact +`ln -s` command for whichever mirror is missing. + +Claude Code dereferences symlinks that resolve inside the marketplace when it +copies a plugin into its cache, so installed users get file contents, not links. + +## Not included, and why + +- **A reviewer skill.** `lunora advisor` already runs the static and runtime + lint set — RLS coverage, ownership checks, validators, index usage, secrets — + and scores them per procedure. A markdown checklist restating a working linter + is a checklist that drifts from it. The `lunora` router skill points at the + command instead. +- **A runtime-error monitor.** Plugin monitors need something to tail, and + `lunora dev` writes no error log. Add one when it does. diff --git a/plugins/lunora/hooks/hooks.json b/plugins/lunora/hooks/hooks.json new file mode 100644 index 0000000000..315f00d109 --- /dev/null +++ b/plugins/lunora/hooks/hooks.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/verify-turn.mjs\"", + "timeout": 180 + } + ] + } + ] + } +} diff --git a/plugins/lunora/scripts/verify-turn.mjs b/plugins/lunora/scripts/verify-turn.mjs new file mode 100644 index 0000000000..5d9c5f2aef --- /dev/null +++ b/plugins/lunora/scripts/verify-turn.mjs @@ -0,0 +1,225 @@ +#!/usr/bin/env node +/** + * `Stop` hook — turn the "run `lunora verify` before you call it done" rule into + * a mechanism instead of an instruction. + * + * The skills already say to verify. Agents skip it, and the failure mode is + * quiet: the turn ends, the user reads a confident summary, and the type error + * surfaces on their next `lunora dev`. This hook re-runs the check at turn end + * and, when it fails, blocks the stop with the compiler's own output so the + * next turn starts from the real errors rather than from a re-description. + * + * `lunora verify` is the whole gate in one command: wrangler config validation, + * a codegen dry-run (so `lunora/_generated/` staleness is caught, not inherited) + * and `tsc --noEmit`. + * + * Four guards keep it from becoming a nag: + * + * 1. **Not a Lunora project** — a `lunora/` directory NEXT TO a wrangler config, + * the same pair `lunora mcp install` treats as the marker. The directory + * alone is not enough: this repo alone has a `packages/lunora`, a `lunora` + * under four of the SDKs and one under every template, and a developer with + * a `lunora` checkout beside their other projects would otherwise turn every + * unrelated repo into a Lunora one. + * 2. **Nothing changed under `lunora/`** — the working tree is clean there, so + * whatever the turn did, it didn't touch the backend. Skipped without paying + * for a `tsc` run. Non-git projects verify unconditionally. + * 3. **`stop_hook_active`** — Claude Code sets this once it is already + * continuing because of a stop hook, which is the documented way to avoid + * blocking forever. It also caps a hook at 8 consecutive blocks on its own, + * so there is no counter to keep here. + * 4. **No project-local `lunora`** — an environment fact, not a type error. + * + * Contract: stdin is the hook event JSON, stdout is the hook result JSON. + * Anything unexpected exits 0 with `{}` — a broken hook must never wedge a + * session. + */ +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { pathToFileURL } from "node:url"; + +/** How much of the verify output to hand back. Enough for a real error list, not the whole log. */ +const MAX_REASON_CHARS = 2000; + +/** Directories to walk up while looking for the project root. */ +const MAX_WALK_UP = 8; + +/** Candidate wrangler config filenames, mirroring `WRANGLER_FILES` in `@lunora/config`. */ +const WRANGLER_FILES = ["wrangler.jsonc", "wrangler.json"]; + +/** A `lunora/` directory next to a wrangler config — the marker `lunora mcp install` uses. */ +const isLunoraProject = (directory, exists) => exists(join(directory, "lunora")) && WRANGLER_FILES.some((name) => exists(join(directory, name))); + +/** + * Nearest ancestor of `startDirectory` (inclusive) that looks like a Lunora + * project — the root `lunora verify` needs to run from. `undefined` when there + * is none, which is the "not a Lunora project" exit. + * + * The walk stops at a `.git` directory. Without that boundary it escapes the + * repository entirely and can select a sibling checkout's parent as "the + * project root". + */ +const findProjectRoot = (startDirectory, exists = existsSync) => { + let directory = startDirectory; + + for (let index = 0; index < MAX_WALK_UP; index += 1) { + if (isLunoraProject(directory, exists)) { + return directory; + } + + const parent = dirname(directory); + + if (parent === directory || exists(join(directory, ".git"))) { + return undefined; + } + + directory = parent; + } + + return undefined; +}; + +/** + * Whether the turn plausibly touched the backend. + * + * `git status --porcelain -- lunora` empty means the backend is committed as-is, + * so a `tsc` run would only re-prove what the pre-commit gate already proved. + * A git failure (no repo, no git binary) returns `true`: verifying is the safe + * answer when we cannot tell. + */ +const backendIsDirty = (root, run = spawnSync) => { + const result = run("git", ["status", "--porcelain", "--", "lunora"], { cwd: root, encoding: "utf8" }); + + if (result.error !== undefined || result.status !== 0) { + return true; + } + + return result.stdout.trim().length > 0; +}; + +/** + * The project's own `lunora` binary, walking up for a hoisted `node_modules` in + * a monorepo. `undefined` means `@lunora/cli` isn't a dependency here. + * + * Deliberately not `npx`: `npx -y @lunora/cli` resolves the `latest` dist-tag, + * which for this package is an empty `0.0.0` placeholder with no `bin` at all, + * and `npx --no-install lunora` still round-trips to the registry for a package + * name that does not exist there. Both fail on every turn end. + */ +const findLunoraBin = (root, exists = existsSync) => { + let directory = root; + + for (let index = 0; index < MAX_WALK_UP; index += 1) { + for (const name of ["lunora", "lunora.cmd"]) { + const binary = join(directory, "node_modules", ".bin", name); + + if (exists(binary)) { + return binary; + } + } + + const parent = dirname(directory); + + if (parent === directory) { + return undefined; + } + + directory = parent; + } + + return undefined; +}; + +/** + * `shell: true` for the Windows `.cmd` shim only. Node refuses to spawn a + * `.cmd`/`.bat` without a shell (the CVE-2024-27980 hardening), so the Windows + * branch of {@link findLunoraBin} would otherwise resolve a binary the hook can + * never execute — a gate that silently never runs. `binary` is a path we built + * ourselves from `node_modules/.bin`, not anything from the event payload. + */ +const runVerify = (binary, root) => spawnSync(binary, ["verify"], { cwd: root, encoding: "utf8", shell: binary.endsWith(".cmd"), timeout: 170_000 }); + +/** + * Turn a verify result into a hook decision. Split out from the I/O so the + * block and pass branches are testable without spawning anything. + * + * The reason takes the TAIL of the output: `lunora verify` streams wrangler and + * codegen progress first and prints its `verify: errors:` summary last, so the + * head is the noise and the tail is the answer. + */ +const decide = ({ output, status }) => { + if (status === 0) { + return { block: false }; + } + + return { + block: true, + reason: + "`lunora verify` failed after that turn (wrangler config + codegen dry-run + `tsc --noEmit`). " + + "Fix the errors below, re-run `lunora verify` until it is clean, then finish. " + + "If an error pre-dates this turn and is out of scope, say so to the user rather than silently leaving it:\n\n" + + output.trim().slice(-MAX_REASON_CHARS), + }; +}; + +const readStdin = async () => { + const chunks = []; + + for await (const chunk of process.stdin) { + chunks.push(chunk); + } + + return Buffer.concat(chunks).toString("utf8"); +}; + +const main = async () => { + let event = {}; + + try { + event = JSON.parse(await readStdin()); + } catch { + return {}; + } + + // Plan mode writes nothing, and `stop_hook_active` means we already blocked + // once and Claude Code is continuing because of it. + if (event.permission_mode === "plan" || event.stop_hook_active === true) { + return {}; + } + + const root = findProjectRoot(event.cwd ?? process.cwd()); + + if (root === undefined || !backendIsDirty(root)) { + return {}; + } + + const binary = findLunoraBin(root); + + if (binary === undefined) { + return {}; + } + + const result = runVerify(binary, root); + + // A timed-out or un-spawnable verify proves nothing either way. Say so + // instead of returning a bare `{}`, which reads as "checked, and it passed". + if (result.error !== undefined) { + return { systemMessage: `lunora verify could not run (${result.error.message}) — the backend was NOT type-checked this turn.` }; + } + + const decision = decide({ output: `${result.stdout ?? ""}${result.stderr ?? ""}`, status: result.status }); + + return decision.block ? { decision: "block", reason: decision.reason } : {}; +}; + +// Only when executed directly — the test file imports the helpers above. +if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + process.stdout.write(`${JSON.stringify(await main())}\n`); + } catch { + process.stdout.write("{}\n"); + } +} + +export { backendIsDirty, decide, findLunoraBin, findProjectRoot }; diff --git a/plugins/lunora/scripts/verify-turn.test.mjs b/plugins/lunora/scripts/verify-turn.test.mjs new file mode 100644 index 0000000000..4c1f087ef1 --- /dev/null +++ b/plugins/lunora/scripts/verify-turn.test.mjs @@ -0,0 +1,82 @@ +/** + * `node --test plugins/lunora/scripts/verify-turn.test.mjs` — the branches of + * the Stop-hook gate that decide whether a turn gets blocked. Everything here + * is pure; the filesystem and spawn calls are injected. + */ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { backendIsDirty, decide, findLunoraBin, findProjectRoot } from "./verify-turn.mjs"; + +/** An `existsSync` double over a fixed set of paths. */ +const fs = (...paths) => { + const set = new Set(paths); + + return (path) => set.has(path); +}; + +test("findProjectRoot needs a lunora/ directory AND a wrangler config", () => { + assert.equal(findProjectRoot("/repo", fs("/repo/lunora", "/repo/wrangler.jsonc")), "/repo"); + assert.equal(findProjectRoot("/repo", fs("/repo/lunora", "/repo/wrangler.json")), "/repo"); + // A bare `lunora/` is not a project — packages/lunora, sdks/*/lunora, a + // sibling checkout named `lunora`. + assert.equal(findProjectRoot("/repo", fs("/repo/lunora")), undefined); +}); + +test("findProjectRoot walks up from a subdirectory", () => { + assert.equal(findProjectRoot("/repo/src/features", fs("/repo/lunora", "/repo/wrangler.jsonc")), "/repo"); +}); + +test("findProjectRoot stops at the repository boundary", () => { + // A sibling checkout at /work/lunora + /work/wrangler.jsonc must not be + // selected from inside an unrelated repo at /work/other. + const layout = fs("/work/other/.git", "/work/lunora", "/work/wrangler.jsonc"); + + assert.equal(findProjectRoot("/work/other/src", layout), undefined); +}); + +test("backendIsDirty verifies when git cannot answer", () => { + assert.equal( + backendIsDirty("/repo", () => ({ error: new Error("no git"), status: null, stdout: "" })), + true, + ); + assert.equal( + backendIsDirty("/repo", () => ({ status: 128, stdout: "" })), + true, + ); +}); + +test("backendIsDirty skips a clean backend and runs on a dirty one", () => { + assert.equal( + backendIsDirty("/repo", () => ({ status: 0, stdout: "\n" })), + false, + ); + assert.equal( + backendIsDirty("/repo", () => ({ status: 0, stdout: " M lunora/schema.ts\n" })), + true, + ); +}); + +test("a clean verify does not block", () => { + assert.deepEqual(decide({ output: "verify: project is valid", status: 0 }), { block: false }); +}); + +test("a failed verify blocks and hands back the TAIL of the output", () => { + // `lunora verify` streams progress first and prints its error summary last, + // so a head-slice would drop the part that names the failing step. + const output = `${"codegen progress noise\n".repeat(500)}lunora/messages.ts(12,5): error TS2345: nope\nverify: errors:\n - type errors: tsc --noEmit exited 2`; + const decision = decide({ output, status: 1 }); + + assert.equal(decision.block, true); + assert.match(decision.reason, /verify: errors:/u); + assert.match(decision.reason, /error TS2345: nope/u); +}); + +test("findLunoraBin walks up to a hoisted node_modules and gives up when there is none", () => { + assert.equal(findLunoraBin("/repo/apps/web", fs("/repo/node_modules/.bin/lunora")), "/repo/node_modules/.bin/lunora"); + assert.equal(findLunoraBin("/repo/apps/web", fs("/repo/node_modules/.bin/lunora.cmd")), "/repo/node_modules/.bin/lunora.cmd"); + assert.equal( + findLunoraBin("/repo/apps/web", () => false), + undefined, + ); +}); diff --git a/plugins/lunora/skills/lunora b/plugins/lunora/skills/lunora new file mode 120000 index 0000000000..7f830ec8bd --- /dev/null +++ b/plugins/lunora/skills/lunora @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora \ No newline at end of file diff --git a/plugins/lunora/skills/lunora-create-package b/plugins/lunora/skills/lunora-create-package new file mode 120000 index 0000000000..763295638f --- /dev/null +++ b/plugins/lunora/skills/lunora-create-package @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora-create-package \ No newline at end of file diff --git a/plugins/lunora/skills/lunora-deploy b/plugins/lunora/skills/lunora-deploy new file mode 120000 index 0000000000..c87ce8dcd7 --- /dev/null +++ b/plugins/lunora/skills/lunora-deploy @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora-deploy \ No newline at end of file diff --git a/plugins/lunora/skills/lunora-functions b/plugins/lunora/skills/lunora-functions new file mode 120000 index 0000000000..72f5475d21 --- /dev/null +++ b/plugins/lunora/skills/lunora-functions @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora-functions \ No newline at end of file diff --git a/plugins/lunora/skills/lunora-migration-helper b/plugins/lunora/skills/lunora-migration-helper new file mode 120000 index 0000000000..9e8d67898e --- /dev/null +++ b/plugins/lunora/skills/lunora-migration-helper @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora-migration-helper \ No newline at end of file diff --git a/plugins/lunora/skills/lunora-performance-audit b/plugins/lunora/skills/lunora-performance-audit new file mode 120000 index 0000000000..f3df0958d4 --- /dev/null +++ b/plugins/lunora/skills/lunora-performance-audit @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora-performance-audit \ No newline at end of file diff --git a/plugins/lunora/skills/lunora-quickstart b/plugins/lunora/skills/lunora-quickstart new file mode 120000 index 0000000000..2d4417f451 --- /dev/null +++ b/plugins/lunora/skills/lunora-quickstart @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora-quickstart \ No newline at end of file diff --git a/plugins/lunora/skills/lunora-realtime b/plugins/lunora/skills/lunora-realtime new file mode 120000 index 0000000000..d9d4ce5bd8 --- /dev/null +++ b/plugins/lunora/skills/lunora-realtime @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora-realtime \ No newline at end of file diff --git a/plugins/lunora/skills/lunora-setup-auth b/plugins/lunora/skills/lunora-setup-auth new file mode 120000 index 0000000000..339309381c --- /dev/null +++ b/plugins/lunora/skills/lunora-setup-auth @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora-setup-auth \ No newline at end of file diff --git a/plugins/lunora/skills/lunora-setup-hyperdrive b/plugins/lunora/skills/lunora-setup-hyperdrive new file mode 120000 index 0000000000..696436ea19 --- /dev/null +++ b/plugins/lunora/skills/lunora-setup-hyperdrive @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora-setup-hyperdrive \ No newline at end of file diff --git a/plugins/lunora/skills/lunora-setup-hyperdrive-global b/plugins/lunora/skills/lunora-setup-hyperdrive-global new file mode 120000 index 0000000000..5ba9875043 --- /dev/null +++ b/plugins/lunora/skills/lunora-setup-hyperdrive-global @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora-setup-hyperdrive-global \ No newline at end of file diff --git a/plugins/lunora/skills/lunora-setup-mail b/plugins/lunora/skills/lunora-setup-mail new file mode 120000 index 0000000000..0accdcb0b2 --- /dev/null +++ b/plugins/lunora/skills/lunora-setup-mail @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora-setup-mail \ No newline at end of file diff --git a/plugins/lunora/skills/lunora-setup-scheduler b/plugins/lunora/skills/lunora-setup-scheduler new file mode 120000 index 0000000000..bc4a3ced2b --- /dev/null +++ b/plugins/lunora/skills/lunora-setup-scheduler @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora-setup-scheduler \ No newline at end of file diff --git a/plugins/lunora/skills/lunora-setup-storage b/plugins/lunora/skills/lunora-setup-storage new file mode 120000 index 0000000000..7d575c0a1f --- /dev/null +++ b/plugins/lunora/skills/lunora-setup-storage @@ -0,0 +1 @@ +../../../packages/cli/skills/lunora-setup-storage \ No newline at end of file diff --git a/scripts/check-skill-mirrors.js b/scripts/check-skill-mirrors.js index fffc0a676a..31640f5411 100644 --- a/scripts/check-skill-mirrors.js +++ b/scripts/check-skill-mirrors.js @@ -2,12 +2,14 @@ * Guards the first-party Agent Skill mirrors against silent rot. * * The source of truth is `packages/cli/skills/` (shipped in the `@lunora/cli` - * tarball via the `files` allowlist). It is mirrored twice so agents working - * inside this repo pick the skills up directly: + * tarball via the `files` allowlist). It is mirrored three times — twice so + * agents working inside this repo pick the skills up directly, and once as the + * payload of the Claude Code / Codex plugin: * * packages/cli/skills/ <- .agents/skills/ <- .claude/skills/ + * <- plugins/lunora/skills/ * - * Both hops are symlinks, and both are tracked in git — which is exactly why + * Every hop is a symlink, and every one is tracked in git — which is exactly why * they rot invisibly. The `cirrus` -> `lunora` rename renamed the link NAMES * but not their TARGETS, leaving 11 of 14 skills dangling in both mirrors for * over a month: `git status` stays clean, nothing typechecks a symlink, and the @@ -32,7 +34,7 @@ * Run on every `pnpm install` via the root `postinstall` script. */ -import { existsSync, lstatSync, readdirSync, readlinkSync } from "node:fs"; +import { existsSync, lstatSync, readdirSync, readFileSync, readlinkSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -43,7 +45,7 @@ const rootDir = join(__dirname, ".."); const sourceDir = join(rootDir, "packages", "cli", "skills"); /** - * The mirror chain, in resolution order. Each entry links to the PREVIOUS + * The mirror chain, in resolution order. The in-repo hops link to the PREVIOUS * layer, not directly to the source: `.claude/skills` -> `.agents/skills` -> * `packages/cli/skills`. Keeping the hops relative means a worktree checkout * resolves without any absolute-path rewriting. @@ -56,6 +58,15 @@ const MIRRORS = [ staleWhenDangling: false, targetPrefix: "../../packages/cli/skills/", }, + { + // The Claude Code / Codex plugin's payload. This one links straight to + // the source rather than through `.agents`: Claude Code dereferences a + // symlink when it copies the plugin into its cache, and a link through a + // second link is one more hop to get wrong for zero benefit. + dir: join(rootDir, "plugins", "lunora", "skills"), + staleWhenDangling: false, + targetPrefix: "../../../packages/cli/skills/", + }, { dir: join(rootDir, ".claude", "skills"), // .agents/skills also holds ~45 third-party skills (accessibility, @@ -149,9 +160,58 @@ function isSymlink(path) { } } +/** + * The plugin ships one manifest per host (`.claude-plugin/plugin.json` for + * Claude Code, `.codex-plugin/plugin.json` for Codex) and neither host reads the + * other's. Their shared identity fields are therefore hand-copied, and nothing + * else in the repo would notice one being bumped without the other — the + * marketplace would then advertise two different versions of the same plugin. + * `description` is deliberately excluded: the two hosts describe different + * feature sets. + */ +const LOCKSTEP_FIELDS = ["name", "version", "license", "repository", "keywords"]; + +const checkPluginManifestLockstep = () => { + const paths = { + claude: join(rootDir, "plugins", "lunora", ".claude-plugin", "plugin.json"), + codex: join(rootDir, "plugins", "lunora", ".codex-plugin", "plugin.json"), + }; + + const manifests = {}; + + for (const [host, path] of Object.entries(paths)) { + try { + manifests[host] = JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + fail(`${path.slice(rootDir.length + 1)} is missing or not valid JSON (${error.message})`); + + return; + } + } + + for (const field of LOCKSTEP_FIELDS) { + const claude = JSON.stringify(manifests.claude[field]); + const codex = JSON.stringify(manifests.codex[field]); + + if (claude !== codex) { + fail(`plugins/lunora: "${field}" differs between the Claude (${claude}) and Codex (${codex}) manifests — they describe one plugin`); + } + } + + if (JSON.stringify(manifests.claude.author) !== JSON.stringify(manifests.codex.author)) { + fail(`plugins/lunora: "author" differs between the Claude and Codex manifests`); + } +}; + +checkPluginManifestLockstep(); + +const mirrorList = MIRRORS.map(({ dir }) => dir.slice(rootDir.length + 1)).join(", "); + if (hasFailure) { - console.error("\ncheck-skill-mirrors: the .agents/.claude skill mirrors are out of sync with packages/cli/skills.\n"); + console.error( + `\ncheck-skill-mirrors: fix the problems above — the mirrors (${mirrorList}) and the plugin manifests must agree with packages/cli/skills.\n`, + ); process.exit(1); } -console.log(`check-skill-mirrors: ${skills.length} skills mirrored correctly into .agents/skills and .claude/skills`); +console.log(`check-skill-mirrors: ${skills.length} skills mirrored correctly into ${mirrorList}; plugin manifests in lockstep`); From d379256c5ec8b31457223bf989ae6bf98ca2c203 Mon Sep 17 00:00:00 2001 From: Daniel Bannert Date: Tue, 11 Aug 2026 12:30:38 +0200 Subject: [PATCH 08/10] chore(examples): regenerate _generated dispatch types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propagates the `durable` field added to the emitted dispatch type in c8b624004 out to the 13 committed example apps. Their `_generated/` had been left at the previous emitter output, so `RegisteredLunoraFunction` lacked `durable?: { ttlMs?: number }` and `executeStream` still returned the narrower shape. Nothing would have caught this. The vis tasks regenerate an app's generated dir via their `codegen` dependsOn before linting, so CI builds against fresh output while the stale files stay committed, and no gate diffs the two. The examples are read as reference code, so the staleness is user-visible even though every check is green. `apps/playground` is not included — its `lunora/_generated/` is gitignored. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01U7xe5PYB3zzmRrRod9eMTn --- examples/auth-playground/lunora/_generated/functions.ts | 6 ++++++ examples/auth-playground/lunora/_generated/shard.ts | 3 ++- examples/blog/lunora/_generated/functions.ts | 6 ++++++ examples/blog/lunora/_generated/shard.ts | 3 ++- examples/chess/lunora/_generated/functions.ts | 6 ++++++ examples/chess/lunora/_generated/shard.ts | 3 ++- examples/expo/lunora/_generated/functions.ts | 6 ++++++ examples/expo/lunora/_generated/shard.ts | 3 ++- examples/feedback-board/lunora/_generated/functions.ts | 6 ++++++ examples/feedback-board/lunora/_generated/shard.ts | 3 ++- examples/kanban-board/lunora/_generated/functions.ts | 6 ++++++ examples/kanban-board/lunora/_generated/shard.ts | 3 ++- examples/notify-demo/lunora/_generated/functions.ts | 6 ++++++ examples/notify-demo/lunora/_generated/shard.ts | 3 ++- examples/offline-rejections/lunora/_generated/functions.ts | 6 ++++++ examples/offline-rejections/lunora/_generated/shard.ts | 3 ++- examples/payment-demo/lunora/_generated/functions.ts | 6 ++++++ examples/payment-demo/lunora/_generated/shard.ts | 3 ++- examples/realtime-cursors/lunora/_generated/functions.ts | 6 ++++++ examples/realtime-cursors/lunora/_generated/shard.ts | 3 ++- examples/tanstack-start/lunora/_generated/functions.ts | 6 ++++++ examples/tanstack-start/lunora/_generated/shard.ts | 3 ++- examples/team-chat/lunora/_generated/functions.ts | 6 ++++++ examples/team-chat/lunora/_generated/shard.ts | 3 ++- examples/todo-app/lunora/_generated/functions.ts | 6 ++++++ examples/todo-app/lunora/_generated/shard.ts | 3 ++- 26 files changed, 104 insertions(+), 13 deletions(-) diff --git a/examples/auth-playground/lunora/_generated/functions.ts b/examples/auth-playground/lunora/_generated/functions.ts index 01d3cfc1f4..eee0e494e3 100644 --- a/examples/auth-playground/lunora/_generated/functions.ts +++ b/examples/auth-playground/lunora/_generated/functions.ts @@ -17,6 +17,12 @@ import type { Id } from "./dataModel.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/examples/auth-playground/lunora/_generated/shard.ts b/examples/auth-playground/lunora/_generated/shard.ts index dedaa6056a..c286f55262 100644 --- a/examples/auth-playground/lunora/_generated/shard.ts +++ b/examples/auth-playground/lunora/_generated/shard.ts @@ -459,7 +459,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -469,6 +469,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/examples/blog/lunora/_generated/functions.ts b/examples/blog/lunora/_generated/functions.ts index 0a9b97401f..7c7572561d 100644 --- a/examples/blog/lunora/_generated/functions.ts +++ b/examples/blog/lunora/_generated/functions.ts @@ -19,6 +19,12 @@ import type { Id } from "./dataModel.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/examples/blog/lunora/_generated/shard.ts b/examples/blog/lunora/_generated/shard.ts index 0c7e02d6e2..2281f774c7 100644 --- a/examples/blog/lunora/_generated/shard.ts +++ b/examples/blog/lunora/_generated/shard.ts @@ -1118,7 +1118,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -1128,6 +1128,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/examples/chess/lunora/_generated/functions.ts b/examples/chess/lunora/_generated/functions.ts index 5d2573d72b..75c2c1baa9 100644 --- a/examples/chess/lunora/_generated/functions.ts +++ b/examples/chess/lunora/_generated/functions.ts @@ -19,6 +19,12 @@ import type { Id } from "./dataModel.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/examples/chess/lunora/_generated/shard.ts b/examples/chess/lunora/_generated/shard.ts index 7f913db773..1dcae14299 100644 --- a/examples/chess/lunora/_generated/shard.ts +++ b/examples/chess/lunora/_generated/shard.ts @@ -1185,7 +1185,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -1195,6 +1195,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/examples/expo/lunora/_generated/functions.ts b/examples/expo/lunora/_generated/functions.ts index c2e01be01b..fa69733398 100644 --- a/examples/expo/lunora/_generated/functions.ts +++ b/examples/expo/lunora/_generated/functions.ts @@ -17,6 +17,12 @@ import type { Id } from "./dataModel.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/examples/expo/lunora/_generated/shard.ts b/examples/expo/lunora/_generated/shard.ts index 86df3af6c7..ffe61844e9 100644 --- a/examples/expo/lunora/_generated/shard.ts +++ b/examples/expo/lunora/_generated/shard.ts @@ -377,7 +377,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -387,6 +387,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/examples/feedback-board/lunora/_generated/functions.ts b/examples/feedback-board/lunora/_generated/functions.ts index 450368fab2..6f40ebd0d3 100644 --- a/examples/feedback-board/lunora/_generated/functions.ts +++ b/examples/feedback-board/lunora/_generated/functions.ts @@ -18,6 +18,12 @@ import type { Id } from "./dataModel.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/examples/feedback-board/lunora/_generated/shard.ts b/examples/feedback-board/lunora/_generated/shard.ts index 29ee7a515a..7f3ee015b4 100644 --- a/examples/feedback-board/lunora/_generated/shard.ts +++ b/examples/feedback-board/lunora/_generated/shard.ts @@ -843,7 +843,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -853,6 +853,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/examples/kanban-board/lunora/_generated/functions.ts b/examples/kanban-board/lunora/_generated/functions.ts index 2e8fbbab3f..c37110633b 100644 --- a/examples/kanban-board/lunora/_generated/functions.ts +++ b/examples/kanban-board/lunora/_generated/functions.ts @@ -17,6 +17,12 @@ import type { Id } from "./dataModel.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/examples/kanban-board/lunora/_generated/shard.ts b/examples/kanban-board/lunora/_generated/shard.ts index c023a275c9..5fa0b34cf4 100644 --- a/examples/kanban-board/lunora/_generated/shard.ts +++ b/examples/kanban-board/lunora/_generated/shard.ts @@ -432,7 +432,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -442,6 +442,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/examples/notify-demo/lunora/_generated/functions.ts b/examples/notify-demo/lunora/_generated/functions.ts index 520cb00e49..7a1690c2d6 100644 --- a/examples/notify-demo/lunora/_generated/functions.ts +++ b/examples/notify-demo/lunora/_generated/functions.ts @@ -17,6 +17,12 @@ import type { Id } from "./dataModel.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/examples/notify-demo/lunora/_generated/shard.ts b/examples/notify-demo/lunora/_generated/shard.ts index 72480e28cd..bd237b5bd5 100644 --- a/examples/notify-demo/lunora/_generated/shard.ts +++ b/examples/notify-demo/lunora/_generated/shard.ts @@ -611,7 +611,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -621,6 +621,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/examples/offline-rejections/lunora/_generated/functions.ts b/examples/offline-rejections/lunora/_generated/functions.ts index 718b6a7473..de46920e94 100644 --- a/examples/offline-rejections/lunora/_generated/functions.ts +++ b/examples/offline-rejections/lunora/_generated/functions.ts @@ -17,6 +17,12 @@ import type { Id } from "./dataModel.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/examples/offline-rejections/lunora/_generated/shard.ts b/examples/offline-rejections/lunora/_generated/shard.ts index 3389b551ab..0270d5e7d1 100644 --- a/examples/offline-rejections/lunora/_generated/shard.ts +++ b/examples/offline-rejections/lunora/_generated/shard.ts @@ -410,7 +410,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -420,6 +420,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/examples/payment-demo/lunora/_generated/functions.ts b/examples/payment-demo/lunora/_generated/functions.ts index af52b3a474..fc0edc8112 100644 --- a/examples/payment-demo/lunora/_generated/functions.ts +++ b/examples/payment-demo/lunora/_generated/functions.ts @@ -16,6 +16,12 @@ import type { ActionCtx, MutationCtx, QueryCtx } from "./server.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/examples/payment-demo/lunora/_generated/shard.ts b/examples/payment-demo/lunora/_generated/shard.ts index 962424afd2..49ce3fe651 100644 --- a/examples/payment-demo/lunora/_generated/shard.ts +++ b/examples/payment-demo/lunora/_generated/shard.ts @@ -1017,7 +1017,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -1027,6 +1027,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/examples/realtime-cursors/lunora/_generated/functions.ts b/examples/realtime-cursors/lunora/_generated/functions.ts index 1271174ea4..ce984435bd 100644 --- a/examples/realtime-cursors/lunora/_generated/functions.ts +++ b/examples/realtime-cursors/lunora/_generated/functions.ts @@ -17,6 +17,12 @@ import type { Id } from "./dataModel.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/examples/realtime-cursors/lunora/_generated/shard.ts b/examples/realtime-cursors/lunora/_generated/shard.ts index 5d9db8bbbe..a4b8911d68 100644 --- a/examples/realtime-cursors/lunora/_generated/shard.ts +++ b/examples/realtime-cursors/lunora/_generated/shard.ts @@ -629,7 +629,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -639,6 +639,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/examples/tanstack-start/lunora/_generated/functions.ts b/examples/tanstack-start/lunora/_generated/functions.ts index f86c70868d..81931d0d3a 100644 --- a/examples/tanstack-start/lunora/_generated/functions.ts +++ b/examples/tanstack-start/lunora/_generated/functions.ts @@ -17,6 +17,12 @@ import type { Id } from "./dataModel.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/examples/tanstack-start/lunora/_generated/shard.ts b/examples/tanstack-start/lunora/_generated/shard.ts index 15915fabfe..ebe17e892a 100644 --- a/examples/tanstack-start/lunora/_generated/shard.ts +++ b/examples/tanstack-start/lunora/_generated/shard.ts @@ -377,7 +377,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -387,6 +387,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/examples/team-chat/lunora/_generated/functions.ts b/examples/team-chat/lunora/_generated/functions.ts index f7651c621d..7dcd8f5bd6 100644 --- a/examples/team-chat/lunora/_generated/functions.ts +++ b/examples/team-chat/lunora/_generated/functions.ts @@ -20,6 +20,12 @@ import type { Id } from "./dataModel.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/examples/team-chat/lunora/_generated/shard.ts b/examples/team-chat/lunora/_generated/shard.ts index 81f777e023..edb4852fa4 100644 --- a/examples/team-chat/lunora/_generated/shard.ts +++ b/examples/team-chat/lunora/_generated/shard.ts @@ -926,7 +926,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -936,6 +936,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } diff --git a/examples/todo-app/lunora/_generated/functions.ts b/examples/todo-app/lunora/_generated/functions.ts index 572d666e8c..ea4e49bc7e 100644 --- a/examples/todo-app/lunora/_generated/functions.ts +++ b/examples/todo-app/lunora/_generated/functions.ts @@ -17,6 +17,12 @@ import type { Id } from "./dataModel.js"; export interface RegisteredLunoraFunction { kind: "action" | "mutation" | "query" | "stream"; args: Record; + /** + * Present on a `"stream"` declared `durable`: its run is persisted and + * survives the socket that opened it. Absent on every other kind, and on an + * ephemeral stream. + */ + durable?: { ttlMs?: number }; /** * For `"action" | "mutation" | "query"` the handler is awaited and its result returned. * For `"stream"` the handler returns an `AsyncIterable` synchronously and takes an diff --git a/examples/todo-app/lunora/_generated/shard.ts b/examples/todo-app/lunora/_generated/shard.ts index 605b42c007..62d5276781 100644 --- a/examples/todo-app/lunora/_generated/shard.ts +++ b/examples/todo-app/lunora/_generated/shard.ts @@ -515,7 +515,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt return { ranges: footprint.ranges(), result, tables: footprint.tables }; } - protected override executeStream(functionPath: string, args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable } { + protected override executeStream(functionPath: string, args: Record): null | { durable?: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable } { const registered = LUNORA_FUNCTIONS[functionPath]; if (!registered || registered.kind !== "stream" || registered.visibility === "internal") { @@ -525,6 +525,7 @@ export const createShardDO = (config: ShardDOConfig = {}): new (state: ShardDOSt this.ensureMigrated(); return { + ...(registered.durable ? { durable: registered.durable as { ttlMs?: number } } : {}), iterator: (signal) => (registered.handler as (context: unknown, args: Record, signal: AbortSignal) => AsyncIterable)(this.buildCtx({ functionPath }), args, signal), }; } From 2a01bd63d394c79b53f49c16e79d6d651ee2c418 Mon Sep 17 00:00:00 2001 From: Daniel Bannert Date: Tue, 11 Aug 2026 12:48:42 +0200 Subject: [PATCH 09/10] fix: address the automated review on the PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine comments; seven were right, two were not. The user turn was appended twice, and the first one landed before a parked run owned the thread — defeating the ordering the queue exists to provide, and contradicting the comment directly above it. Hoisting the call left the original in place. The keyed append meant no duplicate row, which is exactly why it was invisible. An unguarded store read after the ack could hang a consumer forever. `attach` reads `__stream_runs` after the host has already sent `ack`; on a pre-migration shard or a stub handle that throw escaped through a fire-and-forget catch, so the client held an acked stream that never settled and its host-side slot leaked. The attach now fails its sink, which is the one path that releases both. Anonymous callers shared one run. Falling back to a constant when the socket has no `userId` collapsed every anonymous caller onto one key — the second caller read the first one's transcript and the handler never ran for them. That is the same leak the identity scope closes for signed-in users, on the path with no identity to check. Keyed on the client's own stable id instead, so anonymous callers stay separate and can still resume their own run across a reload. The chunk ceiling bounded the wrong thing: 50,000 chunks of 1 MiB is ~50 GiB against a 10 GiB per-DO limit, so the cap could not prevent what its comment claimed. A byte ceiling now trips alongside it. A token budget on a sharded limit still threw. `getValue` reports the raw config while `limit` enforces `capacity / shards`, so clamping to the raw value exceeded the enforced capacity — leaving the oversized call uncharged, the exact escape the clamp was added to close. A RAG source id derived from a mutable column orphaned its chunks: when the id moved, the old ones stayed indexed and retrievable forever. Eval statistics were computed from each bucket's LAST sample, so a bucket holding 0 and 1 reported a minimum of 1 — wrong, and wrong in the flattering direction. Buckets carry min/max/sum/count for this. Declined: the `exports` map is not alphabetical and never was (`./skill-markdown` precedes `./sandbox`), and `vis sort-package-json` accepts it — export order is semantic, not cosmetic. The advisor-flag comment concerns another session's file. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018hGhKLpXkFj3aWUYhZmWQv --- api-snapshots/shard-engine.api.md | 6 ++ .../src/content/docs/concepts/components.mdx | 6 +- packages/agent/src/agent-loop.ts | 12 ++-- packages/ai/__tests__/rag/sync.test.ts | 18 +++++ packages/ai/src/rag/sync.ts | 21 ++++-- .../__tests__/shard-do.durable-stream.test.ts | 51 ++++++++++++--- packages/do/src/shard-do.ts | 33 +++++++++- .../ratelimit/__tests__/token-budget.test.ts | 13 ++++ packages/ratelimit/src/token-budget.ts | 26 ++++++-- .../shard-engine/src/durable-stream-runner.ts | 65 +++++++++++++++---- packages/shard-engine/src/index.ts | 2 +- .../features/evals/eval-scores.test.ts | 7 +- .../studio/src/features/evals/eval-scores.ts | 27 +++++--- 13 files changed, 230 insertions(+), 57 deletions(-) diff --git a/api-snapshots/shard-engine.api.md b/api-snapshots/shard-engine.api.md index 442f40fb58..1dd1f86b51 100644 --- a/api-snapshots/shard-engine.api.md +++ b/api-snapshots/shard-engine.api.md @@ -1187,6 +1187,12 @@ const MAIL_RETENTION = 500; const MAIL_TABLE = "__lunora_mail"; ``` +### `MAX_DURABLE_STREAM_BYTES` (const) + +```ts +const MAX_DURABLE_STREAM_BYTES: number; +``` + ### `MAX_DURABLE_STREAM_CHUNKS` (const) ```ts diff --git a/apps/docs/src/content/docs/concepts/components.mdx b/apps/docs/src/content/docs/concepts/components.mdx index 5daa7b30e1..85957e2c7a 100644 --- a/apps/docs/src/content/docs/concepts/components.mdx +++ b/apps/docs/src/content/docs/concepts/components.mdx @@ -41,7 +41,8 @@ export const voting = defineComponent("voting", { }, // Optional: everything after `.use(voting.middleware)` sees `ctx.voting`. - middleware: ({ ctx, next }) => next({ ctx: { ...ctx, voting: makeVotingApi(ctx) } }), + // Anything you put here shows up as `ctx.voting` for procedures that opt in. + middleware: ({ ctx, next }) => next({ ctx: { ...ctx, voting: { subject: (id: string) => `voting:${id}` } } }), }); ``` @@ -56,12 +57,15 @@ Three parts, all optional except the name: ## Installing one ```ts title="lunora/schema.ts" +import { defineSchema, defineTable, v } from "@lunora/server"; import { voting } from "my-component"; export const schema = defineSchema({ posts: defineTable({ title: v.string() }) }).extend(voting.extension); ``` ```ts title="lunora/voting.ts" +import { voting } from "my-component"; + // One line: re-exporting is what puts the component's functions in your API. export const { castVote, tally } = voting.functions; ``` diff --git a/packages/agent/src/agent-loop.ts b/packages/agent/src/agent-loop.ts index a5df8c94e5..d10118b92e 100644 --- a/packages/agent/src/agent-loop.ts +++ b/packages/agent/src/agent-loop.ts @@ -1034,17 +1034,17 @@ const runAgentLoop = async (options: AgentLoopOptions): Promise await run(patchThread, { key: params.threadKey, ...patch }); }; + /** The user turn that opens the run. Runs after any queue wait, so a parked run appends only once it owns the thread. */ + const persistUserTurn = async (): Promise => { + await persist({ content: params.input, messageKey: `${instanceId}:user`, role: "user" }); + }; + /** * End the run: write the terminal status and, in the same mutation, hand the * thread to the next queued run. Only then is the dequeued run woken — it * already owns the thread by that point, so there is no window in which two * runs believe they may append. */ - /** The user turn that opens the run. Runs after any queue wait, so a parked run appends only once it owns the thread. */ - const persistUserTurn = async (): Promise => { - await persist({ content: params.input, messageKey: `${instanceId}:user`, role: "user" }); - }; - const finishRun = async (patch: { error?: string; status: "error" | "idle"; usage?: AgentUsage }): Promise => { const outcome = (await run(completeRun, { instanceId, key: params.threadKey, ...patch })) as { dequeued?: string } | undefined; @@ -1088,8 +1088,6 @@ const runAgentLoop = async (options: AgentLoopOptions): Promise await terminatePriorInstance(env, exportName, bootstrap.priorInstanceId); } - await persistUserTurn(); - // Memory step: dispatch the configured retrieval action once per run and // inject the assembled context into every turn's prompt. const memoryContext = await retrieveMemoryContext(agent, params.input, params.owner, paths, step, run); diff --git a/packages/ai/__tests__/rag/sync.test.ts b/packages/ai/__tests__/rag/sync.test.ts index 664d9b19c3..64309779ee 100644 --- a/packages/ai/__tests__/rag/sync.test.ts +++ b/packages/ai/__tests__/rag/sync.test.ts @@ -60,6 +60,24 @@ describe("ragSyncTriggers", () => { expect(scheduled[1]?.args).toStrictEqual({ deleted: true, id: "doc-2" }); }); + it("removes the old chunks when a derived source id moves", async () => { + expect.assertions(2); + + const { context, scheduled } = fakeContext(); + const sync = ragSyncTriggers({ + action: ACTION, + id: (document) => `slug:${String(document["slug"])}`, + text: (document) => document["body"] as string, + }); + + // The slug changed, so the chunks under the old id would otherwise stay + // indexed and retrievable forever. + await sync.afterUpdate(context, { doc: { body: "same", slug: "new" }, id: "row-1", previous: { body: "same", slug: "old" } }); + + expect(scheduled[0]?.args).toStrictEqual({ deleted: true, id: "slug:old" }); + expect(scheduled[1]?.args).toStrictEqual({ id: "slug:new", text: "same" }); + }); + it("honours a custom source id and delay", async () => { expect.assertions(1); diff --git a/packages/ai/src/rag/sync.ts b/packages/ai/src/rag/sync.ts index f977b74dec..4e7759a490 100644 --- a/packages/ai/src/rag/sync.ts +++ b/packages/ai/src/rag/sync.ts @@ -133,16 +133,25 @@ const ragSyncTriggers = = Record { - const text = textOf(event.doc); - const before = textOf(event.previous); - - // An edit that didn't touch the indexed text costs nothing: no action - // dispatch, no embedding call, no vector write. - if (text === before || event.doc === undefined) { + if (event.doc === undefined) { return; } + const text = textOf(event.doc); + const before = textOf(event.previous); const id = sourceId(event.doc, event.id); + // `options.id` may derive from a mutable column (a slug, a version). + // When it moves, the chunks under the OLD id are orphaned — indexed + // forever, and retrievable — unless they are removed explicitly. + const previousId = event.previous === undefined ? id : sourceId(event.previous, event.id); + + if (previousId !== id) { + await schedule(context, { deleted: true, id: previousId }); + } else if (text === before) { + // Same id, same text: an edit that touched neither costs nothing — + // no action dispatch, no embedding call, no vector write. + return; + } // The text went away (cleared, or the row no longer qualifies): the // old chunks must go too, or retrieval keeps serving deleted content. diff --git a/packages/do/__tests__/shard-do.durable-stream.test.ts b/packages/do/__tests__/shard-do.durable-stream.test.ts index 4d35c479c8..a4752c47eb 100644 --- a/packages/do/__tests__/shard-do.durable-stream.test.ts +++ b/packages/do/__tests__/shard-do.durable-stream.test.ts @@ -138,7 +138,7 @@ describe("shardDO durable streams", () => { const first = createFakeWebSocket(); - shard.registerSocket(first, { subs: {} }); + shard.registerSocket(first, { clientId: "client-a", subs: {} }); await shard.driveMessage(first, { id: "stream_1", query: { functionPath: "chat:answer" }, type: "stream" }); await waitForTerminator(first); @@ -152,7 +152,7 @@ describe("shardDO durable streams", () => { // A reload: same args, same run, resuming after the first chunk. const resumed = createFakeWebSocket(); - shard.registerSocket(resumed, { subs: {} }); + shard.registerSocket(resumed, { clientId: "client-a", subs: {} }); await shard.driveMessage(resumed, { id: "stream_9", query: { functionPath: "chat:answer" }, sinceChunk: 1, type: "stream" }); await waitForTerminator(resumed); @@ -181,8 +181,8 @@ describe("shardDO durable streams", () => { const a = createFakeWebSocket(); const b = createFakeWebSocket(); - shard.registerSocket(a, { subs: {} }); - shard.registerSocket(b, { subs: {} }); + shard.registerSocket(a, { clientId: "client-a", subs: {} }); + shard.registerSocket(b, { clientId: "client-a", subs: {} }); await shard.driveMessage(a, { id: "stream_a", query: { functionPath: "chat:answer" }, type: "stream" }); // Let the producer emit its first chunk before the second client attaches. @@ -223,7 +223,7 @@ describe("shardDO durable streams", () => { const first = createFakeWebSocket(); - shard.registerSocket(first, { subs: {} }); + shard.registerSocket(first, { clientId: "client-a", subs: {} }); await shard.driveMessage(first, { id: "stream_1", query: { functionPath: "chat:answer" }, type: "stream" }); await new Promise((resolve) => { setTimeout(resolve, 5); @@ -239,7 +239,7 @@ describe("shardDO durable streams", () => { // tail would have to be re-generated, which duplicates. const resumed = createFakeWebSocket(); - revived.registerSocket(resumed, { subs: {} }); + revived.registerSocket(resumed, { clientId: "client-a", subs: {} }); await revived.driveMessage(resumed, { id: "stream_2", query: { functionPath: "chat:answer" }, sinceChunk: 1, type: "stream" }); await waitForTerminator(resumed); @@ -254,7 +254,7 @@ describe("shardDO durable streams", () => { const fresh = createFakeWebSocket(); - revived.registerSocket(fresh, { subs: {} }); + revived.registerSocket(fresh, { clientId: "client-a", subs: {} }); await revived.driveMessage(fresh, { id: "stream_3", query: { functionPath: "chat:answer" }, type: "stream" }); await waitForTerminator(fresh); @@ -280,7 +280,7 @@ describe("shardDO durable streams", () => { const first = createFakeWebSocket(); - shard.registerSocket(first, { subs: {} }); + shard.registerSocket(first, { clientId: "client-a", subs: {} }); await shard.driveMessage(first, { id: "stream_1", query: { functionPath: "chat:answer" }, type: "stream" }); await waitForTerminator(first); @@ -289,7 +289,7 @@ describe("shardDO durable streams", () => { // response cache with a 24-hour TTL, including for a failed run. const later = createFakeWebSocket(); - shard.registerSocket(later, { subs: {} }); + shard.registerSocket(later, { clientId: "client-a", subs: {} }); await shard.driveMessage(later, { id: "stream_2", query: { functionPath: "chat:answer" }, type: "stream" }); await waitForTerminator(later); @@ -301,6 +301,39 @@ describe("shardDO durable streams", () => { expect(shard.starts).toBe(2); }); + it("never shares a run between two anonymous callers", async () => { + expect.assertions(2); + + const shard = new DurableStreamShard(state, {}); + let answers = 0; + + shard.registered.set("chat:answer", async function* answer() { + answers += 1; + yield `answer-${String(answers)}`; + }); + + const first = createFakeWebSocket(); + const second = createFakeWebSocket(); + + // No `userId` on either socket. Collapsing both onto one key would hand + // the second caller the first one's transcript without ever running the + // handler — the same leak the identity scope closes for signed-in users. + shard.registerSocket(first, { clientId: "client-1", subs: {} }); + shard.registerSocket(second, { clientId: "client-2", subs: {} }); + + await shard.driveMessage(first, { id: "stream_1", query: { functionPath: "chat:answer" }, type: "stream" }); + await waitForTerminator(first); + await shard.driveMessage(second, { id: "stream_2", query: { functionPath: "chat:answer" }, type: "stream" }); + await waitForTerminator(second); + + expect(shard.starts).toBe(2); + expect( + parseFrames(second) + .filter((frame) => frame.type === "chunk") + .map((frame) => frame.data), + ).toStrictEqual(["answer-2"]); + }); + it("never shares a run across identities", async () => { expect.assertions(2); diff --git a/packages/do/src/shard-do.ts b/packages/do/src/shard-do.ts index 96b75b5a2d..28ad81fc1e 100644 --- a/packages/do/src/shard-do.ts +++ b/packages/do/src/shard-do.ts @@ -727,6 +727,26 @@ const IDEMPOTENCY_RETENTION_MS = 86_400_000; */ const IDEMPOTENCY_GC_INTERVAL_MS = 3_600_000; +/** + * The run-key component for a caller with no verified identity. + * + * Anonymous callers must not share a transcript with each other — collapsing + * them onto a constant hands the second caller the first one's answer without + * ever running the handler. But they should still resume their OWN run across a + * reload, which is the whole point of a durable stream, so the key has to be + * something stable across sockets. + * + * The client's own `clientId` (the stable id it sends on `connect`, already + * trusted for the mutation watermark) is that. It is client-supplied, so it + * separates honest callers rather than defending against a hostile one — which + * is the right bar here: `userId` takes precedence whenever the socket has an + * identity to check, so this path only ever governs anonymous traffic sharing + * its own answers. A client that sent none falls back to its connection, which + * isolates correctly and simply cannot resume. + */ +const anonymousStreamCaller = (attachment: SocketAttachment, streamId: string): string => + attachment.clientId === undefined ? `conn:${attachment.connectionId ?? streamId}` : `client:${attachment.clientId}`; + /** * The stream wire vocabulary, bound to one socket + stream id. * @@ -7661,8 +7681,17 @@ abstract class ShardDO { registration: { durable: { ttlMs?: number }; iterator: (signal: AbortSignal) => AsyncIterable }, sinceChunk: number, ): Promise { - const owner = this.readAttachment(ws).userId ?? ""; - const runKey = `${owner}\u0000${functionPath}:${stableWireKey(args)}`; + const attachment = this.readAttachment(ws); + // An ANONYMOUS caller shares nothing. Falling back to a constant here + // collapsed every anonymous caller onto one key, so the second caller + // read the first one's transcript and the handler never ran for them — + // the same cross-caller leak the identity scope closes for signed-in + // users, on the path where there is no identity to check. The socket's + // connection id keeps each one to its own run; the cost is that an + // anonymous reload starts over, which is the honest answer when nothing + // durable identifies the caller. + const caller = attachment.userId ?? anonymousStreamCaller(attachment, id); + const runKey = `${caller}\u0000${functionPath}:${stableWireKey(args)}`; const cancellers = socketMap(this.streamCancellers, ws); const controller = new AbortController(); const frames = streamFrames(ws, id); diff --git a/packages/ratelimit/__tests__/token-budget.test.ts b/packages/ratelimit/__tests__/token-budget.test.ts index 0460071ce0..27788adfb5 100644 --- a/packages/ratelimit/__tests__/token-budget.test.ts +++ b/packages/ratelimit/__tests__/token-budget.test.ts @@ -48,6 +48,19 @@ describe("tokenBudget", () => { await expect(budget.check("user-x")).resolves.toMatchObject({ ok: false }); }); + it("clamps to the per-shard capacity a sharded limit actually enforces", async () => { + expect.assertions(2); + + // 4 shards → each bucket enforces 1000/4 = 250. Clamping to the raw 1000 + // would exceed the enforced capacity and throw, leaving the call + // uncharged — the exact escape this clamp exists to prevent. + const sharded = new RateLimiter({ config: { tokens: { capacity: 1000, kind: "token bucket", period: 60_000, rate: 1000, shards: 4 } } }); + const budget = tokenBudget(sharded, "tokens"); + + await expect(budget.record("user-s", 5000)).resolves.toMatchObject({ ok: true }); + await expect(budget.check("user-s")).resolves.toMatchObject({ ok: false }); + }); + it("keeps budgets separate per key", async () => { expect.assertions(1); diff --git a/packages/ratelimit/src/token-budget.ts b/packages/ratelimit/src/token-budget.ts index 65dc651ef0..dfa308acea 100644 --- a/packages/ratelimit/src/token-budget.ts +++ b/packages/ratelimit/src/token-budget.ts @@ -82,15 +82,27 @@ const tokenBudget = (limiter: RateLimiter, name: Na return limiter.check(name, { key }); } - // Clamp to the bucket's capacity. The limiter refuses a count larger - // than capacity outright, and for a token budget that is exactly - // backwards: a single long-context call over a per-window budget is - // the ordinary case, and throwing would charge it NOTHING — the one - // call the budget exists to catch would escape it entirely. Charging - // the whole bucket is the strongest true statement available. + // Clamp to the capacity the limiter will actually enforce. It refuses + // a count larger than capacity outright, and for a token budget that + // is exactly backwards: a single long-context call over a per-window + // budget is the ordinary case, and throwing would charge it NOTHING — + // the one call the budget exists to catch would escape it entirely. + // Charging the whole bucket is the strongest true statement available. + // + // `getValue` reports the RAW config, while a sharded limit enforces + // `capacity / shards` on the one bucket this key routes to. Clamping + // to the raw value would still throw on a sharded budget, so the cap + // is derived per shard, floored to the positive integer `limit` + // requires. const { config } = await limiter.getValue(name, { key }); + // `limit` only accepts a positive integer, so the floor cannot go + // below 1. A limit whose per-shard capacity is under one token is + // unusable by ANY caller — `check` throws on it too — so that is a + // configuration error to fix at the limit, not something a charge + // made after the fact can absorb. + const capacity = Math.max(1, Math.floor((config.capacity ?? config.rate) / (config.shards ?? 1))); - return limiter.limit(name, { count: Math.min(Math.ceil(tokens), config.capacity ?? config.rate), key, reserve: true }); + return limiter.limit(name, { count: Math.min(Math.ceil(tokens), capacity), key, reserve: true }); }, }; }; diff --git a/packages/shard-engine/src/durable-stream-runner.ts b/packages/shard-engine/src/durable-stream-runner.ts index bd059838df..18149b7143 100644 --- a/packages/shard-engine/src/durable-stream-runner.ts +++ b/packages/shard-engine/src/durable-stream-runner.ts @@ -31,6 +31,16 @@ import { appendStreamChunk, claimStreamRun, deleteStreamRun, finishStreamRun, re */ const MAX_DURABLE_STREAM_CHUNKS = 50_000; +/** + * Hard ceiling on the BYTES one run may persist. + * + * The chunk count alone bounds nothing that matters: 50,000 chunks of 1 MiB is + * ~50 GiB against a per-DO ceiling of 10 GiB, and this store shares the shard's + * SQLite with application data. 64 MiB is far past any token stream and far + * under the point where one run threatens the shard. + */ +const MAX_DURABLE_STREAM_BYTES = 64 * 1024 * 1024; + /** Default retention for a finished transcript: long enough to survive a reload and a commute. */ const DEFAULT_TTL_MS = 86_400_000; @@ -122,6 +132,37 @@ class DurableStreamRunner { * neither replayed nor delivered live. */ public async attach(request: DurableStreamAttach): Promise { + try { + await this.attachOrThrow(request); + } catch (error: unknown) { + // The host has already acked by the time this runs, so a throw that + // escapes leaves the consumer waiting on a stream that can never + // settle — the store read is the likely source (a pre-migration + // shard, a stub handle). Fail the sink instead: it is the only path + // that releases the consumer AND its host-side slot. + const { body, redacted } = toErrorBody(error, { fallbackCode: "INTERNAL_SERVER_ERROR", redactedMessage: "internal error" }); + + if (redacted) { + // eslint-disable-next-line no-console -- server-side diagnostic for an internal/unhandled attach error + console.error("[@lunora/shard-engine] durable stream attach failed:", error); + } + + request.sink.fail({ code: body.code, message: body.message }); + } + } + + /** Detach a consumer without disturbing the producer — cancelling a durable stream leaves the run going. */ + public detach(runKey: string, sink: DurableStreamSink): void { + this.runs.get(runKey)?.sinks.delete(sink); + } + + /** Whether a run is currently producing on this instance. */ + public isLive(runKey: string): boolean { + return this.runs.has(runKey); + } + + /** The attach state machine proper — see {@link DurableStreamRunner.attach}, which owns its failure path. */ + private async attachOrThrow(request: DurableStreamAttach): Promise { const sql = this.sql(); const { runKey, sinceChunk, sink } = request; const run = readStreamRun(sql, runKey); @@ -189,16 +230,6 @@ class DurableStreamRunner { await producing; } - /** Detach a consumer without disturbing the producer — cancelling a durable stream leaves the run going. */ - public detach(runKey: string, sink: DurableStreamSink): void { - this.runs.get(runKey)?.sinks.delete(sink); - } - - /** Whether a run is currently producing on this instance. */ - public isLive(runKey: string): boolean { - return this.runs.has(runKey); - } - /** * Drive one run: persist each chunk under the next `seq`, then fan it out to * whichever consumers are attached right now — possibly none, since a run @@ -211,22 +242,28 @@ class DurableStreamRunner { // run. The controller exists to satisfy the handler's signature. const controller = new AbortController(); let seq = 0; + let bytes = 0; try { for await (const chunk of iterator(controller.signal)) { const data = encodeWire(chunk); + const encoded = JSON.stringify(data); seq += 1; + bytes += encoded.length; - if (seq > MAX_DURABLE_STREAM_CHUNKS) { + // Whichever ceiling trips first. The count catches a chatty + // generator, the byte total catches a few enormous chunks — and + // only the second one actually bounds what lands in SQLite. + if (seq > MAX_DURABLE_STREAM_CHUNKS || bytes > MAX_DURABLE_STREAM_BYTES) { throw new LunoraError( "STREAM_TOO_LONG", - `durable stream exceeded ${String(MAX_DURABLE_STREAM_CHUNKS)} chunks; yield larger chunks or drop \`durable\``, + `durable stream exceeded its ceiling (${String(MAX_DURABLE_STREAM_CHUNKS)} chunks / ${String(MAX_DURABLE_STREAM_BYTES)} bytes); yield less, or drop \`durable\``, { status: 507 }, ); } - appendStreamChunk(sql, runKey, seq, JSON.stringify(data)); + appendStreamChunk(sql, runKey, seq, encoded); for (const sink of state.sinks) { // A consumer that refused the frame is gone: drop it rather @@ -284,4 +321,4 @@ class DurableStreamRunner { } export type { DurableAttachDecision, DurableStreamAttach, DurableStreamSink }; -export { decideDurableAttach, DurableStreamRunner, MAX_DURABLE_STREAM_CHUNKS }; +export { decideDurableAttach, DurableStreamRunner, MAX_DURABLE_STREAM_BYTES, MAX_DURABLE_STREAM_CHUNKS }; diff --git a/packages/shard-engine/src/index.ts b/packages/shard-engine/src/index.ts index 04b97ed700..3b6a35c568 100644 --- a/packages/shard-engine/src/index.ts +++ b/packages/shard-engine/src/index.ts @@ -118,7 +118,7 @@ export { trimStreamRuns, } from "./durable-stream"; export type { DurableAttachDecision, DurableStreamAttach, DurableStreamSink } from "./durable-stream-runner"; -export { decideDurableAttach, DurableStreamRunner, MAX_DURABLE_STREAM_CHUNKS } from "./durable-stream-runner"; +export { decideDurableAttach, DurableStreamRunner, MAX_DURABLE_STREAM_BYTES, MAX_DURABLE_STREAM_CHUNKS } from "./durable-stream-runner"; export type { ExternalSourceDiffResult } from "./external-source-diff"; export { diffExternalSource } from "./external-source-diff"; export { liftSourceId, normalizeSourceDocument, normalizeSourceValue } from "./external-source-lift"; diff --git a/packages/studio/__tests__/features/evals/eval-scores.test.ts b/packages/studio/__tests__/features/evals/eval-scores.test.ts index e5881cfbc2..133f445afb 100644 --- a/packages/studio/__tests__/features/evals/eval-scores.test.ts +++ b/packages/studio/__tests__/features/evals/eval-scores.test.ts @@ -67,7 +67,9 @@ describe("eval score extraction", () => { kind: "gauge", name: "gen_ai.evaluation.tone.score", points: [ - { bucketMs: 0, count: 1, last: 0.2, max: 0.2, min: 0.2, sum: 0.2 }, + // Two scores in one bucket: `last` alone would report a + // minimum of 0.4 and a mean of 0.4, flattering both. + { bucketMs: 0, count: 2, last: 0.4, max: 0.4, min: 0, sum: 0.4 }, { bucketMs: 60_000, count: 1, last: 1, max: 1, min: 1, sum: 1 }, ], }, @@ -77,7 +79,8 @@ describe("eval score extraction", () => { const [withTrend] = buildEvalCards(runs, extractEvalTrends(history)); // Durable half wins the summary: the ring's single 0.5 is not the history. - expect(withTrend).toMatchObject({ latest: 1, max: 1, min: 0.2, name: "tone" }); + // mean = (0.4 + 1) / (2 + 1) samples, min from the bucket's own `min`. + expect(withTrend).toMatchObject({ latest: 1, max: 1, mean: 1.4 / 3, min: 0, name: "tone" }); expect(withTrend?.points).toHaveLength(2); // Its live runs still ride along — that is what makes a score clickable. expect(withTrend?.runs).toHaveLength(1); diff --git a/packages/studio/src/features/evals/eval-scores.ts b/packages/studio/src/features/evals/eval-scores.ts index c3c14bc848..a8b43983c3 100644 --- a/packages/studio/src/features/evals/eval-scores.ts +++ b/packages/studio/src/features/evals/eval-scores.ts @@ -10,6 +10,12 @@ */ import type { MetricHistoryPoint, MetricHistoryResult, TraceSummary } from "../../lib/admin"; +/** Min/mean/max over raw scores — the live-ring fallback when no durable buckets exist. */ +const summarise = (values: ReadonlyArray): undefined | { max: number; mean: number; min: number } => + values.length === 0 + ? undefined + : { max: Math.max(...values), mean: values.reduce((total, value) => total + value, 0) / values.length, min: Math.min(...values) }; + /** `gen_ai.evaluation..score` — the name is the capture group. */ const SCORE_KEY = /^gen_ai\.evaluation\.(.+)\.score$/u; @@ -109,18 +115,23 @@ export const buildEvalCards = (runs: ReadonlyArray, trends: ReadonlyArr for (const name of names) { const ownRuns = runs.filter((run) => run.name === name); const points = trends.find((trend) => trend.name === name)?.points ?? []; - const values = points.length > 0 ? points.map((point) => point.last) : ownRuns.map((run) => run.score); const latest = points.at(-1)?.last ?? ownRuns[0]?.score; + // A bucket is an aggregate, not a sample: `last` is only its final score. + // Summarising a bucket holding 0 and 1 by its `last` reports a minimum of + // 1 and a mean of 1 — both wrong, and wrong in the flattering direction. + // The bucket carries `min`/`max`/`sum`/`count` precisely for this. + const summary = + points.length > 0 + ? { + max: Math.max(...points.map((point) => point.max)), + mean: points.reduce((total, point) => total + point.sum, 0) / points.reduce((total, point) => total + point.count, 0), + min: Math.min(...points.map((point) => point.min)), + } + : summarise(ownRuns.map((run) => run.score)); cards.push({ ...(latest === undefined ? {} : { latest }), - ...(values.length === 0 - ? {} - : { - max: Math.max(...values), - mean: values.reduce((total, value) => total + value, 0) / values.length, - min: Math.min(...values), - }), + ...summary, name, points, runs: ownRuns, From 6b20985b45870b27a595a2d574f6a54ae9d235c8 Mon Sep 17 00:00:00 2001 From: Daniel Bannert Date: Tue, 11 Aug 2026 13:00:33 +0200 Subject: [PATCH 10/10] fix(scripts): reject a malformed plugin manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `JSON.parse` returns `null`, arrays, and strings as happily as objects, and the lockstep check fed the result straight into field comparisons. Each malformed shape defeated it differently: `null` threw a raw TypeError out of a script that runs from `postinstall` — which surfaces as every CI job failing in its setup step, with the cause invisible in the job that reports it — while an array, a string, or `{}` left every compared field `undefined` on both sides, so the manifests "matched" and a check whose entire job is catching a malformed manifest passed one. Each parsed manifest is now required to be a non-null, non-array object carrying every field the comparison reads. Verified by feeding the real check all four shapes. The closing summary also described the manifest half as agreeing with `packages/cli/skills`; it compares the two manifests with each other, so it pointed maintainers at the wrong files. Both from the automated review of #405, whose commit this branch now carries. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018hGhKLpXkFj3aWUYhZmWQv --- scripts/check-skill-mirrors.js | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/scripts/check-skill-mirrors.js b/scripts/check-skill-mirrors.js index 31640f5411..6efff36eb7 100644 --- a/scripts/check-skill-mirrors.js +++ b/scripts/check-skill-mirrors.js @@ -180,13 +180,38 @@ const checkPluginManifestLockstep = () => { const manifests = {}; for (const [host, path] of Object.entries(paths)) { + const relative = path.slice(rootDir.length + 1); + let parsed; + try { - manifests[host] = JSON.parse(readFileSync(path, "utf8")); + parsed = JSON.parse(readFileSync(path, "utf8")); } catch (error) { - fail(`${path.slice(rootDir.length + 1)} is missing or not valid JSON (${error.message})`); + fail(`${relative} is missing or not valid JSON (${error.message})`); + + return; + } + + // `JSON.parse` happily returns `null`, an array, or a string. Each one + // defeats the comparison below in its own way: `null` throws a raw + // TypeError out of this script — and because it runs from `postinstall`, + // that reads as every CI job failing in its setup step — while an array, + // a string, or `{}` makes every compared field `undefined` on BOTH sides, + // so a malformed manifest passes a check whose whole job is to catch one. + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail(`${relative} must be a JSON object describing the plugin`); return; } + + const missing = [...LOCKSTEP_FIELDS, "author"].filter((field) => parsed[field] === undefined); + + if (missing.length > 0) { + fail(`${relative} is missing required field(s): ${missing.join(", ")}`); + + return; + } + + manifests[host] = parsed; } for (const field of LOCKSTEP_FIELDS) { @@ -209,7 +234,7 @@ const mirrorList = MIRRORS.map(({ dir }) => dir.slice(rootDir.length + 1)).join( if (hasFailure) { console.error( - `\ncheck-skill-mirrors: fix the problems above — the mirrors (${mirrorList}) and the plugin manifests must agree with packages/cli/skills.\n`, + `\ncheck-skill-mirrors: fix the problems above — the mirrors (${mirrorList}) must agree with packages/cli/skills, and the two plugin manifests must agree with each other.\n`, ); process.exit(1); }